diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml index edf0bc57a33b..1c0d284492cf 100644 --- a/.github/workflows/gaia.yml +++ b/.github/workflows/gaia.yml @@ -6,21 +6,22 @@ on: branches: - main paths: - - 'research/gaia/**' + - 'research/**' - '.github/workflows/gaia.yml' - - '!research/gaia/**.md' + - '!research/**.md' pull_request: branches: - main paths: - - 'research/gaia/**' + - 'research/**' - '.github/workflows/gaia.yml' - - '!research/gaia/**.md' + - '!research/**.md' jobs: gaia-test: - runs-on: ubuntu-20.04 - if: false + runs-on: self-hosted + container: + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.21 defaults: run: shell: bash --noprofile --norc -eo pipefail {0} @@ -31,40 +32,50 @@ jobs: - name: Install Dependencies run: | - # Due to an observation of changing hostname in github runners, - # append 127.0.0.1 to etc/hosts to avoid DNS lookup. - r=`cat /etc/hosts | grep $(hostname) || true` - if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi - cat /etc/hosts + # install cargo cause github change $HOME to /github/home, so we can't use + # `source ~/.bashrc` or `source ${HOME}/.bashrc` to find cargo command. + sudo chown -R $(id -u):$(id -g) ${HOME} + sudo chmod -R a+wrx ${HOME} ${GITHUB_WORKSPACE} + wget --no-verbose https://golang.org/dl/go1.15.5.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.15.5.linux-amd64.tar.gz + curl -sf -L https://static.rust-lang.org/rustup.sh | sh -s -- -y --profile minimal --default-toolchain 1.54.0 + echo "source ~/.cargo/env" >> ~/.bashrc + source ${HOME}/.bashrc + rustup component add rustfmt + # install rdkafka + sudo yum install -y librdkafka-devel + # install cppkafka + git clone -b 0.4.0 --single-branch --depth=1 \ + https://github.com/mfontanini/cppkafka.git /tmp/cppkafka && \ + cd /tmp/cppkafka && git submodule update --init && \ + mkdir -p build && cd build && \ + cmake .. && make -j && sudo make install && \ + rm -fr /tmp/cppkafka - # install dependencies - ${GITHUB_WORKSPACE}/scripts/install_deps.sh --dev --verbose - - - name: Build GraphScope + - name: Build Ir on Experimental Store run: | - source ${HOME}/.graphscope_env - make BUILD_TYPE=debug + source ${HOME}/.bashrc + cd ${GITHUB_WORKSPACE}/research/query_service/ir/compiler + make build - - name: Build MaxGraph Store + - name: Build Ir on Groot Store run: | - source ${HOME}/.graphscope_env + source ${HOME}/.bashrc cd ${GITHUB_WORKSPACE}/interactive_engine mvn clean install -DskipTests -Pv2 - - name: Gaia test on MaxGraph Store + - name: Ir Unit Test run: | - source ${HOME}/.graphscope_env - cd ${GITHUB_WORKSPACE}/interactive_engine/gaia-adaptor && ./gremlin_test.sh + source ${HOME}/.bashrc + cd ${GITHUB_WORKSPACE}/research/query_service/ir/compiler && make test - - name: Run Backup Test on Maxgraph Store + - name: Ir Integration Test on Experimental Store run: | - source ${HOME}/.graphscope_env - cd ${GITHUB_WORKSPACE}/interactive_engine/sdk && ./backup_test.sh + source ${HOME}/.bashrc + cd ${GITHUB_WORKSPACE}/research/query_service/ir/compiler && ./ir_exprimental_ci.sh - - name: Gaia runtime test + - name: Ir Integration Test on Groot Store run: | - source ${HOME}/.graphscope_env + source ${HOME}/.bashrc + cd ${GITHUB_WORKSPACE}/interactive_engine/ir-adaptor && ./ir_groot_ci.sh - cd ${GITHUB_WORKSPACE}/research/query_service/gremlin/gremlin_core/tests - sh prepare_all_test_plan.sh - cargo test diff --git a/.github/workflows/pegasus.yml b/.github/workflows/pegasus.yml new file mode 100644 index 000000000000..618fbd67c15a --- /dev/null +++ b/.github/workflows/pegasus.yml @@ -0,0 +1,84 @@ +name: GraphScope Pegasus CI + +on: + push: + branches: + - gaia-x + paths: + - 'research/engine/pegasus/**' + - '.github/workflows/pegasus.yml' + + pull_request: + branches: + - gaia-x + paths: + - 'research/engine/pegasus/**' + - '.github/workflows/pegasus.yml' + +env: + CARGO_TERM_COLOR: always + +jobs: + peagsus-build: + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash --noprofile --norc -eo pipefail {0} + + steps: + - uses: actions/checkout@v2 + + - name: Build & Test + run: | + echo $(pwd) + cd research/engine/pegasus + cargo build --verbose + cargo test --verbose + + - name: Run example + run: | + cd research/engine/pegasus + + cargo build --release --examples + + # Run word_count_toy + target/release/examples/word_count_toy + + target/release/examples/word_count_toy -w 2 + + target/release/examples/word_count_toy -w 4 + + # Run logistic regression + target/release/examples/logistic_regression --data pegasus/examples/data/binary.csv + + k8s-test: + runs-on: self-hosted + defaults: + run: + shell: bash --noprofile --norc -eo pipefail {0} + + steps: + - uses: actions/checkout@v2 + + - name : Prepare Image + run: | + sudo docker pull registry.cn-hongkong.aliyuncs.com/graphscope/pegasus-base:latest + sudo docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/pegasus:${{ github.sha }} \ + --network=host \ + -f research/engine/pegasus/test/k8s/pegasus.Dockerfile . + echo " - name: CONFIG_MAP_NAME" >> research/engine/pegasus/test/k8s/pegasus-set.yaml + echo " value: \"${{ github.sha }}\"" >> research/engine/pegasus/test/k8s/pegasus-set.yaml + echo " image: registry.cn-hongkong.aliyuncs.com/graphscope/pegasus:${{ github.sha }}" >> research/engine/pegasus/test/k8s/pegasus-set.yaml + + - name: Start k8s cluster + run: | + kubectl get ns pegasus-ci || kubectl create ns pegasus-ci + kubectl --namespace pegasus-ci create -f research/engine/pegasus/test/k8s/role_binding.yaml + kubectl --namespace pegasus-ci create -f research/engine/pegasus/test/k8s/pegasus-set.yaml + bash research/engine/pegasus/test/k8s/read_result.sh ${{ github.sha }} + + - name: Clean + if: always() + run: | + kubectl delete ns pegasus-ci || true + docker rmi -f registry.cn-hongkong.aliyuncs.com/graphscope/pegasus:${{ github.sha }} --wait=false || true diff --git a/.gitignore b/.gitignore index 0b3286e2a164..c1f063a9169d 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,7 @@ interactive_engine/executor/store/src/db/proto/* interactive_engine/data_load_tools/dependency-reduced-pom.xml interactive_engine/gaia-adaptor/dependency-reduced-pom.xml interactive_engine/executor/Cargo.lock +research/engine/pegasus/benchmark/src/graph/storage/clickhouse/pb_gen/* # java sdk java/**/dependency-reduced-pom.xml diff --git a/Makefile b/Makefile index 1472f881c533..ee1b4bc8b27a 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,7 @@ gie: mkdir -p $(WORKING_DIR)/.install_prefix && \ tar -xf $(WORKING_DIR)/interactive_engine/assembly/target/maxgraph-assembly-0.0.1-SNAPSHOT.tar.gz --strip-components 1 -C $(WORKING_DIR)/.install_prefix && \ cp $(WORKING_DIR)/interactive_engine/executor/target/$(BUILD_TYPE)/executor $(WORKING_DIR)/.install_prefix/bin/executor && \ + cp $(WORKING_DIR)/interactive_engine/executor/target/$(BUILD_TYPE)/gaia_executor $(WORKING_DIR)/.install_prefix/bin/gaia_executor && \ sudo cp -r $(WORKING_DIR)/.install_prefix/* $(INSTALL_PREFIX) && \ rm -fr $(WORKING_DIR)/.install_prefix diff --git a/interactive_engine/distribution/src/assembly/assembly.xml b/interactive_engine/distribution/src/assembly/assembly.xml index 7332fb429241..02603ee524c5 100644 --- a/interactive_engine/distribution/src/assembly/assembly.xml +++ b/interactive_engine/distribution/src/assembly/assembly.xml @@ -13,6 +13,13 @@ native + + ${project.parent.basedir}/../research/query_service/ir/target/release + + libir_core* + + native + ${project.parent.basedir}/lgraph/build/install diff --git a/interactive_engine/executor/Cargo.toml b/interactive_engine/executor/Cargo.toml index 658e68a2f525..ee1a4d92d138 100644 --- a/interactive_engine/executor/Cargo.toml +++ b/interactive_engine/executor/Cargo.toml @@ -5,7 +5,8 @@ members = [ "store", "server", "Pegasus", - "ffi" + "ffi", + "gaia_runtime" ] [profile.release] diff --git a/interactive_engine/executor/ffi/Cargo.toml b/interactive_engine/executor/ffi/Cargo.toml index 9160420c9cf0..48dbf86b4cd0 100644 --- a/interactive_engine/executor/ffi/Cargo.toml +++ b/interactive_engine/executor/ffi/Cargo.toml @@ -13,6 +13,10 @@ maxgraph-store = { path = "../store" } maxgraph-common = { path = "../../rust-common" } maxgraph-server = { path = "../server" } pegasus = { path = "../Pegasus", package = "pegasus" } +gaia_pegasus = { path = "../../../research/engine/pegasus/pegasus", package = "pegasus" } +pegasus_network = { path = "../../../research/engine/pegasus/network" } +pegasus_server = { path = "../../../research/engine/pegasus/server-v0" } +graph_proxy = { path = "../../../research/query_service/ir/graph_proxy" } itertools = "0.7.8" log = "0.3" log4rs = "0.8.0" diff --git a/interactive_engine/executor/ffi/src/executor/gaia/engine_ports_response.rs b/interactive_engine/executor/ffi/src/executor/gaia/engine_ports_response.rs new file mode 100644 index 000000000000..39a26d978284 --- /dev/null +++ b/interactive_engine/executor/ffi/src/executor/gaia/engine_ports_response.rs @@ -0,0 +1,64 @@ +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use std::os::raw::c_char; +use std::ffi::CString; + +#[repr(C)] +#[allow(non_snake_case)] +pub struct EnginePortsResponse { + success: bool, + errMsg: *const c_char, + enginePort: i32, + rpcPort: i32, +} + +impl EnginePortsResponse { + pub fn new(engine_port: i32, rpc_port: i32) -> Box { + Box::new(EnginePortsResponse { + success: true, + errMsg: std::ptr::null(), + enginePort: engine_port, + rpcPort: rpc_port, + }) + } + + pub fn new_with_error(err_msg: &str) -> Box { + let msg = CString::new(err_msg).unwrap(); + let response = EnginePortsResponse { + success: false, + errMsg: msg.as_ptr(), + enginePort: 0, + rpcPort: 0, + }; + ::std::mem::forget(msg); + Box::new(response) + } +} + +impl Drop for EnginePortsResponse { + fn drop(&mut self) { + unsafe { + if !self.errMsg.is_null() { + CString::from_raw(self.errMsg as *mut c_char); + } + } + } +} + +#[no_mangle] +#[allow(non_snake_case)] +pub extern fn dropEnginePortsResponse(_: Box) {} diff --git a/interactive_engine/executor/ffi/src/executor/gaia/gaia_library.rs b/interactive_engine/executor/ffi/src/executor/gaia/gaia_library.rs new file mode 100644 index 000000000000..0567723b1f45 --- /dev/null +++ b/interactive_engine/executor/ffi/src/executor/gaia/gaia_library.rs @@ -0,0 +1,101 @@ +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use std::os::raw::{c_void, c_char}; +use maxgraph_store::db::common::bytes::util::parse_pb; +use maxgraph_store::db::api::GraphConfigBuilder; +use std::sync::Arc; +use maxgraph_store::db::common::unsafe_util::to_mut; +use maxgraph_store::db::graph::store::GraphStore; +use std::ffi::CStr; +use std::net::{SocketAddr, ToSocketAddrs}; +use crate::executor::gaia::gaia_server::GaiaServer; +use crate::executor::gaia::engine_ports_response::EnginePortsResponse; +use maxgraph_store::db::proto::model::ConfigPb; + +pub type EngineHandle = *const c_void; +pub type GraphHandle = *const c_void; + +#[no_mangle] +pub extern fn initialize(config_bytes: *const u8, len: usize) -> EngineHandle { + let config_buf = unsafe { ::std::slice::from_raw_parts(config_bytes, len) }; + let config_pb = parse_pb::(config_buf).expect("parse config pb failed"); + let mut config_builder = GraphConfigBuilder::new(); + config_builder.set_storage_options(config_pb.get_configs().clone()); + let config = Arc::new(config_builder.build()); + let handle = Box::new(GaiaServer::new(config)); + Box::into_raw(handle) as EngineHandle +} + +#[no_mangle] +pub extern fn addPartition(engine_handle: EngineHandle, partition_id: i32, graph_handle: GraphHandle) { + let engine_ptr = unsafe { + to_mut(&*(engine_handle as *const GaiaServer)) + }; + let graph_ptr = unsafe { + Arc::from_raw(&*(graph_handle as *const GraphStore)) + }; + engine_ptr.add_partition(partition_id as u32, graph_ptr); +} + +#[no_mangle] +pub extern fn updatePartitionRouting(engine_handle: EngineHandle, partition_id: i32, server_id: i32) { + let engine_ptr = unsafe { + to_mut(&*(engine_handle as *const GaiaServer)) + }; + engine_ptr.update_partition_routing(partition_id as u32, server_id as u32); +} + +#[no_mangle] +pub extern fn startEngine(engine_handle: EngineHandle) -> Box { + let engine_ptr = unsafe { + to_mut(&*(engine_handle as *const GaiaServer)) + }; + match engine_ptr.start() { + Ok((engine_port, server_port)) => { + EnginePortsResponse::new(engine_port as i32, server_port as i32) + }, + Err(e) => { + let msg = format!("{:?}", e); + EnginePortsResponse::new_with_error(&msg) + }, + } +} + +#[no_mangle] +pub extern fn stopEngine(engine_handle: EngineHandle) { + let engine_ptr = unsafe { + to_mut(&*(engine_handle as *const GaiaServer)) + }; + engine_ptr.stop(); +} + +#[no_mangle] +pub extern fn updatePeerView(engine_handle: EngineHandle, peer_view_string_raw: *const c_char) { + let slice = unsafe { CStr::from_ptr(peer_view_string_raw) }.to_bytes(); + let peer_view_string = std::str::from_utf8(slice).unwrap(); + let peer_view = peer_view_string.split(",").map(|item| { + let mut fields = item.split("#"); + let id = fields.next().unwrap().parse::().unwrap(); + let addr_str = fields.next().unwrap(); + let mut addr_iter = addr_str.to_socket_addrs().expect(format!("parse addr failed [{}]", addr_str).as_str()); + (id, addr_iter.next().unwrap()) + }).collect::>(); + let engine_ptr = unsafe { + to_mut(&*(engine_handle as *const GaiaServer)) + }; + engine_ptr.update_peer_view(peer_view); +} diff --git a/interactive_engine/executor/ffi/src/executor/gaia/gaia_server.rs b/interactive_engine/executor/ffi/src/executor/gaia/gaia_server.rs new file mode 100644 index 000000000000..f42c27b99684 --- /dev/null +++ b/interactive_engine/executor/ffi/src/executor/gaia/gaia_server.rs @@ -0,0 +1,141 @@ +// +//! Copyright 2021 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use maxgraph_store::db::api::{GraphConfig, GraphResult, GraphError}; +use std::sync::Arc; +use maxgraph_store::db::graph::store::GraphStore; +use std::net::SocketAddr; +use maxgraph_runtime::store::groot::global_graph::GlobalGraph; +use gaia_pegasus::Configuration as GaiaConfig; +use maxgraph_store::db::api::GraphErrorCode::EngineError; +use tokio::runtime::Runtime; +use pegasus_server::service::Service; +use pegasus_server::rpc::{start_rpc_server, RpcService}; +use pegasus_network::SimpleServerDetector; +use pegasus_network::config::{NetworkConfig, ServerAddr}; +use graph_proxy::{InitializeJobCompiler, QueryMaxGraph}; +use maxgraph_store::api::PartitionId; + +pub struct GaiaServer { + config: Arc, + graph: Arc, + detector: Arc, + rpc_runtime: Runtime, +} + +impl GaiaServer { + pub fn new(config: Arc) -> Self { + let partition_count = config.get_storage_option("partition.count") + .expect("required config partition.count is missing").parse().expect("parse partition.count failed"); + GaiaServer { + config, + graph: Arc::new(GlobalGraph::empty(partition_count)), + detector: Arc::new(SimpleServerDetector::new()), + rpc_runtime: Runtime::new().unwrap(), + } + } + + pub fn add_partition(&mut self, partition_id: PartitionId, graph_partition: Arc) { + Arc::get_mut(&mut self.graph).unwrap().add_partition(partition_id, graph_partition); + } + + pub fn update_partition_routing(&mut self, partition_id: PartitionId, worker_id: u32) { + Arc::get_mut(&mut self.graph).unwrap().update_partition_routing(partition_id, worker_id); + } + + pub fn start(&self) -> GraphResult<(u16, u16)> { + let report = match self.config.get_storage_option("gaia.report") { + None => false, + Some(report_string) => report_string.parse() + .map_err(|e| GraphError::new(EngineError, format!("{:?}", e)))?, + }; + let rpc_port = match self.config.get_storage_option("gaia.rpc.port") { + None => { 0 }, + Some(server_port_string) => { + server_port_string.parse().map_err(|e| GraphError::new(EngineError, format!("{:?}", e)))? + }, + }; + let addr = format!("{}:{}", "0.0.0.0", rpc_port).parse() + .map_err(|e| GraphError::new(EngineError, format!("{:?}", e)))?; + let gaia_config = make_gaia_config(self.config.clone()); + let socket_addr = gaia_pegasus::startup_with(gaia_config, self.detector.clone()) + .map_err(|e| GraphError::new(EngineError, format!("{:?}", e)))? + .ok_or(GraphError::new(EngineError, "gaia engine return None addr".to_string()))?; + + let rpc_port = self.rpc_runtime.block_on(async{ + let query_maxgraph = QueryMaxGraph::new(self.graph.clone(), self.graph.clone()); + let job_compiler = query_maxgraph.initialize_job_compiler(); + let service = Service::new(job_compiler); + let rpc_service = RpcService::new(service, report); + let local_addr = start_rpc_server(addr, rpc_service, false).await.unwrap(); + local_addr.port() + }); + Ok((socket_addr.port(), rpc_port)) + } + + pub fn update_peer_view(&self, peer_view: Vec<(u64, SocketAddr)>) { + self.detector.update_peer_view(peer_view.into_iter()); + } + + pub fn stop(&self) { + gaia_pegasus::shutdown_all(); + } +} + +fn make_gaia_config(graph_config: Arc) -> GaiaConfig { + let server_id = graph_config.get_storage_option("node.idx").expect("required config node.idx is missing") + .parse().expect("parse node.idx failed"); + let ip = "0.0.0.0".to_string(); + let port = match graph_config.get_storage_option("gaia.engine.port") { + None => { 0 }, + Some(server_port_string) => { + server_port_string.parse().expect("parse gaia.engine.port failed") + }, + }; + let worker_num = match graph_config.get_storage_option("worker.num") { + None => 1, + Some(worker_num_string) => worker_num_string.parse().expect("parse worker.num failed"), + }; + let nonblocking = graph_config.get_storage_option("gaia.nonblocking") + .map(|config_str| config_str.parse().expect("parse gaia.nonblocking failed")); + let read_timeout_ms = graph_config.get_storage_option("gaia.read.timeout.ms") + .map(|config_str| config_str.parse().expect("parse gaia.read.timeout.ms failed")); + let write_timeout_ms = graph_config.get_storage_option("gaia.write.timeout.ms") + .map(|config_str| config_str.parse().expect("parse gaia.write.timeout.ms failed")); + let read_slab_size = graph_config.get_storage_option("gaia.read.slab.size") + .map(|config_str| config_str.parse().expect("parse gaia.read.slab.size failed")); + let no_delay = graph_config.get_storage_option("gaia.no.delay") + .map(|config_str| config_str.parse().expect("parse gaia.no.delay failed")); + let send_buffer = graph_config.get_storage_option("gaia.send.buffer") + .map(|config_str| config_str.parse().expect("parse gaia.send.buffer failed")); + let heartbeat_sec = graph_config.get_storage_option("gaia.heartbeat.sec") + .map(|config_str| config_str.parse().expect("parse gaia.heartbeat.sec failed")); + let max_pool_size = graph_config.get_storage_option("gaia.max.pool.size") + .map(|config_str| config_str.parse().expect("parse gaia.max.pool.size failed")); + let mut network_config = NetworkConfig::new(server_id, ServerAddr::new(ip, port), worker_num); + network_config.nonblocking(nonblocking) + .read_timeout_ms(read_timeout_ms) + .write_timeout_ms(write_timeout_ms) + .read_slab_size(read_slab_size) + .no_delay(no_delay) + .send_buffer(send_buffer) + .heartbeat_sec(heartbeat_sec); + GaiaConfig { + network: Some(network_config), + max_pool_size, + } +} + diff --git a/interactive_engine/executor/ffi/src/executor/gaia/mod.rs b/interactive_engine/executor/ffi/src/executor/gaia/mod.rs new file mode 100644 index 000000000000..883ff6b122fb --- /dev/null +++ b/interactive_engine/executor/ffi/src/executor/gaia/mod.rs @@ -0,0 +1,19 @@ +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +mod gaia_library; +mod gaia_server; +mod engine_ports_response; diff --git a/interactive_engine/executor/ffi/src/executor/mod.rs b/interactive_engine/executor/ffi/src/executor/mod.rs index 031312f4c7d2..90578ad08738 100644 --- a/interactive_engine/executor/ffi/src/executor/mod.rs +++ b/interactive_engine/executor/ffi/src/executor/mod.rs @@ -1,2 +1,18 @@ -mod pegasus; +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! +mod gaia; +mod pegasus; diff --git a/interactive_engine/executor/ffi/src/lib.rs b/interactive_engine/executor/ffi/src/lib.rs index 60702aec6c22..98d6e7f2f91b 100644 --- a/interactive_engine/executor/ffi/src/lib.rs +++ b/interactive_engine/executor/ffi/src/lib.rs @@ -2,6 +2,7 @@ extern crate maxgraph_runtime; extern crate maxgraph_store; extern crate maxgraph_common; extern crate maxgraph_server; +extern crate gaia_pegasus; extern crate itertools; extern crate pegasus; #[macro_use] @@ -9,6 +10,7 @@ extern crate log; extern crate log4rs; extern crate grpcio; extern crate lazy_static; +extern crate pegasus_network; mod executor; mod store; diff --git a/interactive_engine/executor/gaia_runtime/Cargo.toml b/interactive_engine/executor/gaia_runtime/Cargo.toml new file mode 100644 index 000000000000..614ff57db0be --- /dev/null +++ b/interactive_engine/executor/gaia_runtime/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "gaia_runtime" +version = "0.1.0" +authors = ["BingqingLyu "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +abomonation = "0.7" +abomonation_derive = "0.3" +bincode = "1.0.1" +byteorder = "1.3.1" +crossbeam-channel = "0.4.4" +crossbeam-queue = "0.1" +futures-cpupool = "0.1.8" +getopts = "0.2.14" +grpcio = "=0.4.1" +quote = "=1.0.1" +itertools = "0.7.8" +lazy_static = "1.2.0" +log = "0.3" +log4rs = "0.8.0" +maxgraph-common = { path = "../../rust-common" } +maxgraph-store = { path = "../store" } +maxgraph-server = { path = "../server" } +maxgraph-runtime = { path = "../runtime" } +protobuf = { version = "~2.0", features = ["with-bytes"] } +rand = "0.7.3" +regex = "1" +serde = "1.0.72" +serde_derive = "1.0.72" +serde_json = "1.0.3" +structopt = "0.2" +zookeeper = { git = "https://github.com/bonifaido/rust-zookeeper.git", rev = "fd222dd" } +lru_time_cache = "0.8.0" +pegasus = { path = "../Pegasus" } +libc = "0.2" + +tokio = { version = "1.0", features = ["macros", "sync"] } +futures = { version = "0.3.0", features = ["thread-pool"] } +gaia_pegasus = { path = "../../../research/engine/pegasus/pegasus", package = "pegasus" } +pegasus_network = { path = "../../../research/engine/pegasus/network" } +pegasus_server = { path = "../../../research/engine/pegasus/server-v0" } +graph_proxy = { path = "../../../research/query_service/ir/graph_proxy" } + +[dev-dependencies] +env_logger = "0.6" + +[build-dependencies] +cmake = "0.1" diff --git a/interactive_engine/executor/gaia_runtime/src/bin/gaia_executor.rs b/interactive_engine/executor/gaia_runtime/src/bin/gaia_executor.rs new file mode 100644 index 000000000000..34c8b2790f28 --- /dev/null +++ b/interactive_engine/executor/gaia_runtime/src/bin/gaia_executor.rs @@ -0,0 +1,298 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +#![allow(bare_trait_objects)] + +extern crate futures; +extern crate grpcio; +#[macro_use] +extern crate log; +extern crate gaia_pegasus; +extern crate graph_proxy; +extern crate log4rs; +extern crate maxgraph_common; +extern crate maxgraph_runtime; +extern crate maxgraph_server; +extern crate maxgraph_store; +extern crate pegasus; +extern crate pegasus_server; +extern crate protobuf; +extern crate structopt; + +use gaia_runtime::server::init_with_rpc_service; +use gaia_runtime::server::manager::GaiaServerManager; +use graph_proxy::{InitializeJobCompiler, QueryVineyard}; +use grpcio::ChannelBuilder; +use grpcio::EnvBuilder; +use maxgraph_common::proto::data::*; +use maxgraph_common::proto::hb::*; +use maxgraph_common::proto::query_flow::*; +use maxgraph_common::util; +use maxgraph_common::util::get_local_ip; +use maxgraph_common::util::log4rs::init_log4rs; +use maxgraph_runtime::server::manager::*; +use maxgraph_runtime::server::RuntimeInfo; +use maxgraph_server::StoreContext; +use maxgraph_store::api::graph_partition::GraphPartitionManager; +use maxgraph_store::api::prelude::*; +use maxgraph_store::config::{StoreConfig, VINEYARD_GRAPH}; +use pegasus_server::rpc::{start_rpc_server, RpcService}; +use pegasus_server::service::Service; +use protobuf::Message; +use std::collections::HashMap; +use std::env; +use std::sync::atomic::AtomicBool; +use std::sync::mpsc::{channel, Sender}; +use std::sync::{Arc, Mutex, RwLock}; +use std::thread; +use std::time::Duration; +use tokio::runtime::Runtime; + +fn main() { + if let Some(_) = env::args().find(|arg| arg == "--show-build-info") { + util::get_build_info(); + return; + } + init_log4rs(); + let mut store_config = { + let args: Vec = std::env::args().collect(); + if args.len() <= 6 && args[1] == "--config" { + let mut store_config = StoreConfig::init_from_file(&args[2], &args[4]); + if args.len() == 6 { + store_config.graph_name = (&args[5]).to_string(); + } + store_config + } else { + StoreConfig::init() + } + }; + let (alive_id, partitions) = get_init_info(&store_config); + info!("alive_id: {:?}, partitions: {:?}", alive_id, partitions); + store_config.update_alive_id(alive_id); + info!("{:?}", store_config); + + let worker_num = store_config.timely_worker_per_process; + let store_config = Arc::new(store_config); + if store_config.graph_type.to_lowercase().eq(VINEYARD_GRAPH) { + info!( + "Start executor with vineyard graph object id {:?}", + store_config.vineyard_graph_id + ); + use maxgraph_runtime::store::ffi::FFIGraphStore; + let ffi_store = FFIGraphStore::new(store_config.vineyard_graph_id, worker_num as i32); + let partition_manager = ffi_store.get_partition_manager(); + run_main(store_config, Arc::new(ffi_store), Arc::new(partition_manager)); + } else { + unimplemented!("only start vineyard graph from executor") + } +} + +fn run_main( + store_config: Arc, + graph: Arc>, + partition_manager: Arc, +) where + V: Vertex + 'static, + VI: Iterator + Send + 'static, + E: Edge + 'static, + EI: Iterator + Send + 'static, +{ + let process_partition_list = partition_manager.get_process_partition_list(); + info!("process_partition_list: {:?}", process_partition_list); + let runtime_info = Arc::new(Mutex::new(RuntimeInfo::new( + store_config.timely_worker_per_process, + process_partition_list, + ))); + let runtime_info_clone = runtime_info.clone(); + let (hb_resp_sender, hb_resp_receiver) = channel(); + let signal = Arc::new(AtomicBool::new(false)); + let gaia_server_manager = + GaiaServerManager::new(hb_resp_receiver, runtime_info, signal.clone()); + + let partition_worker_mapping = gaia_server_manager.get_partition_worker_mapping(); + let worker_partition_list_mapping = gaia_server_manager.get_worker_partition_list_mapping(); + let server_manager = Box::new(gaia_server_manager); + let _manager_guards = ServerManager::start_server( + server_manager, + store_config.clone(), + Box::new(recover_prepare), + ) + .unwrap(); + + let gaia_service = GaiaService::new( + store_config.clone(), + graph.clone(), + partition_manager.clone(), + partition_worker_mapping, + worker_partition_list_mapping, + ); + let (_, gaia_rpc_service_port) = gaia_service.start_rpc_service(); + let store_context = StoreContext::new(graph, partition_manager); + start_hb_rpc_service( + runtime_info_clone, + store_config, + gaia_rpc_service_port, + hb_resp_sender, + store_context, + ); + thread::sleep(Duration::from_secs(u64::max_value())); +} + +fn recover_prepare(prepared: &[u8]) -> Result, String> { + ::protobuf::parse_from_bytes::(prepared) + .map_err(|err| err.to_string()) + .and_then(move |desc| { + info!("parse {} bytes to {:?} ", prepared.len(), desc); + Ok(desc.write_to_bytes().expect("query flow to bytes")) + }) +} + +fn start_hb_rpc_service( + runtime_info: Arc>, + store_config: Arc, + gaia_service_port: u16, + hb_resp_sender: Sender>, + store_context: StoreContext, +) where + VV: 'static + Vertex, + VVI: 'static + Iterator + Send, + EE: 'static + Edge, + EEI: 'static + Iterator + Send, +{ + // build hb information + let mut hb_providers = Vec::new(); + let mut hb_resp_senders = Vec::new(); + let hb_provider = move |ref mut server_hb_req: &mut ServerHBReq| { + server_hb_req.set_runtimeReq(build_runtime_req(runtime_info.clone())); + server_hb_req + .mut_endpoint() + .set_runtimCtrlAndAsyncPort(gaia_service_port as i32); + }; + + hb_providers.push(Box::new(hb_provider)); + hb_resp_senders.push(hb_resp_sender); + + let store_config_clone = store_config.clone(); + init_with_rpc_service( + store_config_clone, + hb_providers, + hb_resp_senders, + store_context, + ); +} + +fn build_runtime_req(runtime_info: Arc>) -> RuntimeHBReq { + let hb_req = runtime_info.lock().expect("Lock runtime hb req failed"); + + let mut runtime_req = RuntimeHBReq::new(); + runtime_req.set_serverStatus(hb_req.get_server_status()); + runtime_req.set_runtimePort(hb_req.get_server_port() as i32); + runtime_req.set_worker_num_per_process(hb_req.get_worker_num_per_process()); + runtime_req.set_process_partition_list(hb_req.get_process_partition_list().to_vec()); + debug!("Build runtime request {:?} in heartbeat", &runtime_req); + + runtime_req +} + +/// return: (aliveId, partiiton assignments) +fn get_init_info(config: &StoreConfig) -> (u64, Vec) { + use maxgraph_common::proto::data_grpc::*; + use maxgraph_common::util::ip; + use maxgraph_server::client::ZKClient; + + let zk_url = format!("{}/{}", config.zk_url, config.graph_name); + let zk = ZKClient::new(&zk_url, config.zk_timeout_ms, config.get_zk_auth()); + let addr = zk.get_coordinator_addr(); + + let channel = + ChannelBuilder::new(Arc::new(EnvBuilder::new().build())).connect(addr.to_string().as_str()); + + let client = ServerDataApiClient::new(channel); + + let mut request = GetExecutorAliveIdRequest::new(); + request.set_serverId(config.worker_id); + request.set_ip(ip::get_local_ip()); + let response = client.get_executor_alive_id(&request).unwrap(); + let alive_id = response.get_aliveId(); + let mut request = GetPartitionAssignmentRequest::new(); + request.set_serverId(config.worker_id); + let response = client.get_partition_assignment(&request).unwrap(); + let partitions = response.get_partitionId().to_vec(); + (alive_id, partitions) +} + +pub struct GaiaService +where + V: Vertex + 'static, + VI: Iterator + Send + 'static, + E: Edge + 'static, + EI: Iterator + Send + 'static, +{ + store_config: Arc, + graph: Arc>, + partition_manager: Arc, + // mapping of partition id -> worker id + partition_worker_mapping: Arc>>>, + // mapping of worker id -> partition list + worker_partition_list_mapping: Arc>>>>, + rpc_runtime: Runtime, +} + +impl GaiaService +where + V: Vertex + 'static, + VI: Iterator + Send + 'static, + E: Edge + 'static, + EI: Iterator + Send + 'static, +{ + pub fn new( + store_config: Arc, + graph: Arc>, + partition_manager: Arc, + partition_worker_mapping: Arc>>>, + worker_partition_list_mapping: Arc>>>>, + ) -> GaiaService { + GaiaService { + store_config, + graph, + partition_manager, + partition_worker_mapping, + worker_partition_list_mapping, + rpc_runtime: Runtime::new().unwrap(), + } + } + + pub fn start_rpc_service(&self) -> (String, u16) { + let rpc_port = self.rpc_runtime.block_on(async { + let query_vineyard = QueryVineyard::new( + self.graph.clone(), + self.partition_manager.clone(), + self.partition_worker_mapping.clone(), + self.worker_partition_list_mapping.clone(), + self.store_config.worker_num as usize, + ); + let job_compiler = query_vineyard.initialize_job_compiler(); + let service = Service::new(job_compiler); + let addr = format!("{}:{}", "0.0.0.0", self.store_config.rpc_port); + // TODO: add report in store_config + let rpc_service = RpcService::new(service, true); + let local_addr = start_rpc_server(addr.parse().unwrap(), rpc_service, false).await.unwrap(); + local_addr.port() + }); + let ip = get_local_ip(); + info!("start rpc server on {} {}", ip, rpc_port); + (ip, rpc_port) + } +} diff --git a/interactive_engine/executor/gaia_runtime/src/lib.rs b/interactive_engine/executor/gaia_runtime/src/lib.rs new file mode 100644 index 000000000000..dc7d73df61db --- /dev/null +++ b/interactive_engine/executor/gaia_runtime/src/lib.rs @@ -0,0 +1,19 @@ +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +#[macro_use] +extern crate log; +pub mod server; diff --git a/interactive_engine/executor/gaia_runtime/src/server/manager.rs b/interactive_engine/executor/gaia_runtime/src/server/manager.rs new file mode 100644 index 000000000000..5635b6db1e33 --- /dev/null +++ b/interactive_engine/executor/gaia_runtime/src/server/manager.rs @@ -0,0 +1,236 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +//! timely_server_manager responsible for controlling the timely_server, including starting, running and down +//! it also collects the status information of timely_server and reports to runtime_manager (coordinator) +extern crate protobuf; + +use gaia_pegasus::Configuration; +use maxgraph_common::proto::hb::*; +use maxgraph_common::util::log; +use maxgraph_runtime::server::allocate::register_tcp_listener; +use maxgraph_runtime::server::manager::{ManagerGuards, ServerManager, ServerManagerCommon}; +use maxgraph_runtime::server::network_manager::{NetworkManager, PegasusNetworkCenter}; +use maxgraph_runtime::server::RuntimeAddress; +use maxgraph_runtime::server::RuntimeInfo; +use maxgraph_runtime::store::task_partition_manager::TaskPartitionManager; +use maxgraph_store::config::StoreConfig; +use pegasus::{network_connection, ConfigArgs}; +use pegasus::Pegasus; +use pegasus_network::config::{NetworkConfig, ServerAddr}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::Receiver; +use std::sync::{Arc, Mutex, RwLock}; +use std::thread; +use std::time; +use std::vec::Vec; + +pub struct GaiaServerManager { + server_manager_common: ServerManagerCommon, + // we preserve pegasus_runtime for heartbeat + pegasus_runtime: Arc>, + // mapping of partition id -> worker id + partition_worker_mapping: Arc>>>, + // mapping of worker id -> partition list + worker_partition_list_mapping: Arc>>>>, + signal: Arc, +} + +impl GaiaServerManager { + pub fn new( + receiver: Receiver>, + runtime_info: Arc>, + signal: Arc, + ) -> GaiaServerManager { + GaiaServerManager { + server_manager_common: ServerManagerCommon::new(receiver, runtime_info), + pegasus_runtime: Arc::new(None), + partition_worker_mapping: Arc::new(RwLock::new(None)), + worker_partition_list_mapping: Arc::new(RwLock::new(None)), + signal, + } + } + + #[inline] + pub fn get_server(&self) -> Arc> { + self.pegasus_runtime.clone() + } + + #[inline] + pub fn get_partition_worker_mapping(&self) -> Arc>>> { + self.partition_worker_mapping.clone() + } + + fn initial_partition_worker_mapping(&self, partition_task_list: HashMap) { + let mut partition_worker_mapping = self.partition_worker_mapping.write().unwrap(); + partition_worker_mapping.replace(partition_task_list); + } + + #[inline] + pub fn get_worker_partition_list_mapping(&self) -> Arc>>>> { + self.worker_partition_list_mapping.clone() + } + + fn initial_worker_partition_list_mapping(&self, task_partition_lists: HashMap>) { + let mut worker_partition_list_mapping = self.worker_partition_list_mapping.write().unwrap(); + worker_partition_list_mapping.replace(task_partition_lists); + } + + /// Initialize pegasus runtime + /// + /// Pegasus runtime is shared by query rpc thread and server manager thread, but it will only be initialized once by + /// server manager thread and query rpc thread only read the value. + /// Consequently here use `unsafe` but not `Mutex` or `RwLock` to initialize pegasus runtime. + fn initial_pegasus_runtime(&self, process_id: usize, store_config: Arc) { + let pegasus_runtime = ConfigArgs::distribute(process_id, store_config.pegasus_thread_pool_size as usize, store_config.worker_num as usize, "".to_string()).build(); + unsafe { + let pegasus_pointer = Arc::into_raw(self.pegasus_runtime.clone()) as *mut Option; + (*pegasus_pointer).replace(pegasus_runtime); + } + } +} + +impl ServerManager for GaiaServerManager { + type Data = Vec; + fn start_server( + self: Box, + store_config: Arc, + _recover: Box Result>, + ) -> Result, String> { + let manager_switch = self.server_manager_common.manager_switch.clone(); + let handle = thread::Builder::new().name("Gaia Server Manager".to_owned()).spawn(move || { + let listener = register_tcp_listener(); + self.server_manager_common.update_port_and_status(listener.local_addr().expect("Build local address failed.").port(), RuntimeHBReq_RuntimeStatus::STARTING); + + let mut network_manager = NetworkManager::initialize(listener); + let mut network_center = PegasusNetworkCenter::new(); + + while self.server_manager_common.manager_switch.load(Ordering::Relaxed) { + let hb_resp = self.server_manager_common.get_hb_response(); + if hb_resp.is_none() || network_manager.is_serving() { + thread::sleep(time::Duration::from_millis(store_config.hb_interval_ms)); + continue; + } + + let timely_server_resp = hb_resp.as_ref().unwrap().get_runtimeResp(); + + let group_id = timely_server_resp.get_groupId(); + let worker_id = timely_server_resp.get_workerId(); + let address_list = timely_server_resp.get_addresses(); + let task_partition_list = timely_server_resp.get_task_partition_list(); + + if self.pegasus_runtime.is_none() { + info!("Begin start pegasus with process id {} in group {}.", worker_id, group_id); + self.initial_pegasus_runtime(worker_id as usize, store_config.clone()); + } + if !address_list.is_empty() && !task_partition_list.is_empty() { + // start gaia_pegasus + let configuration = build_gaia_config(worker_id as usize, address_list, store_config.clone()); + info!("gaia configuration {:?}", configuration); + if let Err(err) = gaia_pegasus::startup(configuration) { + error!("start pegasus failed {:?}", err); + } else { + info!("start pegasus successfully"); + } + + let (ip_list, _store_ip_list) = parse_store_ip_list(address_list); + info!("Receive task partition info {:?} from coordinator", task_partition_list); + let task_partition_manager = TaskPartitionManager::parse_proto(task_partition_list); + self.initial_partition_worker_mapping(task_partition_manager.get_partition_task_list()); + self.initial_worker_partition_list_mapping(task_partition_manager.get_task_partition_list_mapping()); + + network_manager.update_number(worker_id as usize, ip_list.len()); + network_center.initialize(ip_list); + + self.signal.store(true, Ordering::Relaxed); + } else { + continue; + } + + let (start_addresses, await_addresses) = network_manager.check_network_status(Box::new(network_center.clone())); + info!("worker {} in group {} is starting, caused by connection between {:?} and {:?} is not working.", + worker_id, group_id, start_addresses, await_addresses); + + match network_manager.reconnect(start_addresses, await_addresses, store_config.hb_interval_ms) { + Ok(result) => { + info!("worker {} in group {} connect to {:?} success.", worker_id, group_id, result); + log::log_runtime(store_config.graph_name.as_str(), store_config.worker_id, group_id, worker_id, log::RuntimeEvent::ServerDown, + self.server_manager_common.version, format!("worker {} in group {} connect to {:?} success.", worker_id, group_id, result).as_str()); + for (index, address, tcp_stream) in result.into_iter() { + let connection = network_connection::Connection::new(index, address, tcp_stream); + self.pegasus_runtime.as_ref().as_ref().unwrap().reset_single_network(connection.clone()); + network_manager.reset_network(connection); + } + } + Err(e) => { + error!("worker {} in group {} reset network failed, caused by {:?}.", worker_id, group_id, e); + log::log_runtime(store_config.graph_name.as_str(), store_config.worker_id, group_id, worker_id, log::RuntimeEvent::ServerDown, + self.server_manager_common.version, format!("reset network error, caused by {:?}", e).as_str()); + } + } + + if network_manager.is_serving() { + self.server_manager_common.change_server_status(RuntimeHBReq_RuntimeStatus::RUNNING); + info!("worker {} in group {} is running.", worker_id, group_id); + log::log_runtime(store_config.graph_name.as_str(), store_config.worker_id, group_id, worker_id, log::RuntimeEvent::ServerDown, + self.server_manager_common.version, format!("worker is running successfully.").as_str()); + } + } + }); + + match handle { + Ok(handle) => Ok(ManagerGuards::new(handle, manager_switch)), + Err(e) => Err(format!("Build server manager thread fail: {:?}", e)), + } + } +} + +fn build_gaia_config(worker_id: usize, address_list: &[RuntimeAddressProto], store_config: Arc) -> Configuration { + let peers = parse_store_ip_list_for_gaia(address_list, store_config); + info!("gaia peers list: {:?}", peers); + // TODO: more configuration from store_config for pegasus + let network_config = NetworkConfig::with(worker_id as u64, peers); + Configuration { + network: Some(network_config), + max_pool_size: None, + } +} + +fn parse_store_ip_list_for_gaia(address_list: &[RuntimeAddressProto], store_config: Arc) -> Vec { + let mut peers_list = Vec::with_capacity(address_list.len()); + for address in address_list { + let peer_config = ServerAddr::new(address.get_ip().to_string(), store_config.gaia_engine_port as u16); + peers_list.push(peer_config); + } + peers_list +} + +fn parse_store_ip_list(address_list: &[RuntimeAddressProto]) -> (Vec, Vec) { + let mut ip_list = Vec::with_capacity(address_list.len()); + let mut store_address_list = Vec::with_capacity(address_list.len()); + for address in address_list { + ip_list.push(format!( + "{}:{}", + address.get_ip(), + address.get_runtime_port() + )); + let store_address = + RuntimeAddress::new(address.get_ip().to_string(), address.get_store_port()); + store_address_list.push(store_address); + } + + (ip_list, store_address_list) +} diff --git a/interactive_engine/executor/gaia_runtime/src/server/mod.rs b/interactive_engine/executor/gaia_runtime/src/server/mod.rs new file mode 100644 index 000000000000..1a63af2d82fb --- /dev/null +++ b/interactive_engine/executor/gaia_runtime/src/server/mod.rs @@ -0,0 +1,91 @@ +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +pub mod manager; + +use grpcio::{ChannelBuilder, Environment, ServerBuilder}; +use maxgraph_common::proto::gremlin_query_grpc; +use maxgraph_common::proto::hb::*; +use maxgraph_common::util::get_local_ip; +use maxgraph_server::heartbeat::Heartbeat; +use maxgraph_server::service::GremlinRpcService; +use maxgraph_server::{Store, StoreContext}; +use maxgraph_store::api::prelude::*; +use maxgraph_store::config::StoreConfig; + +use std::sync::mpsc::channel; +use std::sync::mpsc::Sender; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +pub fn init_with_rpc_service( + config: Arc, + hb_providers: Vec>, + hb_resp_senders: Vec>>, + store_context: StoreContext, +) where + VV: 'static + Vertex, + VVI: 'static + Iterator + Send, + EE: 'static + Edge, + EEI: 'static + Iterator + Send, + FS: 'static + Fn(&mut ServerHBReq) + Send, +{ + let store = Arc::new(Store::new(config.clone())); + let gremlin_service = GremlinRpcService::new(Arc::new(store_context)); + let (host, port) = start_all(store.clone(), gremlin_service); + info!("Start rpc service successfully {} {}", host, port); + let mut hb = Heartbeat::new( + host, + port as u32, + store.clone(), + hb_providers, + hb_resp_senders, + ); + hb.start(); +} + +pub(crate) fn start_all( + store: Arc, + gremlin_server: GremlinRpcService, +) -> (String, u16) +where + VV: 'static + Vertex, + VVI: 'static + Send + Iterator, + EE: 'static + Edge, + EEI: 'static + Send + Iterator, +{ + let (tx, rx) = channel(); + thread::spawn(move || { + let config = store.get_config(); + let gremlin_service = gremlin_query_grpc::create_gremlin_service(gremlin_server); + let env = Arc::new(Environment::new(config.rpc_thread_count as usize)); + let server_builder = ServerBuilder::new(env.clone()) + .channel_args(ChannelBuilder::new(env).reuse_port(false).build_args()) + .register_service(gremlin_service) + .bind("0.0.0.0", config.rpc_port as u16); + let mut server = server_builder.build().expect("Error when build rpc server"); + + server.start(); + let (_, port) = server.bind_addrs()[0]; + tx.send(port).unwrap(); + thread::sleep(Duration::from_millis(u64::max_value())); + }); + let ip = get_local_ip(); + let port = rx.recv().unwrap(); + info!("start service success, bind address: {}:{}", ip, port); + (ip, port) +} diff --git a/interactive_engine/groot-server/pom.xml b/interactive_engine/groot-server/pom.xml index 1fe2b37a41cd..59ca0a84f644 100644 --- a/interactive_engine/groot-server/pom.xml +++ b/interactive_engine/groot-server/pom.xml @@ -217,6 +217,11 @@ io.dropwizard.metrics metrics-core + + ir-compiler + com.alibaba.graphscope + 1.0-SNAPSHOT + diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ServiceProducerFactory.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ServiceProducerFactory.java index 121b33ef3b28..110ec310963d 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ServiceProducerFactory.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ServiceProducerFactory.java @@ -2,6 +2,7 @@ import com.alibaba.maxgraph.common.config.CommonConfig; import com.alibaba.maxgraph.common.config.Configs; +import com.alibaba.maxgraph.servers.ir.IrServiceProducer; import com.alibaba.maxgraph.servers.maxgraph.MaxGraphServiceProducer; public class ServiceProducerFactory { @@ -11,6 +12,8 @@ public static ComputeServiceProducer getProducer(Configs configs) { switch (engineType) { case "MAXGRAPH": return new MaxGraphServiceProducer(configs); + case "GAIA": + return new IrServiceProducer(configs); default: throw new IllegalArgumentException("Unknown engine type [" + engineType + "]"); } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/ExecutorEngine.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/ExecutorEngine.java new file mode 100644 index 000000000000..abd47c364aaa --- /dev/null +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/ExecutorEngine.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.maxgraph.servers.ir; + +import com.alibaba.graphscope.groot.discovery.NodeDiscovery; +import com.alibaba.graphscope.groot.store.GraphPartition; + +public interface ExecutorEngine extends NodeDiscovery.Listener { + + void init(); + + void addPartition(GraphPartition partition); + + void updatePartitionRouting(int partitionId, int serverId); + + void start(); + + void stop(); +} diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GaiaEngine.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GaiaEngine.java new file mode 100644 index 000000000000..d9f7aab9df3b --- /dev/null +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GaiaEngine.java @@ -0,0 +1,133 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.maxgraph.servers.ir; + +import com.alibaba.graphscope.groot.discovery.*; +import com.alibaba.graphscope.groot.store.GraphPartition; +import com.alibaba.graphscope.groot.store.jna.JnaGraphStore; +import com.alibaba.maxgraph.common.RoleType; +import com.alibaba.maxgraph.common.config.CommonConfig; +import com.alibaba.maxgraph.common.config.Configs; +import com.alibaba.maxgraph.compiler.api.exception.MaxGraphException; +import com.alibaba.maxgraph.servers.jna.GaiaLibrary; +import com.alibaba.maxgraph.servers.jna.GaiaPortsResponse; +import com.sun.jna.Pointer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +public class GaiaEngine implements ExecutorEngine { + private static final Logger logger = LoggerFactory.getLogger(GaiaEngine.class); + + private Configs configs; + private Pointer pointer; + private NodeDiscovery engineDiscovery; + private NodeDiscovery rpcDiscovery; + private LocalNodeProvider engineNodeProvider; + private LocalNodeProvider rpcNodeProvider; + private int nodeCount; + + private Map engineNodes = new ConcurrentHashMap<>(); + + public GaiaEngine(Configs configs, DiscoveryFactory discoveryFactory) { + this.configs = configs; + this.engineNodeProvider = new LocalNodeProvider(RoleType.GAIA_ENGINE, this.configs); + this.rpcNodeProvider = new LocalNodeProvider(RoleType.GAIA_RPC, this.configs); + this.engineDiscovery = discoveryFactory.makeDiscovery(this.engineNodeProvider); + this.rpcDiscovery = discoveryFactory.makeDiscovery(this.rpcNodeProvider); + this.nodeCount = CommonConfig.STORE_NODE_COUNT.get(configs); + } + + @Override + public void init() { + Configs engineConfigs = + Configs.newBuilder(this.configs) + .put( + "worker.num", + String.valueOf(CommonConfig.STORE_NODE_COUNT.get(this.configs))) + .build(); + byte[] configBytes = engineConfigs.toProto().toByteArray(); + this.pointer = GaiaLibrary.INSTANCE.initialize(configBytes, configBytes.length); + } + + @Override + public void addPartition(GraphPartition partition) { + int partitionId = partition.getId(); + if (partition instanceof JnaGraphStore) { + GaiaLibrary.INSTANCE.addPartition( + this.pointer, partitionId, ((JnaGraphStore) partition).getPointer()); + } + } + + @Override + public void updatePartitionRouting(int partitionId, int serverId) { + GaiaLibrary.INSTANCE.updatePartitionRouting(this.pointer, partitionId, serverId); + } + + @Override + public void start() { + try (GaiaPortsResponse gaiaPortsResponse = GaiaLibrary.INSTANCE.startEngine(this.pointer)) { + if (!gaiaPortsResponse.success) { + throw new MaxGraphException(gaiaPortsResponse.errMsg); + } + engineNodeProvider.apply(gaiaPortsResponse.enginePort); + rpcNodeProvider.apply(gaiaPortsResponse.rpcPort); + } + + this.engineDiscovery.start(); + this.engineDiscovery.addListener(this); + this.rpcDiscovery.start(); + } + + @Override + public void stop() { + this.engineDiscovery.removeListener(this); + this.engineDiscovery.stop(); + this.rpcDiscovery.stop(); + GaiaLibrary.INSTANCE.stopEngine(this.pointer); + } + + @Override + public void nodesJoin(RoleType role, Map nodes) { + if (role == RoleType.GAIA_ENGINE) { + this.engineNodes.putAll(nodes); + if (this.engineNodes.size() == this.nodeCount) { + String peerViewString = + nodes.values().stream() + .map( + n -> + String.format( + "%s#%s:%s", + n.getIdx(), n.getHost(), n.getPort())) + .collect(Collectors.joining(",")); + logger.info("updatePeerView [" + peerViewString + "]"); + GaiaLibrary.INSTANCE.updatePeerView(this.pointer, peerViewString); + } + } + } + + @Override + public void nodesLeft(RoleType role, Map nodes) { + if (role == RoleType.GAIA_ENGINE) { + nodes.keySet().forEach(k -> this.engineNodes.remove(k)); + } + } +} diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GaiaService.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GaiaService.java new file mode 100644 index 000000000000..1d0cd8f811cb --- /dev/null +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GaiaService.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.maxgraph.servers.ir; + +import com.alibaba.graphscope.groot.meta.MetaService; +import com.alibaba.graphscope.groot.store.GraphPartition; +import com.alibaba.graphscope.groot.store.StoreService; +import com.alibaba.maxgraph.common.config.CommonConfig; +import com.alibaba.maxgraph.common.config.Configs; +import com.alibaba.maxgraph.servers.AbstractService; + +public class GaiaService implements AbstractService { + + private Configs configs; + private ExecutorEngine engine; + private StoreService storeService; + private MetaService metaService; + + public GaiaService( + Configs configs, + ExecutorEngine engine, + StoreService storeService, + MetaService metaService) { + this.configs = configs; + this.engine = engine; + this.storeService = storeService; + this.metaService = metaService; + } + + @Override + public void start() { + this.engine.init(); + for (GraphPartition partition : this.storeService.getIdToPartition().values()) { + this.engine.addPartition(partition); + } + + int partitionCount = CommonConfig.PARTITION_COUNT.get(this.configs); + for (int i = 0; i < partitionCount; i++) { + this.engine.updatePartitionRouting(i, metaService.getStoreIdByPartition(i)); + } + this.engine.start(); + } + + @Override + public void stop() { + this.engine.stop(); + } +} diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GrootMetaFetcher.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GrootMetaFetcher.java new file mode 100644 index 000000000000..37dcfa99ec8b --- /dev/null +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/GrootMetaFetcher.java @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.maxgraph.servers.ir; + +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.common.utils.JsonUtils; +import com.alibaba.maxgraph.compiler.api.schema.*; +import com.google.common.collect.ImmutableMap; + +import org.apache.commons.lang3.tuple.Pair; + +import java.util.*; +import java.util.stream.Collectors; + +public class GrootMetaFetcher extends IrMetaFetcher { + private SchemaFetcher schemaFetcher; + + public GrootMetaFetcher(SchemaFetcher schemaFetcher) { + this.schemaFetcher = schemaFetcher; + } + + @Override + protected Optional getIrMeta() { + Pair pair = this.schemaFetcher.getSchemaSnapshotPair(); + GraphSchema schema; + if (pair != null && (schema = pair.getLeft()) != null) { + return Optional.of(parseSchema(schema)); + } else { + return Optional.empty(); + } + } + + private String parseSchema(GraphSchema graphSchema) { + List vertices = graphSchema.getVertexList(); + List edges = graphSchema.getEdgeList(); + List entities = new ArrayList(); + List relations = new ArrayList(); + vertices.forEach( + v -> { + entities.add(getVertex(graphSchema, v)); + }); + edges.forEach( + e -> { + relations.add(getEdge(graphSchema, e)); + }); + Map schemaMap = + ImmutableMap.of( + "entities", + entities, + "relations", + relations, + "is_table_id", + true, + "is_column_id", + true); + return JsonUtils.toJson(schemaMap); + } + + private Map getVertex(GraphSchema graphSchema, GraphVertex vertex) { + return getElement(graphSchema, vertex); + } + + private Map getEdge(GraphSchema graphSchema, GraphEdge edge) { + Map entity = new LinkedHashMap(getElement(graphSchema, edge)); + List relations = edge.getRelationList(); + List entityPairs = + relations.stream() + .map( + k -> { + GraphVertex src = k.getSource(); + GraphVertex dst = k.getTarget(); + return ImmutableMap.of( + "src", + ImmutableMap.of( + "id", + src.getLabelId(), + "name", + src.getLabel()), + "dst", + ImmutableMap.of( + "id", + dst.getLabelId(), + "name", + dst.getLabel())); + }) + .collect(Collectors.toList()); + entity.put("entity_pairs", entityPairs); + return entity; + } + + private Map getElement(GraphSchema graphSchema, GraphElement entity) { + String label = entity.getLabel(); + int labelId = entity.getLabelId(); + List properties = entity.getPropertyList(); + List columns = + properties.stream() + .map( + k -> { + int typeId = getDataTypeId(k.getDataType()); + String name = k.getName(); + int nameId = graphSchema.getPropertyId(name); + return ImmutableMap.of( + "key", + ImmutableMap.of("id", nameId, "name", name), + "data_type", + typeId); + }) + .collect(Collectors.toList()); + return ImmutableMap.of( + "label", ImmutableMap.of("id", labelId, "name", label), "columns", columns); + } + + private int getDataTypeId(DataType dataType) { + switch (dataType) { + case BOOL: + return 0; + case INT: + return 1; + case LONG: + return 2; + case DOUBLE: + return 3; + case STRING: + return 4; + case BYTES: + return 5; + case INT_LIST: + return 6; + case LONG_LIST: + return 7; + case DOUBLE_LIST: + return 8; + case STRING_LIST: + return 9; + case UNKNOWN: + default: + return 11; + } + } +} diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/IrServiceProducer.java new file mode 100644 index 000000000000..7a7f1809039c --- /dev/null +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/IrServiceProducer.java @@ -0,0 +1,122 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.maxgraph.servers.ir; + +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.gremlin.integration.result.TestGraphFactory; +import com.alibaba.graphscope.gremlin.service.IrGremlinServer; +import com.alibaba.graphscope.groot.discovery.DiscoveryFactory; +import com.alibaba.graphscope.groot.discovery.NodeDiscovery; +import com.alibaba.graphscope.groot.frontend.WriteSessionGenerator; +import com.alibaba.graphscope.groot.frontend.write.GraphWriter; +import com.alibaba.graphscope.groot.meta.MetaService; +import com.alibaba.graphscope.groot.rpc.ChannelManager; +import com.alibaba.graphscope.groot.store.StoreService; +import com.alibaba.maxgraph.common.RoleType; +import com.alibaba.maxgraph.common.config.CommonConfig; +import com.alibaba.maxgraph.common.config.Configs; +import com.alibaba.maxgraph.common.config.GremlinConfig; +import com.alibaba.maxgraph.compiler.api.schema.SchemaFetcher; +import com.alibaba.maxgraph.servers.AbstractService; +import com.alibaba.maxgraph.servers.ComputeServiceProducer; + +import java.util.HashMap; +import java.util.Map; + +public class IrServiceProducer implements ComputeServiceProducer { + private Configs configs; + + public IrServiceProducer(Configs configs) { + this.configs = configs; + } + + @Override + public AbstractService makeGraphService( + SchemaFetcher schemaFetcher, + ChannelManager channelManager, + NodeDiscovery discovery, + GraphWriter graphWriter, + WriteSessionGenerator writeSessionGenerator, + MetaService metaService) { + return makeGraphService(schemaFetcher, channelManager); + } + + @Override + public AbstractService makeGraphService( + SchemaFetcher schemaFetcher, ChannelManager channelManager) { + int executorCount = CommonConfig.STORE_NODE_COUNT.get(configs); + RpcChannelFetcher channelFetcher = + new RpcChannelManagerFetcher(channelManager, executorCount, RoleType.GAIA_RPC); + com.alibaba.graphscope.common.config.Configs irConfigs = getConfigs(); + IrMetaFetcher irMetaFetcher = new GrootMetaFetcher(schemaFetcher); + + return new AbstractService() { + private IrGremlinServer irGremlinServer = + new IrGremlinServer(GremlinConfig.GREMLIN_PORT.get(configs)); + + @Override + public void start() { + try { + irGremlinServer.start( + irConfigs, irMetaFetcher, channelFetcher, TestGraphFactory.GROOT); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void stop() { + try { + irGremlinServer.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + } + + @Override + public AbstractService makeExecutorService( + StoreService storeService, MetaService metaService, DiscoveryFactory discoveryFactory) { + ExecutorEngine executorEngine = new GaiaEngine(configs, discoveryFactory); + return new GaiaService(configs, executorEngine, storeService, metaService); + } + + private com.alibaba.graphscope.common.config.Configs getConfigs() { + Map configMap = new HashMap<>(); + + int executorCount = CommonConfig.STORE_NODE_COUNT.get(configs); + configMap.put(PegasusConfig.PEGASUS_SERVER_NUM.getKey(), String.valueOf(executorCount)); + + addToConfigMapIfExist(PegasusConfig.PEGASUS_WORKER_NUM.getKey(), configMap); + addToConfigMapIfExist(PegasusConfig.PEGASUS_TIMEOUT.getKey(), configMap); + addToConfigMapIfExist(PegasusConfig.PEGASUS_BATCH_SIZE.getKey(), configMap); + addToConfigMapIfExist(PegasusConfig.PEGASUS_OUTPUT_CAPACITY.getKey(), configMap); + addToConfigMapIfExist(PegasusConfig.PEGASUS_MEMORY_LIMIT.getKey(), configMap); + + return new com.alibaba.graphscope.common.config.Configs(configMap); + } + + private void addToConfigMapIfExist(String key, Map configMap) { + String value = configs.get(key); + if (value != null) { + configMap.put(key, value); + } + } +} diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/RpcChannelManagerFetcher.java b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/RpcChannelManagerFetcher.java new file mode 100644 index 000000000000..7e1ef952dcff --- /dev/null +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/maxgraph/servers/ir/RpcChannelManagerFetcher.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.maxgraph.servers.ir; + +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.graphscope.groot.rpc.ChannelManager; +import com.alibaba.maxgraph.common.RoleType; +import com.alibaba.pegasus.RpcChannel; + +import java.util.ArrayList; +import java.util.List; + +public class RpcChannelManagerFetcher implements RpcChannelFetcher { + private ChannelManager manager; + private int pegasusServerNum; + private RoleType targetRole; + + public RpcChannelManagerFetcher( + ChannelManager manager, int pegasusServerNum, RoleType targetRole) { + this.manager = manager; + this.pegasusServerNum = pegasusServerNum; + this.targetRole = targetRole; + this.manager.registerRole(this.targetRole); + } + + @Override + public List fetch() { + List channels = new ArrayList<>(); + for (int i = 0; i < pegasusServerNum; ++i) { + channels.add(new RpcChannel(manager.getChannel(this.targetRole, i))); + } + return channels; + } + + @Override + public boolean isDynamic() { + return true; + } +} diff --git a/interactive_engine/groot-server/src/main/resources/ir/integration/ir.modern.properties.json b/interactive_engine/groot-server/src/main/resources/ir/integration/ir.modern.properties.json new file mode 100644 index 000000000000..6f1aff76c588 --- /dev/null +++ b/interactive_engine/groot-server/src/main/resources/ir/integration/ir.modern.properties.json @@ -0,0 +1,48 @@ +{ + "vertex_properties": { + "1": { + "name": "marko", + "age": 29 + }, + "2": { + "name": "vadas", + "age": 27 + }, + "72057594037927939": { + "name": "lop", + "lang": "java" + }, + "4": { + "name": "josh", + "age": 32 + }, + "72057594037927941": { + "name": "ripple", + "lang": "java" + }, + "6": { + "name": "peter", + "age": 35 + } + }, + "edge_properties": { + "0": { + "weight": 0.5 + }, + "1": { + "weight": 0.4 + }, + "2": { + "weight": 1.0 + }, + "3": { + "weight": 0.4 + }, + "4": { + "weight": 1.0 + }, + "5": { + "weight": 0.2 + } + } +} \ No newline at end of file diff --git a/interactive_engine/ir-adaptor/ir_groot_ci.sh b/interactive_engine/ir-adaptor/ir_groot_ci.sh new file mode 100755 index 000000000000..6e69dd008525 --- /dev/null +++ b/interactive_engine/ir-adaptor/ir_groot_ci.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -x +base_dir=$(cd $(dirname $0); pwd) +ps -ef | grep "com.alibaba.maxgraph.servers.MaxNode" | grep -v grep | awk '{print $2}' | xargs kill -9 +cd ${base_dir}/.. +cd ./distribution/target/ && tar xvzf maxgraph.tar.gz && cd maxgraph + +# start server +maxgraph_dir=$(pwd) +sed -e "s@LOG4RS_CONFIG@${maxgraph_dir}/conf/log4rs.yml@g" \ + -e "s@engine.type=maxgraph@engine.type=gaia@g" \ + conf/config.template > /tmp/max_node_gaia.config +LOG_NAME=maxnode MAXGRAPH_CONF_FILE=/tmp/max_node_gaia.config ./bin/store_ctl.sh max_node_gaia & +sleep 10 + +# load data +cd ${base_dir}/../sdk && mvn -Dtest=com.alibaba.graphscope.groot.sdk.DataLoadingTest test +cd ${base_dir} && mvn test -Dtest=com.alibaba.graphscope.ir.maxgraph.IrGremlinTest -Dgremlin.endpoint="localhost:12312" +exit_code=$? +ps -ef | grep "com.alibaba.maxgraph.servers.MaxNode" | grep -v grep | awk '{print $2}' | xargs kill -9 + +# clean data +rm -fr ${maxgraph_dir}/data || true +if [ $exit_code -ne 0 ]; then + echo "ir integration test on groot store fail" + exit 1 +fi diff --git a/interactive_engine/ir-adaptor/pom.xml b/interactive_engine/ir-adaptor/pom.xml new file mode 100644 index 000000000000..a76c3e2b598e --- /dev/null +++ b/interactive_engine/ir-adaptor/pom.xml @@ -0,0 +1,26 @@ + + + + maxgraph-parent + com.alibaba.maxgraph + 0.0.1-SNAPSHOT + + 4.0.0 + + ir-adaptor + + + + ir-compiler + com.alibaba.graphscope + 1.0-SNAPSHOT + + + com.alibaba.maxgraph + maxgraph-frontendservice + ${project.parent.version} + + + \ No newline at end of file diff --git a/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinServiceMain.java b/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinServiceMain.java new file mode 100644 index 000000000000..186c9131b314 --- /dev/null +++ b/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinServiceMain.java @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.ir.maxgraph; + +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.gremlin.integration.result.TestGraphFactory; +import com.alibaba.graphscope.gremlin.service.IrGremlinServer; +import com.alibaba.maxgraph.common.cluster.InstanceConfig; +import com.alibaba.maxgraph.common.rpc.RpcAddressFetcher; +import com.alibaba.maxgraph.frontendservice.ClientManager; +import com.alibaba.maxgraph.frontendservice.server.ExecutorAddressFetcher; + +import java.util.HashMap; +import java.util.Map; + +// for vineyard service +public class IrGremlinServiceMain { + public IrGremlinServiceMain(InstanceConfig instanceConfig) throws Exception { + Configs configs = getConfigs(instanceConfig); + + IrMetaFetcher irMetaFetcher = getStoreConfigs(instanceConfig); + + ClientManager clientManager = new ClientManager(instanceConfig); + RpcAddressFetcher addressFetcher = new ExecutorAddressFetcher(clientManager); + RpcChannelFetcher channelFetcher = new RpcAddressChannelFetcher(addressFetcher); + + IrGremlinServer gremlinServer = new IrGremlinServer(instanceConfig.getGremlinServerPort()); + gremlinServer.start(configs, irMetaFetcher, channelFetcher, TestGraphFactory.UNKNOWN); + } + + private Configs getConfigs(InstanceConfig instanceConfig) { + Map configMap = new HashMap<>(); + configMap.put( + PegasusConfig.PEGASUS_WORKER_NUM.getKey(), + String.valueOf(instanceConfig.getPegasusWorkerNum())); + configMap.put( + PegasusConfig.PEGASUS_TIMEOUT.getKey(), + String.valueOf(instanceConfig.getPegasusTimeoutMS())); + configMap.put( + PegasusConfig.PEGASUS_BATCH_SIZE.getKey(), + String.valueOf(instanceConfig.getPegasusBatchSize())); + configMap.put( + PegasusConfig.PEGASUS_OUTPUT_CAPACITY.getKey(), + String.valueOf(instanceConfig.getPegasusOutputCapacity())); + configMap.put( + PegasusConfig.PEGASUS_MEMORY_LIMIT.getKey(), + String.valueOf(instanceConfig.getPegasusMemoryLimit())); + configMap.put( + PegasusConfig.PEGASUS_SERVER_NUM.getKey(), + String.valueOf(instanceConfig.getResourceExecutorCount())); + return new Configs(configMap); + } + + private IrMetaFetcher getStoreConfigs(InstanceConfig instanceConfig) { + return new VineyardMetaFetcher(instanceConfig); + } +} diff --git a/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/RpcAddressChannelFetcher.java b/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/RpcAddressChannelFetcher.java new file mode 100644 index 000000000000..4954178637c0 --- /dev/null +++ b/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/RpcAddressChannelFetcher.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.ir.maxgraph; + +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.maxgraph.common.rpc.RpcAddressFetcher; +import com.alibaba.maxgraph.sdkcommon.client.Endpoint; +import com.alibaba.pegasus.RpcChannel; + +import java.util.List; +import java.util.stream.Collectors; + +public class RpcAddressChannelFetcher implements RpcChannelFetcher { + private RpcAddressFetcher addressFetcher; + + public RpcAddressChannelFetcher(RpcAddressFetcher addressFetcher) { + this.addressFetcher = addressFetcher; + } + + @Override + public List fetch() { + List endpoints = addressFetcher.getServiceAddress(); + return endpoints.stream() + .map(k -> new RpcChannel(k.getIp(), k.getRuntimeCtrlAndAsyncPort())) + .collect(Collectors.toList()); + } + + @Override + public boolean isDynamic() { + return true; + } +} diff --git a/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/VineyardMetaFetcher.java b/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/VineyardMetaFetcher.java new file mode 100644 index 000000000000..9ac81d1ccbd2 --- /dev/null +++ b/interactive_engine/ir-adaptor/src/main/java/com/alibaba/graphscope/ir/maxgraph/VineyardMetaFetcher.java @@ -0,0 +1,157 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.ir.maxgraph; + +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.common.utils.JsonUtils; +import com.alibaba.maxgraph.common.cluster.InstanceConfig; +import com.alibaba.maxgraph.compiler.api.schema.*; +import com.alibaba.maxgraph.compiler.schema.JsonFileSchemaFetcher; +import com.google.common.collect.ImmutableMap; + +import java.util.*; +import java.util.stream.Collectors; + +public class VineyardMetaFetcher extends IrMetaFetcher { + private InstanceConfig instanceConfig; + + public VineyardMetaFetcher(InstanceConfig instanceConfig) { + this.instanceConfig = instanceConfig; + super.fetch(); + } + + @Override + protected Optional getIrMeta() { + String schemaPath = instanceConfig.getVineyardSchemaPath(); + JsonFileSchemaFetcher fetcher = new JsonFileSchemaFetcher(schemaPath); + GraphSchema graphSchema = fetcher.getSchemaSnapshotPair().getLeft(); + return Optional.of(parseSchema(graphSchema)); + } + + @Override + public void fetch() { + // do nothing + } + + private String parseSchema(GraphSchema graphSchema) { + List vertices = graphSchema.getVertexList(); + List edges = graphSchema.getEdgeList(); + List entities = new ArrayList(); + List relations = new ArrayList(); + vertices.forEach( + v -> { + entities.add(getVertex(graphSchema, v)); + }); + edges.forEach( + e -> { + relations.add(getEdge(graphSchema, e)); + }); + Map schemaMap = + ImmutableMap.of( + "entities", + entities, + "relations", + relations, + "is_table_id", + true, + "is_column_id", + true); + return JsonUtils.toJson(schemaMap); + } + + private Map getVertex(GraphSchema graphSchema, GraphVertex vertex) { + return getElement(graphSchema, vertex); + } + + private Map getEdge(GraphSchema graphSchema, GraphEdge edge) { + Map entity = new LinkedHashMap(getElement(graphSchema, edge)); + List relations = edge.getRelationList(); + List entityPairs = + relations.stream() + .map( + k -> { + GraphVertex src = k.getSource(); + GraphVertex dst = k.getTarget(); + return ImmutableMap.of( + "src", + ImmutableMap.of( + "id", + src.getLabelId(), + "name", + src.getLabel()), + "dst", + ImmutableMap.of( + "id", + dst.getLabelId(), + "name", + dst.getLabel())); + }) + .collect(Collectors.toList()); + entity.put("entity_pairs", entityPairs); + return entity; + } + + private Map getElement(GraphSchema graphSchema, GraphElement entity) { + String label = entity.getLabel(); + int labelId = entity.getLabelId(); + List properties = entity.getPropertyList(); + List columns = + properties.stream() + .map( + k -> { + int typeId = getDataTypeId(k.getDataType()); + String name = k.getName(); + int nameId = graphSchema.getPropertyId(name); + return ImmutableMap.of( + "key", + ImmutableMap.of("id", nameId, "name", name), + "data_type", + typeId); + }) + .collect(Collectors.toList()); + return ImmutableMap.of( + "label", ImmutableMap.of("id", labelId, "name", label), "columns", columns); + } + + private int getDataTypeId(DataType dataType) { + switch (dataType) { + case BOOL: + return 0; + case INT: + return 1; + case LONG: + return 2; + case DOUBLE: + return 3; + case STRING: + return 4; + case BYTES: + return 5; + case INT_LIST: + return 6; + case LONG_LIST: + return 7; + case DOUBLE_LIST: + return 8; + case STRING_LIST: + return 9; + case UNKNOWN: + default: + return 11; + } + } +} diff --git a/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinTest.java b/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinTest.java new file mode 100644 index 000000000000..2a0abc1a9e60 --- /dev/null +++ b/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinTest.java @@ -0,0 +1,10 @@ +package com.alibaba.graphscope.ir.maxgraph; + +import com.alibaba.graphscope.gremlin.integration.graph.RemoteTestGraphProvider; + +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.junit.runner.RunWith; + +@RunWith(IrGremlinTestSuite.class) +@GraphProviderClass(provider = RemoteTestGraphProvider.class, graph = RemoteTestGraph.class) +public class IrGremlinTest {} diff --git a/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinTestSuite.java b/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinTestSuite.java new file mode 100644 index 000000000000..feb87fa60ac0 --- /dev/null +++ b/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/IrGremlinTestSuite.java @@ -0,0 +1,92 @@ +package com.alibaba.graphscope.ir.maxgraph; + +import org.apache.tinkerpop.gremlin.AbstractGremlinSuite; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerBuilder; + +public class IrGremlinTestSuite extends AbstractGremlinSuite { + + private static final Class[] allTests = + new Class[] { + // branch + RepeatTest.Traversals.class, + UnionTest.Traversals.class, + + // filter + CyclicPathTest.Traversals.class, + DedupTest.Traversals.class, + FilterTest.Traversals.class, + HasTest.Traversals.class, + IsTest.Traversals.class, + RangeTest.Traversals.class, + SimplePathTest.Traversals.class, + WhereTest.Traversals.class, + + // map + org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest.Traversals.class, + GraphTest.Traversals.class, + OrderTest.Traversals.class, + PathTest.Traversals.class, + PropertiesTest.Traversals.class, + SelectTest.Traversals.class, + VertexTest.Traversals.class, + UnfoldTest.Traversals.class, + ValueMapTest.Traversals.class, + GroupTest.Traversals.class, + GroupCountTest.Traversals.class, + + // match + MatchTest.CountMatchTraversals.class, + }; + + private static final Class[] testsToEnforce = + new Class[] { + // branch + RepeatTest.Traversals.class, + UnionTest.Traversals.class, + + // filter + CyclicPathTest.Traversals.class, + DedupTest.Traversals.class, + FilterTest.Traversals.class, + HasTest.Traversals.class, + IsTest.Traversals.class, + RangeTest.Traversals.class, + SimplePathTest.Traversals.class, + WhereTest.Traversals.class, + + // map + org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest.Traversals.class, + GraphTest.Traversals.class, + OrderTest.Traversals.class, + PathTest.Traversals.class, + PropertiesTest.Traversals.class, + SelectTest.Traversals.class, + VertexTest.Traversals.class, + UnfoldTest.Traversals.class, + ValueMapTest.Traversals.class, + GroupTest.Traversals.class, + GroupCountTest.Traversals.class, + + // match + MatchTest.CountMatchTraversals.class, + }; + + public IrGremlinTestSuite(final Class klass, final RunnerBuilder builder) + throws InitializationError { + super(klass, builder, allTests, testsToEnforce, false, TraversalEngine.Type.STANDARD); + } + + public IrGremlinTestSuite( + final Class klass, final RunnerBuilder builder, final Class[] testsToExecute) + throws InitializationError { + super(klass, builder, testsToExecute, testsToEnforce, true, TraversalEngine.Type.STANDARD); + } +} diff --git a/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/RemoteTestGraph.java b/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/RemoteTestGraph.java new file mode 100644 index 000000000000..c43e40cce68b --- /dev/null +++ b/interactive_engine/ir-adaptor/src/test/java/com/alibaba/graphscope/ir/maxgraph/RemoteTestGraph.java @@ -0,0 +1,1476 @@ +package com.alibaba.graphscope.ir.maxgraph; + +import org.apache.commons.configuration2.Configuration; +import org.apache.tinkerpop.gremlin.structure.Graph; + +@Graph.OptIn("com.alibaba.graphscope.ir.maxgraph.IrGremlinTestSuite") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_groupCount_selectXvaluesX_unfold_dedup", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_repeatXdedupX_timesX2X_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_out_in_valuesXnameX_fold_dedupXlocalX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_group_byXlabelX_byXbothE_weight_dedup_foldX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasIdXwithoutXemptyXX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasNotXageX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_containingXarkXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_endingWithXasXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_startingWithXmarXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXlocationX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXlabel_isXsoftwareXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1AsStringX_out_hasXid_2AsStringX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_outXcreatedX_hasXname__mapXlengthX_isXgtX3XXX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXperson_name_containingXoX_andXltXmXXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_not_startingWithXmarXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_not_containingXarkXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_bothE_properties_dedup_hasKeyXweightX_hasValueXltX0d3XX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1X_out_hasXid_2AsString_3AsStringX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasLabelXperson_software_blahX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_not_endingWithXasXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_gtXmX_andXcontainingXoXXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_both_dedup_properties_hasKeyXageX_hasValueXgtX30XX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_EX11X_outV_outE_hasXid_10X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_notXhasIdXwithinXemptyXXX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1X_out_hasXid_lt_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = + "g_V_hasLabelXpersonX_hasXage_notXlteX10X_andXnotXbetweenX11_20XXXX_andXltX29X_orXeqX35XXXX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_both_dedup_properties_hasKeyXageX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_bothE_properties_dedup_hasKeyXweightX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasIdXemptyX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasIdXwithinXemptyXX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_E_hasLabelXuses_traversesX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_EX11X_outV_outE_hasXid_10AsStringX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", + method = "g_V_whereXinXcreatedX_count_isX1XX_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", + method = "g_V_whereXinXcreatedX_count_isXgte_2XX_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_outE_valuesXweightX_fold_orderXlocalX_skipXlocal_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_in_asXaX_in_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_limitXlocal_1X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_in_asXaX_in_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_limitXlocal_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_hasLabelXpersonX_order_byXageX_valuesXnameX_skipX1X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_out_asXaX_out_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_rangeXlocal_1_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_out_asXaX_out_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_rangeXlocal_1_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_out_asXaX_out_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_rangeXlocal_4_5X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_rangeX1_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_repeatXbothX_timesX3X_rangeX5_11X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_in_asXbX_in_asXcX_selectXa_b_cX_byXnameX_limitXlocal_1X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_in_asXbX_in_asXcX_selectXa_b_cX_byXnameX_limitXlocal_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_localXoutE_limitX1X_inVX_limitX3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_hasLabelXpersonX_order_byXageX_skipX1X_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_VX1X_outXcreatedX_inEXcreatedX_rangeX1_3X_outV", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_selectXa_b_cX_byXnameX_rangeXlocal_1_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_selectXa_b_cX_byXnameX_rangeXlocal_1_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_simplePath_byXlabelX_fromXbX_toXcX_path_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_out_asXbX_whereXandXasXaX_outXknowsX_asXbX__orXasXbX_outXcreatedX_hasXname_rippleX__asXbX_inXknowsX_count_isXnotXeqX0XXXXX_selectXa_bX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXa_eqXbXX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_withSideEffectXa_josh_peterX_VX1X_outXcreatedX_inXcreatedX_name_whereXwithinXaXX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXa_outXknowsX_bX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_V_whereXoutXcreatedX_and_outXknowsX_or_inXknowsXX_valuesXnameX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_whereXnotXoutXcreatedXXX_name", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_out_asXbX_whereXin_count_isXeqX3XX_or_whereXoutXcreatedX_and_hasXlabel_personXXX_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_withSideEffectXa_g_VX2XX_VX1X_out_whereXneqXaXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_outEXcreatedX_asXbX_inV_asXcX_inXcreatedX_asXdX_whereXa_ltXbX_orXgtXcXX_andXneqXdXXX_byXageX_byXweightX_byXinXcreatedX_valuesXageX_minX_selectXa_c_dX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXb_hasXname_markoXX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_VX1X_out_aggregateXxX_out_whereXnotXwithinXaXXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_outXcreatedX_asXbX_whereXandXasXbX_in__notXasXaX_outXcreatedX_hasXname_rippleXXX_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_VX1X_repeatXbothEXcreatedX_whereXwithoutXeXX_aggregateXeX_otherVX_emit_path", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXa_neqXbXX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_fold_countXlocalX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_both_both_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_whereXinXkknowsX_outXcreatedX_count_is_0XX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = "g_V_hasLabelXpersonX_asXpX_VXsoftwareX_addInEXuses_pX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = "g_V_outXknowsX_V_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = + "g_V_hasXname_GarciaX_inXsungByX_asXsongX_V_hasXname_Willie_DixonX_inXwrittenByX_whereXeqXsongXX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_group_byXlabelX_byXname_order_byXdescX_foldX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXsongX_order_byXperformances_descX_byXnameX_rangeX110_120X_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_V_asXvX_mapXbothE_weight_foldX_sumXlocalX_asXsX_selectXv_sX_order_byXselectXsX_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_mapXbothE_weight_foldX_order_byXsumXlocalX_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXpersonX_fold_orderXlocalX_byXageX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_V_hasXsong_name_OHBOYX_outXfollowedByX_outXfollowedByX_order_byXperformancesX_byXsongType_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXpersonX_group_byXnameX_byXoutE_weight_sumX_orderXlocalX_byXvaluesX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_descX_byXkeys_ascX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_name_order_byXa1_b1X_byXb2_a2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_properties_order_byXkey_descX_key", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXpersonX_order_byXvalueXageX_descX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_VX1X_elementMap_orderXlocalX_byXkeys_descXunfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_V_hasLabelXpersonX_group_byXnameX_byXoutE_weight_sumX_unfold_order_byXvalues_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_both_hasLabelXpersonX_order_byXage_descX_name", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", +// method = "g_V_order_byXname_incrX_name", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_order_byXoutE_count_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_order_byXname_a1_b1X_byXname_b2_a2X_name", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", +// method = "g_V_outE_order_byXweight_decrX_weight", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_propertiesXage_nameX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_properties_hasXid_nameIdX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_injectXg_VX1X_propertiesXnameX_nextX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_properties_hasXid_nameIdAsStringX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_propertiesXname_ageX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_propertiesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasXperson_name_markoX_barrier_asXaX_outXknows_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_hasXperson_name_markoX_elementMapXnameX_asXaX_unionXidentity_identityX_selectXaX_selectXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXall_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXlast_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_untilXout_outX_repeatXin_asXaX_in_asXbXX_selectXa_bX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXfirst_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_out_aggregateXxX_asXbX_selectXa_bX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_label_groupCount_asXxX_selectXxX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXfirst_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_outXcreatedX_unionXasXprojectX_inXcreatedX_hasXname_markoX_selectXprojectX__asXprojectX_inXcreatedX_inXknowsX_hasXname_markoX_selectXprojectXX_groupCount_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXlast_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasLabelXpersonX_asXpX_mapXbothE_label_groupCountX_asXrX_selectXp_rX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_chooseXoutE_count_isX0X__asXaX__asXbXX_chooseXselectXaX__selectXaX__selectXbXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_selectXkeysX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_unfold_selectXvaluesX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXfirst_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_unfold_selectXkeysX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXlast_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXall_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasXperson_name_markoX_path_asXaX_unionXidentity_identityX_selectXaX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_groupXmX_by_byXbothE_countX_barrier_selectXmX_selectXselectXaXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_asXaX_groupXmX_by_byXbothE_countX_barrier_selectXmX_selectXselectXaXX_byXmathX_plus_XX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_outXknowsX_asXbX_localXselectXa_bX_byXnameXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_hasXname_gremlinX_inEXusesX_order_byXskill_ascX_asXaX_outV_asXbX_selectXa_bX_byXskillX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXlast_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_untilXout_outX_repeatXin_asXaXX_selectXaX_byXtailXlocalX_nameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXa_bX_out_asXcX_path_selectXkeysX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_VX1X_groupXaX_byXconstantXaXX_byXnameX_selectXaX_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXaX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_outXknowsX_asXaX_selectXall_constantXaXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_outE_weight_groupCount_selectXvaluesX_unfold_groupCount_selectXvaluesX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXall_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_hasLabelXsoftwareX_asXnameX_asXlanguageX_asXcreatorsX_selectXname_language_creatorsX_byXnameX_byXlangX_byXinXcreatedX_name_fold_orderXlocalXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXfirst_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_VX1X_asXaX_repeatXout_asXaXX_timesX2X_selectXfirst_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXall_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasXperson_name_markoX_count_asXaX_unionXidentity_identityX_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_selectXvaluesX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX4X_bothE_hasXweight_lt_1X_otherV", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_V_hasLabelXloopsX_bothEXselfX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1_2_3_4X_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_V_hasLabelXpersonX_V_hasLabelXsoftwareX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1X_outEXknowsX_bothV_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_V_hasLabelXloopsX_bothXselfX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest", + method = "g_V_valueMap_unfold_mapXkeyX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest", + method = "g_V_localXoutE_foldX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest", + method = "g_VX1X_repeatXboth_simplePathX_untilXhasIdX6XX_path_byXnameX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_withSideEffectXa__marko_666_noone_blahX_V_groupXaX_byXnameX_byXoutE_label_foldX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_repeatXout_groupXaX_byXnameX_byXcountX_timesX2X_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXnameX_by", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXlabelX_byXbothE_groupXaX_byXlabelX_byXweight_sumX_weight_sumX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_V_hasLabelXpersonX_asXpX_outXcreatedX_group_byXnameX_byXselectXpX_valuesXageX_sumX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_hasXlangX_groupXaX_byXlangX_byXnameX_out_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXoutE_countX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXaX_byXlabelX_byXoutE_weight_sumX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = + "g_V_unionXoutXknowsX__outXcreatedX_inXcreatedXX_groupCount_selectXvaluesX_unfold_sum", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = + "g_V_unionXrepeatXoutX_timesX2X_groupCountXmX_byXlangXX__repeatXinX_timesX2X_groupCountXmX_byXnameXX_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_groupCount_byXbothE_countX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_repeatXout_groupCountXaX_byXnameXX_timesX2X_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_groupCount_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_groupCountXxX_capXxX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_both_groupCountXaX_out_capXaX_selectXkeysX_unfold_both_groupCountXaX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_name_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_groupCountXaX_byXnameX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_name_groupCountXaX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = + "g_V_both_groupCountXaX_byXlabelX_asXbX_barrier_whereXselectXaX_selectXsoftwareX_isXgtX2XXX_selectXbX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_hasXnoX_groupCountXaX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_V_repeatXoutXknowsXX_untilXrepeatXoutXcreatedXX_emitXhasXname_lopXXX_path_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X_emit_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXout_repeatXoutX_timesX1XX_timesX1X_limitX1X_path_by_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_emit_repeatXoutX_timesX2X_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_repeatXoutX_untilXoutE_count_isX0XX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXbothX_untilXname_eq_marko_or_loops_gt_1X_groupCount_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_repeatXgroupCountXmX_byXloopsX_outX_timesX3X_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_VX1X_repeatXrepeatXunionXout_uses_out_traversesXX_whereXloops_isX0X_timesX1X_timeX2X_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_emitXhasXlabel_personXX_repeatXoutX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_V_hasXname_markoX_repeatXoutE_inV_simplePathX_untilXhasXname_rippleXX_path_byXnameX_byXlabelX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_V_untilXconstantXtrueXX_repeatXrepeatXout_createdXX_untilXhasXname_rippleXXXemit_lang", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_VX6X_repeatXa_bothXcreatedX_simplePathX_emitXrepeatXb_bothXknowsXX_untilXloopsXbX_asXb_whereXloopsXaX_asXbX_hasXname_vadasXX_dedup_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_VX3X_repeatXbothX_createdXX_untilXloops_is_40XXemit_repeatXin_knowsXX_emit_loopsXisX1Xdedup_values", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_hasXloop_name_loopX_repeatXinX_timesX5X_path_by_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXgroupCountXmX_byXnameX_outX_timesX2X_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXrepeatXout_createdXX_untilXhasXname_rippleXXXemit_lang", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_emit_repeatXa_outXknows_filterXloops_isX0XX_lang", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X_emit", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_emit_timesX2X_repeatXoutX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1_2X_localXunionXoutE_count__inE_count__outE_weight_sumXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_V_chooseXlabel_is_person__unionX__out_lang__out_nameX__in_labelX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1_2X_localXunionXcountXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_V_chooseXlabel_is_person__unionX__out_lang__out_nameX__in_labelX_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = + "g_V_unionXrepeatXunionXoutXcreatedX__inXcreatedXX_timesX2X__repeatXunionXinXcreatedX__outXcreatedXX_timesX2XX_label_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = + "g_V_asXaX_repeatXbothX_timesX3X_emit_name_asXbX_group_byXselectXaXX_byXselectXbX_dedup_order_foldX_selectXvaluesX_unfold_dedup", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_both_dedup_byXlabelX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_hasXlabel_softwareX_dedup_byXlangX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_out_asXxX_in_asXyX_selectXx_yX_byXnameX_fold_dedupXlocal_x_yX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_name_order_byXa_bX_dedup_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_asXaX_outXcreatedX_asXbX_inXcreatedX_asXcX_dedupXa_bX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_both_dedup_byXoutE_countX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_asXaX_both_asXbX_dedupXa_bX_byXlabelX_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = + "g_V_both_group_by_byXout_dedup_foldX_unfold_selectXvaluesX_unfold_out_order_byXnameX_limitX1X_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXfalseX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_VX1X_out_filterXage_gt_30X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_E_filterXfalseX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_VX1X_filterXage_gt_30X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXname_startsWith_m_OR_name_startsWith_pX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXtrueX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_E_filterXtrueX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXlang_eq_javaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_VX2X_filterXage_gt_30X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXaX_byXnameX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_out_group_byXlabelX_selectXpersonX_unfold_outXcreatedX_name_limitX2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_V_repeatXunionXoutXknowsX_groupXaX_byXageX__outXcreatedX_groupXbX_byXnameX_byXcountXX_groupXaX_byXnameXX_timesX2X_capXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_hasLabelXsongX_group_byXnameX_byXproperties_groupCount_byXlabelXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXname_substring_1X_byXconstantX1XX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXmX_byXnameX_byXinXknowsX_nameX_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXaX_byXname_substring_1X_byXconstantX1XX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_hasXnoX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_VX1X_name_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXbothE_countX_byXgroup_byXlabelXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_hasXnoX_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest", + method = "g_VX1X_asXaX_outXcreatedX_asXbX_inXcreatedX_asXcX_cyclicPath_fromXaX_toXbX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_repeatXoutX_timesX3X_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_repeatXoutX_timesX8X_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_asXaX_hasXname_markoX_asXbX_hasXage_29X_asXcX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_out_out_path_byXnameX_byXageX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_repeatXoutX_timesX2X_path_byXitX_byXnameX_byXlangX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_VX1X_out_path_byXageX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_path_fromXbX_toXcX_byXnameX", + reason = "unsupported") + +// todo: return label is integer +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1X_outE", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX4X_bothEXcreatedX", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX4X_bothE", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_E_hasXlabelXknowsX", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX2X_inE", + reason = "returned label is id") +@Graph.OptOut( + method = + "g_VX1X_outEXknowsX_asXhereX_hasXweight_1X_asXfakeX_inV_hasXname_joshX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") +@Graph.OptOut( + method = "g_VX1X_outEXknowsX_asXhereX_hasXweight_1X_inV_hasXname_joshX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") +@Graph.OptOut( + method = "g_VX1X_outEXknowsX_hasXweight_1X_asXhereX_inV_hasXname_joshX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") +@Graph.OptOut( + method = "g_VX1X_outE_asXhereX_inV_hasXname_vadasX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") + +// add more ignored tests which are out of ir range +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_timesX2X_repeatXoutX_name", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXbothX_timesX10X_asXaX_out_asXbX_selectXa_bX", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X_repeatXinX_timesX2X_name", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1X_unionXrepeatXoutX_timesX2X__outX_name", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_cyclicPath_path", + reason = "cyclicPath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_cyclicPath", + reason = "cyclicPath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_properties_dedup_count", + reason = "properties is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_bothE_properties_dedup_count", + reason = "properties is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_properties_properties_dedup_count", + reason = "properties is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VXv1X_hasXage_gt_30X", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VXv4X_hasXage_gt_30X", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXk_withinXcXX_valuesXkX", + reason = "addV is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_simplePath", + reason = "simplePath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest", + method = "g_V_repeatXboth_simplePathX_timesX3X_path", + reason = "simplePath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = "g_VX1X_V_valuesXnameX", + reason = "g.V().V() is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_VX1X_asXaX_repeatXout_asXaXX_timesX2X_selectXlast_aX", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VXv1X_out", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1X_to_XOUT_knowsX", + reason = "to(Direction.OUT,\"knows\") is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VXlistXv1_v2_v3XX_name", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_VX1X_valueMapXname_locationX_byXunfoldX_by", + reason = "valueMap().by(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMap_withXtokensX", + reason = "valueMap(true) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX_withXtokensX", + reason = "valueMap(true) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX_withXtokens_idsX_byXunfoldX", + reason = "valueMap().with(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_hasLabelXpersonX_filterXoutEXcreatedXX_valueMap_withXtokensX", + reason = "valueMap().with(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX_withXtokens_labelsX_byXunfoldX", + reason = "valueMap().with(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXmX_byXlabelX_byXlabel_countX_capXmX", + reason = "group side effect is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_hasXperson_name_markoX_bothXknowsX_groupCount_byXvaluesXnameX_foldX", + reason = "fold() excluding group().by(...).by(fold()) is unsupported") + +// @Graph.OptOut(method="g_V_unionXout__inX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_outE_asXeX_inV_asXvX_selectXeX_order_byXweight_ascX_selectXvX_valuesXnameX_dedup" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", reason = "will be supported") +// @Graph.OptOut(method="g_V_hasXperson_name_markoX_age" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_V_in_hasIdXneqX1XX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_hasXage_isXgt_30XX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EX7X_hasXlabelXknowsX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + reason = "will be supported") +// @Graph.OptOut(method="g_VX1X_outE_hasXweight_inside_0_06X_inV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_valuesXageX_isXgte_29X_isXlt_34X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_valuesXageX_isXlte_30X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_valuesXageX_isX32X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_out_limitX2X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_outXknowsX_outEXcreatedX_rangeX0_1X_inV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_outXknowsX_outXcreatedX_rangeX0_1X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_asXaX_outXcreatedX_inXcreatedX_whereXeqXaXX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outEXcreatedX_asXbX_inV_asXcX_whereXa_gtXbX_orXeqXbXXX_byXageX_byXweightX_byXweightX_selectXa_cX_byXnameX" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be supported") +// @Graph.OptOut(method="g_VX1X_asXaX_outXcreatedX_inXcreatedX_whereXneqXaXX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +@Graph.OptOut( + method = + "g_VX1X_asXaX_outXcreatedX_inXcreatedX_asXbX_whereXasXbX_outXcreatedX_hasXname_rippleXX_valuesXage_nameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + reason = "values('age', 'name') is unsupported") +// @Graph.OptOut(method="g_VX1X_asXaX_outXcreatedX_inXcreatedX_asXbX_whereXa_neqXbXX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_whereXasXaX_name_isXjoshXX_inXcreatedX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_asXbX_inXcreatedX_asXcX_bothXknowsX_bothXknowsX_asXdX_whereXc__notXeqXaX_orXeqXdXXXX_selectXa_b_c_dX" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_inXcreatedX_asXbX_whereXa_gtXbXX_byXageX_selectXa_bX_byXnameX" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be supported") +// @Graph.OptOut(method="g_V_hasLabelXpersonX_order_byXageX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_asXbX_order_byXshuffleX_selectXa_bX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_order_byXname_ascX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_order_byXnameX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_outE_order_byXweight_descX_weight" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_both_hasLabelXpersonX_order_byXage_descX_limitX5X_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_VX1X_outEXcreatedX_inV_inE_outV_path", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EX11X_propertiesXweightX_asXaX_selectXaX_byXkeyX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_asXaX_hasXname_markoX_asXbX_asXcX_selectXa_b_cX_by_byXnameX_byXageX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "one object can only have one alias") +// @Graph.OptOut(method="g_VX1X_asXaX_outXknowsX_asXbX_selectXa_bX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_V_hasXname_isXmarkoXX_asXaX_selectXaX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_asXaX_name_order_asXbX_selectXa_bX_byXnameX_by_XitX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_whereXoutXknowsXX_selectXaX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_EX11X_propertiesXweightX_asXaX_selectXaX_byXvalueX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "will be supported") +// @Graph.OptOut(method="g_VX1X_outE_otherV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_VX1AsStringX_outXknowsX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_out_outE_inV_inE_inV_both_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_EXe11X", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +// @Graph.OptOut(method="g_EX11X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_outE_hasXweight_1X_outV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_EXlistXe7_e11XX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EX11AsStringX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EXe7_e11X", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_valueMapXname_ageX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_VX1X_outXcreatedX_valueMap", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_valueMap", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_group_byXlabelX_byXlabel_countX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_group_byXageX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", reason = "will +// be supported") + +// @Graph.OptOut(method="g_V_hasXp_neqXvXX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXage_withoutX27X_count" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_VX1X_hasXcircumferenceX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXage_withoutX27_29X_count" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_VX1X_hasXnameX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXblahX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXlangX_group_byXlangX_byXcountX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", reason = +// "existence of property is unsupported") +// @Graph.OptOut(method="g_VX1X_asXaX_out_hasXageX_whereXgtXaXX_byXageX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "existence +// of property is unsupported") + +// complex steps nested in match is unsupported yet, i.e. where(eq)/count/order/match +@Graph.OptOut( + method = + "g_V_matchXa_whereXa_neqXcXX__a_created_b__orXa_knows_vadas__a_0knows_and_a_hasXlabel_personXX__b_0created_c__b_0created_count_isXgtX1XXX_selectXa_b_cX_byXidX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_created_lop_b__b_0created_29_c__c_whereXrepeatXoutX_timesX2XXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_knows_b__andXa_created_c__b_created_c__andXb_created_count_d__a_knows_count_dXXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_hasXsong_name_sunshineX__a_mapX0followedBy_weight_meanX_b__a_0followedBy_c__c_filterXweight_whereXgteXbXXX_outV_dX_selectXdX_byXnameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_created_b__a_repeatXoutX_timesX2XX_selectXa_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_knows_b__b_created_lop__b_matchXb_created_d__d_0created_cX_selectXcX_cX_selectXa_b_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_asXaX_out_asXbX_matchXa_out_count_c__orXa_knows_b__b_in_count_c__and__c_isXgtX2XXXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_outEXcreatedX_order_byXweight_descX_limitX1X_inV_b__b_hasXlang_javaXX_selectXa_bX_byXnameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_created_lop_b__b_0created_29_cX_whereXc_repeatXoutX_timesX2XX_selectXa_b_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_asXaX_out_asXbX_matchXa_out_count_c__b_in_count_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXwhereXandXa_created_b__b_0created_count_isXeqX3XXXX__a_both_b__whereXb_inXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_knows_count_bX_selectXbX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_valueMap_matchXa_selectXnameX_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// steps nested in match is unsupported yet, will be supported latter, i.e values('name') +@Graph.OptOut( + method = "g_V_notXmatchXa_age_b__a_name_cX_whereXb_eqXcXX_selectXaXX_name", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_hasLabelXsongsX_matchXa_name_b__a_performances_cX_selectXb_cX_count", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_outXknowsX_name_bX_identity", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// will be supported in the first milestone, i.e dedup('a', 'b') +@Graph.OptOut( + method = "g_V_matchXa__a_both_b__b_both_cX_dedupXa_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa__a_both_b__b_both_cX_dedupXa_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_both_b__b_both_cX_dedupXa_bX_byXlabelX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXd_0knows_a__d_hasXname_vadasX__a_knows_b__b_created_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_out_bX_selectXb_idX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_knows_b__b_created_c__a_created_cX_dedupXa_b_cX_selectXaX_byXnameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// need grateful graph +@Graph.OptOut( + method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__b_followedBy_c__c_writtenBy_d__whereXd_neqXaXXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// cases unsupported in groot +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1X_out_hasXid_2_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXp_neqXvXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXblahX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1X_hasXcircumferenceX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_EX11X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX", + reason = "unsupported") +public class RemoteTestGraph + extends com.alibaba.graphscope.gremlin.integration.graph.RemoteTestGraph { + + public RemoteTestGraph(Configuration configuration) { + super(configuration); + } +} diff --git a/interactive_engine/pom.xml b/interactive_engine/pom.xml index 690db591bf7b..417ca9a3218a 100644 --- a/interactive_engine/pom.xml +++ b/interactive_engine/pom.xml @@ -26,6 +26,9 @@ tests benchmark distribution + ../research/engine/pegasus/clients/java/client + ../research/query_service/ir/compiler + ir-adaptor diff --git a/interactive_engine/sdk/src/test/java/com/alibaba/graphscope/groot/sdk/DataLoadingTest.java b/interactive_engine/sdk/src/test/java/com/alibaba/graphscope/groot/sdk/DataLoadingTest.java index edfb9e944a8b..408d381fe91d 100644 --- a/interactive_engine/sdk/src/test/java/com/alibaba/graphscope/groot/sdk/DataLoadingTest.java +++ b/interactive_engine/sdk/src/test/java/com/alibaba/graphscope/groot/sdk/DataLoadingTest.java @@ -39,6 +39,7 @@ public void testGetSchema() { @Test public void testZddData() throws Exception { Thread.sleep(5000); + client.initWriteSession(); Map v1 = new HashMap<>(); v1.put("id", "1"); diff --git a/k8s/Makefile b/k8s/Makefile index 357fb282c6a6..5ef017346119 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -113,6 +113,7 @@ graphscope-darwin-py3: rm -fr build dist/*.whl || true && \ sudo strip -s $(WORKING_DIR)/../analytical_engine/exported_symbols_osx.lds /opt/graphscope/bin/grape_engine && \ sudo strip /opt/graphscope/bin/executor && \ + sudo strip /opt/graphscope/bin/gaia_executor && \ export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \ package_name=gs-lib python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \ rm -fr build && \ @@ -152,6 +153,7 @@ graphscope-manylinux2014-py3: rm -fr build dist/*.whl && \ sudo strip /opt/graphscope/bin/grape_engine && \ sudo strip /opt/graphscope/bin/executor && \ + sudo strip /opt/graphscope/bin/gaia_executor && \ sudo strip /opt/graphscope/lib/*.so && \ strip /tmp/gs/builtin/*/*.so && \ package_name=gs-lib python3 setup.py bdist_wheel && \ diff --git a/k8s/graphscope-dev.Dockerfile b/k8s/graphscope-dev.Dockerfile index 1552266bff3e..602361a9d997 100644 --- a/k8s/graphscope-dev.Dockerfile +++ b/k8s/graphscope-dev.Dockerfile @@ -63,6 +63,7 @@ COPY --from=builder /usr/local/bin/zetcd /opt/graphscope/bin/zetcd COPY --from=builder /home/graphscope/gs/k8s/precompile.py /tmp/precompile.py COPY --from=builder /home/graphscope/gs/k8s/kube_ssh /opt/graphscope/bin/kube_ssh COPY --from=builder /home/graphscope/gs/interactive_engine/executor/target/$profile/executor /opt/graphscope/bin/executor +COPY --from=builder /home/graphscope/gs/interactive_engine/executor/target/$profile/gaia_executor /opt/graphscope/bin/gaia_executor COPY --from=builder /home/graphscope/gs/interactive_engine/assembly/target/maxgraph-assembly-0.0.1-SNAPSHOT.tar.gz /opt/graphscope/maxgraph-assembly-0.0.1-SNAPSHOT.tar.gz # install mars diff --git a/k8s/graphscope-store.Dockerfile b/k8s/graphscope-store.Dockerfile index 6ef865701e22..cdbd47e4bda5 100644 --- a/k8s/graphscope-store.Dockerfile +++ b/k8s/graphscope-store.Dockerfile @@ -29,6 +29,7 @@ RUN sudo chown -R $(id -u):$(id -g) /home/graphscope/gs /home/graphscope/.m2 && && cmake .. && make -j && sudo make install \ && rm -fr /tmp/cppkafka \ && echo "build with profile: $profile" \ + && cd /home/graphscope/gs/research/query_service/ir/core && cargo build --release \ && cd /home/graphscope/gs/interactive_engine \ && if [ "$profile" = "release" ]; then \ echo "release mode"; \ diff --git a/research/dyn_type/src/error.rs b/research/dyn_type/src/error.rs index e789a34e984d..fd4ff1405c9e 100644 --- a/research/dyn_type/src/error.rs +++ b/research/dyn_type/src/error.rs @@ -39,6 +39,9 @@ impl fmt::Display for CastError { RawType::Float => write!(f, "can't cast f64 into {}", self.target), RawType::Blob(len) => write!(f, "can't cast Blob({}) into {}", len, self.target), RawType::String => write!(f, "can't cast String into {}", self.target), + RawType::Vector => write!(f, "can't cast Vector into {}", self.target), + RawType::KV => write!(f, "can't cast KV into {}", self.target), + RawType::None => write!(f, "can't cast None into {}", self.target), RawType::Unknown => write!(f, "can't cast unknown dyn type into {}", self.target), } } diff --git a/research/dyn_type/src/lib.rs b/research/dyn_type/src/lib.rs index 03fa0315130a..55db2e445439 100644 --- a/research/dyn_type/src/lib.rs +++ b/research/dyn_type/src/lib.rs @@ -22,16 +22,16 @@ extern crate dyn_clonable; pub mod arith; pub mod error; pub mod object; -pub mod serde_dyn; #[macro_use] pub mod macros; pub mod serde; +pub mod serde_dyn; use dyn_clonable::*; pub use error::CastError; pub use object::{BorrowObject, Object, OwnedOrRef, Primitives}; pub use serde_dyn::{de_dyn_obj, register_type}; -use std::any::Any; +use std::any::{Any, TypeId}; use std::fmt::Debug; use std::io; @@ -39,3 +39,39 @@ use std::io; pub trait DynType: Any + Send + Sync + Clone + Debug { fn to_bytes(&self) -> io::Result>; } + +/// copy from std::any::Any; +impl dyn DynType { + pub fn is(&self) -> bool { + // Get `TypeId` of the type this function is instantiated with. + let t = TypeId::of::(); + + // Get `TypeId` of the type in the trait dyn_type (`self`). + let concrete = self.type_id(); + + // Compare both `TypeId`s on equality. + t == concrete + } + + pub fn try_downcast_ref(&self) -> Option<&T> { + if self.is::() { + // SAFETY: just checked whether we are pointing to the correct type, and we can rely on + // that check for memory safety because we have implemented Any for all types; no other + // impls can exist as they would conflict with our impl. + unsafe { Some(&*(self as *const dyn DynType as *const T)) } + } else { + None + } + } + + pub fn try_downcast_mut(&mut self) -> Option<&mut T> { + if self.is::() { + // SAFETY: just checked whether we are pointing to the correct type, and we can rely on + // that check for memory safety because we have implemented Any for all types; no other + // impls can exist as they would conflict with our impl. + unsafe { Some(&mut *(self as *mut dyn DynType as *mut T)) } + } else { + None + } + } +} diff --git a/research/dyn_type/src/macros.rs b/research/dyn_type/src/macros.rs index bc4e4bb1a98e..7b4eeb933399 100644 --- a/research/dyn_type/src/macros.rs +++ b/research/dyn_type/src/macros.rs @@ -13,8 +13,6 @@ //! See the License for the specific language governing permissions and //! limitations under the License. -// TODO(longbin) We've only implement where $obj is a type that is supported by `$crate::Object::from` function. - #[macro_export] macro_rules! object { ($obj:expr) => { diff --git a/research/dyn_type/src/object.rs b/research/dyn_type/src/object.rs index e10a97fdb3d2..09d698867f30 100644 --- a/research/dyn_type/src/object.rs +++ b/research/dyn_type/src/object.rs @@ -15,9 +15,11 @@ use crate::{try_downcast, try_downcast_ref, CastError, DynType}; use core::any::TypeId; +use itertools::Itertools; use std::any::Any; use std::borrow::Cow; use std::cmp::Ordering; +use std::collections::BTreeMap; use std::convert::TryFrom; use std::fmt::Debug; use std::hash::{Hash, Hasher}; @@ -32,6 +34,9 @@ pub enum RawType { Float, String, Blob(usize), + Vector, + KV, + None, Unknown, } @@ -104,15 +109,9 @@ impl Primitives { pub fn as_i8(&self) -> Result { match self { Primitives::Byte(v) => Ok(*v), - Primitives::Integer(v) => { - i8::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) - } - Primitives::Long(v) => { - i8::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - i8::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Integer(v) => i8::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)), + Primitives::Long(v) => i8::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => i8::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -124,12 +123,8 @@ impl Primitives { Primitives::Integer(v) => { i16::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) } - Primitives::Long(v) => { - i16::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - i16::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Long(v) => i16::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => i16::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -139,12 +134,8 @@ impl Primitives { match self { Primitives::Byte(v) => Ok(*v as i32), Primitives::Integer(v) => Ok(*v), - Primitives::Long(v) => { - i32::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - i32::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Long(v) => i32::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => i32::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -155,9 +146,7 @@ impl Primitives { Primitives::Byte(v) => Ok(*v as i64), Primitives::Integer(v) => Ok(*v as i64), Primitives::Long(v) => Ok(*v), - Primitives::ULLong(v) => { - i64::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::ULLong(v) => i64::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -178,18 +167,10 @@ impl Primitives { #[inline] pub fn as_u8(&self) -> Result { match self { - Primitives::Byte(v) => { - u8::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)) - } - Primitives::Integer(v) => { - u8::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) - } - Primitives::Long(v) => { - u8::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - u8::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Byte(v) => u8::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)), + Primitives::Integer(v) => u8::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)), + Primitives::Long(v) => u8::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => u8::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -197,18 +178,12 @@ impl Primitives { #[inline] pub fn as_u16(&self) -> Result { match self { - Primitives::Byte(v) => { - u16::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)) - } + Primitives::Byte(v) => u16::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)), Primitives::Integer(v) => { u16::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) } - Primitives::Long(v) => { - u16::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - u16::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Long(v) => u16::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => u16::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -216,18 +191,12 @@ impl Primitives { #[inline] pub fn as_u32(&self) -> Result { match self { - Primitives::Byte(v) => { - u32::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)) - } + Primitives::Byte(v) => u32::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)), Primitives::Integer(v) => { u32::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) } - Primitives::Long(v) => { - u32::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - u32::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Long(v) => u32::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => u32::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -235,18 +204,12 @@ impl Primitives { #[inline] pub fn as_u64(&self) -> Result { match self { - Primitives::Byte(v) => { - u64::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)) - } + Primitives::Byte(v) => u64::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)), Primitives::Integer(v) => { u64::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) } - Primitives::Long(v) => { - u64::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } - Primitives::ULLong(v) => { - u64::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)) - } + Primitives::Long(v) => u64::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), + Primitives::ULLong(v) => u64::try_from(*v).map_err(|_| CastError::new::(RawType::ULLong)), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } } @@ -259,15 +222,11 @@ impl Primitives { #[inline] pub fn as_u128(&self) -> Result { match self { - Primitives::Byte(v) => { - u128::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)) - } + Primitives::Byte(v) => u128::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)), Primitives::Integer(v) => { u128::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) } - Primitives::Long(v) => { - u128::try_from(*v).map_err(|_| CastError::new::(RawType::Long)) - } + Primitives::Long(v) => u128::try_from(*v).map_err(|_| CastError::new::(RawType::Long)), Primitives::ULLong(v) => Ok(*v), Primitives::Float(_) => Err(CastError::new::(RawType::Float)), } @@ -276,9 +235,7 @@ impl Primitives { #[inline] pub fn as_f64(&self) -> Result { match self { - Primitives::Byte(v) => { - f64::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)) - } + Primitives::Byte(v) => f64::try_from(*v).map_err(|_| CastError::new::(RawType::Byte)), Primitives::Integer(v) => { f64::try_from(*v).map_err(|_| CastError::new::(RawType::Integer)) } @@ -409,7 +366,10 @@ impl PartialEq for Primitives { }, Primitives::ULLong(v) => match other { Primitives::Float(o) => (*v as f64).eq(o), - _ => other.as_u128().map(|o| o == *v).unwrap_or(false), + _ => other + .as_u128() + .map(|o| o == *v) + .unwrap_or(false), }, Primitives::Float(v) => other.as_f64().map(|o| o == *v).unwrap_or(false), } @@ -437,64 +397,35 @@ impl PartialOrd for Primitives { Primitives::Float(o) => (*v as f64).partial_cmp(o), _ => other.as_u128().map(|o| v.cmp(&o)).ok(), }, - Primitives::Float(v) => other.as_f64().map(|o| v.partial_cmp(&o)).unwrap_or(None), - } - } -} - -/// copy from std::any::Any; -impl dyn DynType { - pub fn is(&self) -> bool { - // Get `TypeId` of the type this function is instantiated with. - let t = TypeId::of::(); - - // Get `TypeId` of the type in the trait dyn_type (`self`). - let concrete = self.type_id(); - - // Compare both `TypeId`s on equality. - t == concrete - } - - pub fn try_downcast_ref(&self) -> Option<&T> { - if self.is::() { - // SAFETY: just checked whether we are pointing to the correct type, and we can rely on - // that check for memory safety because we have implemented Any for all types; no other - // impls can exist as they would conflict with our impl. - unsafe { Some(&*(self as *const dyn DynType as *const T)) } - } else { - None - } - } - - pub fn try_downcast_mut(&mut self) -> Option<&mut T> { - if self.is::() { - // SAFETY: just checked whether we are pointing to the correct type, and we can rely on - // that check for memory safety because we have implemented Any for all types; no other - // impls can exist as they would conflict with our impl. - unsafe { Some(&mut *(self as *mut dyn DynType as *mut T)) } - } else { - None + Primitives::Float(v) => other + .as_f64() + .map(|o| v.partial_cmp(&o)) + .unwrap_or(None), } } } -// Is dyn type needed in dyn_type; #[derive(Clone, Debug)] pub enum Object { Primitive(Primitives), String(String), + Vector(Vec), + KV(BTreeMap), Blob(Box<[u8]>), DynOwned(Box), + None, } impl ToString for Object { fn to_string(&self) -> String { - use Object::*; match self { - Primitive(p) => p.to_string(), - String(s) => s.to_string(), - Blob(_) => unimplemented!(), - DynOwned(_) => unimplemented!(), + Object::Primitive(p) => p.to_string(), + Object::String(s) => s.to_string(), + Object::Vector(v) => format!("{:?}", v), + Object::KV(kv) => format!("{:?}", kv), + Object::Blob(b) => format!("{:?}", b), + Object::DynOwned(_) => "unknown dynamic type".to_string(), + Object::None => "".to_string(), } } } @@ -504,9 +435,12 @@ impl ToString for Object { pub enum BorrowObject<'a> { Primitive(Primitives), String(&'a str), + Vector(&'a [Object]), + KV(&'a BTreeMap), Blob(&'a [u8]), /// To borrow from `Object::DynOwned`, and it can be cloned back to `Object::DynOwned` DynRef(&'a Box), + None, } impl<'a> ToString for BorrowObject<'a> { @@ -515,19 +449,48 @@ impl<'a> ToString for BorrowObject<'a> { match self { Primitive(p) => p.to_string(), String(s) => s.to_string(), - Blob(_) => unimplemented!(), - DynRef(_) => unimplemented!(), + Vector(v) => format!("{:?}", v), + KV(kv) => format!("{:?}", kv), + Blob(b) => format!("{:?}", b), + DynRef(_) => "unknown dynamic type".to_string(), + None => "None".to_string(), } } } +macro_rules! contains_single { + ($self: expr, $single: expr, $ty: ident) => { + match $self { + $crate::$ty::Vector(vec) => match $single { + $crate::$ty::Primitive(_) | $crate::$ty::String(_) => { + for val in vec.iter() { + if $single == val { + return true; + } + } + false + } + _ => false, + }, + $crate::$ty::String(str1) => match $single { + $crate::$ty::String(str2) => str1.contains(str2), + _ => false, + }, + _ => false, + } + }; +} + impl Object { pub fn raw_type(&self) -> RawType { match self { Object::Primitive(p) => p.raw_type(), Object::String(_) => RawType::String, + Object::Vector(_) => RawType::Vector, + Object::KV(_) => RawType::KV, Object::Blob(b) => RawType::Blob(b.len()), Object::DynOwned(_) => RawType::Unknown, + Object::None => RawType::None, } } @@ -535,8 +498,11 @@ impl Object { match self { Object::Primitive(p) => BorrowObject::Primitive(*p), Object::String(v) => BorrowObject::String(v.as_str()), + Object::Vector(v) => BorrowObject::Vector(v.as_slice()), + Object::KV(kv) => BorrowObject::KV(kv), Object::Blob(v) => BorrowObject::Blob(v.as_ref()), Object::DynOwned(v) => BorrowObject::DynRef(v), + Object::None => BorrowObject::None, } } @@ -556,10 +522,10 @@ impl Object { #[inline] pub fn as_i8(&self) -> Result { - if let Object::Primitive(p) = self { - p.as_i8() - } else { - Err(CastError::new::(self.raw_type())) + match self { + Object::Primitive(p) => p.as_i8(), + Object::DynOwned(x) => try_downcast!(x, i8), + _ => Err(CastError::new::(self.raw_type())), } } @@ -601,10 +567,10 @@ impl Object { #[inline] pub fn as_u8(&self) -> Result { - if let Object::Primitive(p) = self { - p.as_u8() - } else { - Err(CastError::new::(self.raw_type())) + match self { + Object::Primitive(p) => p.as_u8(), + Object::DynOwned(x) => try_downcast!(x, u8), + _ => Err(CastError::new::(self.raw_type())), } } @@ -659,7 +625,8 @@ impl Object { Object::String(str) => Ok(Cow::Borrowed(str.as_str())), Object::Blob(b) => Ok(String::from_utf8_lossy(b)), Object::DynOwned(x) => try_downcast!(x, String, as_str).map(|r| Cow::Borrowed(r)), - Object::Primitive(p) => Err(CastError::new::(p.raw_type())), + Object::None => Ok(Cow::Borrowed("")), + _ => Err(CastError::new::(self.raw_type())), } } @@ -670,6 +637,8 @@ impl Object { Object::String(str) => Ok(str.as_bytes()), Object::Blob(v) => Ok(v.as_ref()), Object::DynOwned(x) => try_downcast!(x, Vec, as_slice), + Object::None => Ok(&[]), + _ => Err(CastError::new::<&[u8]>(self.raw_type())), } } @@ -680,10 +649,9 @@ impl Object { Ok(OwnedOrRef::Owned(v)) } Object::String(x) => try_transmute!(x, T, RawType::String).map(|v| OwnedOrRef::Ref(v)), - Object::Blob(x) => { - try_transmute!(x, T, RawType::Blob(x.len())).map(|v| OwnedOrRef::Ref(v)) - } + Object::Blob(x) => try_transmute!(x, T, RawType::Blob(x.len())).map(|v| OwnedOrRef::Ref(v)), Object::DynOwned(x) => try_downcast_ref!(x, T).map(|v| OwnedOrRef::Ref(v)), + _ => Err(CastError::new::>(self.raw_type())), } } @@ -697,8 +665,26 @@ impl Object { Err(CastError::new::(RawType::Unknown)) } } - Object::Primitive(p) => Err(CastError::new::(p.raw_type())), - Object::Blob(_) => unimplemented!(), + _ => Err(CastError::new::(self.raw_type())), + } + } + + fn contains_single(&self, single: &Object) -> bool { + contains_single!(self, single, Object) + } + + pub fn contains(&self, obj: &Object) -> bool { + match obj { + Object::Vector(vec) => { + for val in vec.iter() { + if !self.contains_single(val) { + return false; + } + } + true + } + Object::Primitive(_) | Object::String(_) => self.contains_single(obj), + _ => false, } } } @@ -708,8 +694,11 @@ impl<'a> BorrowObject<'a> { match self { BorrowObject::Primitive(p) => p.raw_type(), BorrowObject::String(_) => RawType::String, + BorrowObject::Vector(_) => RawType::Vector, + BorrowObject::KV(_) => RawType::KV, BorrowObject::Blob(b) => RawType::Blob(b.len()), BorrowObject::DynRef(_) => RawType::Unknown, + BorrowObject::None => RawType::None, } } @@ -832,7 +821,8 @@ impl<'a> BorrowObject<'a> { BorrowObject::String(str) => Ok(Cow::Borrowed(*str)), BorrowObject::Blob(b) => Ok(String::from_utf8_lossy(b)), BorrowObject::DynRef(x) => try_downcast!(x, String, as_str).map(|r| Cow::Borrowed(r)), - BorrowObject::Primitive(p) => Err(CastError::new::(p.raw_type())), + BorrowObject::None => Ok(Cow::Borrowed("")), + _ => Err(CastError::new::(self.raw_type())), } } @@ -843,84 +833,194 @@ impl<'a> BorrowObject<'a> { BorrowObject::String(v) => Ok(v.as_bytes()), BorrowObject::Blob(v) => Ok(*v), BorrowObject::DynRef(v) => try_downcast!(v, Vec, as_slice), + BorrowObject::None => Ok(&[]), + _ => Err(CastError::new::<&[u8]>(self.raw_type())), } } pub fn try_to_owned(&self) -> Option { - match self { - BorrowObject::Primitive(p) => Some(Object::Primitive(*p)), - BorrowObject::String(s) => Some(Object::String((*s).to_owned())), + match *self { + BorrowObject::Primitive(p) => Some(Object::Primitive(p)), + BorrowObject::String(s) => Some(Object::String(s.to_owned())), + BorrowObject::Vector(v) => Some(Object::Vector(v.to_vec())), + BorrowObject::KV(kv) => Some(Object::KV(kv.clone())), BorrowObject::Blob(b) => Some(Object::Blob(b.to_vec().into_boxed_slice())), BorrowObject::DynRef(d) => Some(Object::DynOwned((*d).clone())), + BorrowObject::None => Some(Object::None), } } + + fn contains_single(&self, single: &BorrowObject) -> bool { + contains_single!(self, single, BorrowObject) + } + + pub fn contains(&self, obj: &BorrowObject) -> bool { + match obj { + BorrowObject::Vector(vec) => { + for val in vec.iter() { + if !self.contains_single(&val.as_borrow()) { + return false; + } + } + true + } + BorrowObject::Primitive(_) | BorrowObject::String(_) => self.contains_single(obj), + _ => false, + } + } +} + +impl<'a> PartialEq for BorrowObject<'a> { + fn eq(&self, other: &Object) -> bool { + self.eq(&other.as_borrow()) + } +} + +impl<'a> PartialEq> for Object { + fn eq(&self, other: &BorrowObject<'a>) -> bool { + self.as_borrow().eq(other) + } +} + +macro_rules! eq { + ($self:expr, $other:expr, $ty:ident) => { + match $self { + $crate::$ty::Primitive(p) => $other + .as_primitive() + .map(|o| p == &o) + .unwrap_or(false), + $crate::$ty::Blob(v) => $other + .as_bytes() + .map(|o| o.eq(v.as_ref())) + .unwrap_or(false), + $crate::$ty::String(v) => $other + .as_str() + .map(|o| o.eq(&(*v))) + .unwrap_or(false), + $crate::$ty::Vector(v1) => { + if let $crate::$ty::Vector(v2) = $other { + v1 == v2 + } else { + false + } + } + $crate::$ty::KV(kv1) => { + if let $crate::$ty::KV(kv2) = $other { + kv1 == kv2 + } else { + false + } + } + $crate::$ty::None => { + if let $crate::$ty::None = $other { + true + } else { + false + } + } + _ => false, + } + }; } impl PartialEq for Object { fn eq(&self, other: &Self) -> bool { - match self { - Object::Primitive(p) => other.as_primitive().map(|o| p == &o).unwrap_or(false), - Object::Blob(v) => other.as_bytes().map(|o| o.eq(v.as_ref())).unwrap_or(false), - Object::String(v) => other.as_str().map(|o| o.eq(v.as_str())).unwrap_or(false), - // TODO(longbin) Should be able to compare a DynType - Object::DynOwned(_) => false, - } + eq!(self, other, Object) } } impl Eq for Object {} -impl PartialOrd for Object { - fn partial_cmp(&self, other: &Self) -> Option { - match self { - Object::Primitive(p) => other - .as_primitive() - .map(|o| p.partial_cmp(&o)) - .unwrap_or(None), - Object::Blob(v) => other +macro_rules! partial_cmp { + ($self:expr, $other:expr, $ty:ident) => { + match $self { + $crate::$ty::Primitive(p) => { + if let $crate::$ty::None = $other { + Some(Ordering::Greater) + } else { + $other + .as_primitive() + .map(|o| p.partial_cmp(&o)) + .unwrap_or(None) + } + } + $crate::$ty::Blob(v) => $other .as_bytes() .map(|o| v.as_ref().partial_cmp(o)) .unwrap_or(None), - Object::String(v) => other + $crate::$ty::String(v) => $other .as_str() - .map(|o| v.as_str().partial_cmp(o.as_ref())) + .map(|o| (&(**v)).partial_cmp(&(*o))) .unwrap_or(None), - // TODO(longbin) Should be able to compare a DynType - Object::DynOwned(_) => None, + $crate::$ty::Vector(v1) => { + if let $crate::$ty::Vector(v2) = $other { + v1.partial_cmp(v2) + } else if let $crate::$ty::None = $other { + Some(Ordering::Greater) + } else { + None + } + } + $crate::$ty::KV(kv1) => { + if let $crate::$ty::KV(kv2) = $other { + kv1.partial_cmp(kv2) + } else if let $crate::$ty::None = $other { + Some(Ordering::Greater) + } else { + None + } + } + $crate::$ty::None => { + if let $crate::$ty::None = $other { + Some(Ordering::Equal) + } else { + Some(Ordering::Less) + } + } + _ => None, + } + }; +} + +macro_rules! cmp { + ($self:expr, $other:expr) => { + if let Some(ord) = $self.partial_cmp($other) { + ord + } else { + Ordering::Less } + }; +} + +impl PartialOrd for Object { + fn partial_cmp(&self, other: &Self) -> Option { + partial_cmp!(self, other, Object) + } +} + +impl Ord for Object { + fn cmp(&self, other: &Self) -> Ordering { + cmp!(self, other) } } impl<'a> PartialEq for BorrowObject<'a> { fn eq(&self, other: &Self) -> bool { - match self { - BorrowObject::Primitive(p) => other.as_primitive().map(|o| p == &o).unwrap_or(false), - BorrowObject::String(v) => other.as_str().map(|o| o.eq(*v)).unwrap_or(false), - BorrowObject::Blob(v) => other.as_bytes().map(|o| *v == o).unwrap_or(false), - // TODO(longbin) Should be able to compare a DynType - BorrowObject::DynRef(_) => false, - } + eq!(self, other, BorrowObject) } } +impl<'a> Eq for BorrowObject<'a> {} + impl<'a> PartialOrd for BorrowObject<'a> { fn partial_cmp(&self, other: &Self) -> Option { - match self { - BorrowObject::Primitive(p) => other - .as_primitive() - .map(|o| p.partial_cmp(&o)) - .unwrap_or(None), - BorrowObject::String(v) => other - .as_str() - .map(|o| (*v).partial_cmp(o.as_ref())) - .unwrap_or(None), - BorrowObject::Blob(v) => other - .as_bytes() - .map(|o| (*v).partial_cmp(o)) - .unwrap_or(None), - // TODO(longbin) Should be able to compare a DynType - BorrowObject::DynRef(_) => None, - } + partial_cmp!(self, other, BorrowObject) + } +} + +impl<'a> Ord for BorrowObject<'a> { + fn cmp(&self, other: &Self) -> Ordering { + cmp!(self, other) } } @@ -942,36 +1042,74 @@ fn integer_decode(val: f64) -> (u64, i16, i8) { (mantissa, exponent, sign) } -impl Hash for Object { - fn hash(&self, state: &mut H) { - match self { - Object::Primitive(p) => match p { - Primitives::Byte(v) => { - v.hash(state); +macro_rules! hash { + ($self:expr, $state:expr, $ty:ident) => { + match $self { + $crate::$ty::Primitive(p) => match p { + $crate::Primitives::Byte(v) => { + v.hash($state); } - Primitives::Integer(v) => { - v.hash(state); + $crate::Primitives::Integer(v) => { + v.hash($state); } - Primitives::Long(v) => { - v.hash(state); + $crate::Primitives::Long(v) => { + v.hash($state); } - Primitives::Float(v) => { - integer_decode(*v).hash(state); + $crate::Primitives::Float(v) => { + integer_decode(*v).hash($state); } - Primitives::ULLong(v) => { - v.hash(state); + $crate::Primitives::ULLong(v) => { + v.hash($state); } }, - Object::String(s) => { - s.hash(state); + $crate::$ty::String(s) => { + s.hash($state); + } + $crate::$ty::Blob(b) => { + b.hash($state); } - Object::Blob(b) => { - b.hash(state); + $crate::$ty::Vector(v) => { + v.hash($state); + } + $crate::$ty::KV(kv) => { + for pair in kv.iter() { + pair.hash($state); + } } - // TODO(longbin) Should be able to hash a DynType - Object::DynOwned(_) => { - unimplemented!() + $crate::$ty::None => { + "".hash($state); } + _ => unimplemented!(), + } + }; +} + +impl Hash for Object { + fn hash(&self, state: &mut H) { + hash!(self, state, Object) + } +} + +impl<'a> Hash for BorrowObject<'a> { + fn hash(&self, state: &mut H) { + hash!(self, state, BorrowObject) + } +} + +impl From> for Object { + fn from(obj_opt: Option) -> Self { + match obj_opt { + Some(obj) => obj, + None => Object::None, + } + } +} + +impl<'a> From>> for BorrowObject<'a> { + fn from(obj_opt: Option>) -> Self { + match obj_opt { + Some(obj) => obj, + None => BorrowObject::None, } } } @@ -1128,14 +1266,24 @@ impl From for Object { } } -impl<'a> From> for Object { - fn from(s: BorrowObject<'a>) -> Self { - match s { - BorrowObject::Primitive(p) => Object::Primitive(p), - BorrowObject::Blob(blob) => Object::Blob(blob.to_vec().into_boxed_slice()), - BorrowObject::String(s) => Object::String(s.to_string()), - _ => unimplemented!(), - } +impl> From> for Object { + fn from(vec: Vec) -> Self { + Object::Vector( + vec.into_iter() + .map(|val| val.into()) + .collect_vec(), + ) + } +} + +impl, V: Into> From> for Object { + fn from(tuples: Vec<(K, V)>) -> Self { + Object::KV( + tuples + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect::>(), + ) } } diff --git a/research/dyn_type/src/serde.rs b/research/dyn_type/src/serde.rs index 03a08a2ab838..a9278de9ba17 100644 --- a/research/dyn_type/src/serde.rs +++ b/research/dyn_type/src/serde.rs @@ -14,8 +14,9 @@ //! limitations under the License. use crate::{de_dyn_obj, Object, Primitives}; -use core::any::TypeId; use pegasus_common::codec::{Decode, Encode, ReadExt, WriteExt}; +use std::any::TypeId; +use std::collections::BTreeMap; use std::io; impl Encode for Primitives { @@ -88,19 +89,36 @@ impl Encode for Object { str.write_to(writer)?; Ok(()) } - Object::Blob(b) => { + Object::Vector(vec) => { writer.write_u8(2)?; + vec.write_to(writer)?; + Ok(()) + } + Object::KV(kv) => { + writer.write_u8(3)?; + writer.write_u64(kv.len() as u64)?; + for (key, val) in kv { + key.write_to(writer)?; + val.write_to(writer)?; + } + Ok(()) + } + Object::Blob(b) => { + writer.write_u8(4)?; let len = b.len(); writer.write_u64(len as u64)?; writer.write_all(&(**b))?; Ok(()) } Object::DynOwned(dyn_type) => { - writer.write_u8(3)?; + writer.write_u8(5)?; let bytes = (**dyn_type).to_bytes()?; bytes.write_to(writer)?; Ok(()) } + Object::None => { + writer.write_u8(6) + } } } } @@ -118,6 +136,20 @@ impl Decode for Object { Ok(Object::String(str)) } 2 => { + let vec = >::read_from(reader)?; + Ok(Object::Vector(vec)) + } + 3 => { + let len = ::read_from(reader)?; + let mut map = BTreeMap::new(); + for _ in 0..len { + let key = ::read_from(reader)?; + let value = ::read_from(reader)?; + map.insert(key, value); + } + Ok(Object::KV(map)) + } + 4 => { let len = ::read_from(reader)?; let mut b = vec![]; for _i in 0..len { @@ -126,7 +158,7 @@ impl Decode for Object { } Ok(Object::Blob(b.into_boxed_slice())) } - 3 => { + 5 => { let bytes = >::read_from(reader)?; let mut bytes_reader = &bytes[0..]; let number = ::read_from(&mut bytes_reader)?; @@ -134,6 +166,9 @@ impl Decode for Object { let obj = de_dyn_obj(&t, &mut bytes_reader)?; Ok(Object::DynOwned(obj)) } + 6 => { + Ok(Object::None) + } _ => Err(io::Error::new(io::ErrorKind::Other, "not supported")), } } diff --git a/research/dyn_type/tests/object.rs b/research/dyn_type/tests/object.rs index 5baa331d5aca..e31d93ce1965 100644 --- a/research/dyn_type/tests/object.rs +++ b/research/dyn_type/tests/object.rs @@ -15,14 +15,8 @@ #[cfg(test)] mod tests { - extern crate itertools; - - use self::itertools::Itertools; - use dyn_type::{object, Object, Primitives}; + use dyn_type::{object, BorrowObject, Object, Primitives}; use std::cmp::Ordering; - use std::collections::HashMap; - use std::fmt::Debug; - use std::hash::Hash; #[test] fn test_as_primitive() { @@ -56,10 +50,76 @@ mod tests { assert!(!obj.as_bool().unwrap()); } - fn is_map_eq( - map1: &HashMap, map2: &HashMap, - ) -> bool { - map1.iter().sorted().eq(map2.iter().sorted()) + #[test] + fn test_object_contains() { + // vector of numbers + let object_vec: Object = vec![1, 2, 3].into(); + assert!(object_vec.contains(&1.into())); + assert!(!object_vec.contains(&4.into())); + assert!(object_vec.contains(&vec![1, 3].into())); + assert!(!object_vec.contains(&vec![1, 5].into())); + assert!(!object_vec.contains(&vec![1, 2, 3, 4].into())); + // An i32 array can contain a `u64` value + assert!(object_vec.contains(&1_u64.into())); + + // vector of floats + let object_vec: Object = vec![1.0, 2.0, 3.0].into(); + assert!(object_vec.contains(&1.0.into())); + assert!(!object_vec.contains(&4.0.into())); + assert!(object_vec.contains(&vec![1.0, 3.0].into())); + assert!(!object_vec.contains(&vec![1.0, 5.0].into())); + assert!(!object_vec.contains(&vec![1.0, 2.0, 3.0, 4.0].into())); + // An float-number array can contain a `u64` value + assert!(object_vec.contains(&1_u64.into())); + let object_vec: Object = vec![1.1, 2.0, 3.0].into(); + // An float-number array can contain a `u64` value only their values are equal + assert!(!object_vec.contains(&1_u64.into())); + + // vector of strings + let object_vec: Object = vec!["a".to_string(), "b".to_string(), "c".to_string()].into(); + assert!(object_vec.contains(&"a".into())); + assert!(!object_vec.contains(&"d".into())); + assert!(object_vec.contains(&vec!["a".to_string()].into())); + assert!(!object_vec.contains(&vec!["a".to_string(), "d".to_string()].into())); + + // string object + let object_str: Object = "abcde".to_string().into(); + assert!(object_str.contains(&"a".to_string().into())); + assert!(object_str.contains(&"abc".to_string().into())); + assert!(!object_str.contains(&"ac".to_string().into())); + + // borrow object + let object_vec_b: BorrowObject = object_vec.as_borrow(); + assert!(object_vec_b.contains(&object!("a".to_string()).as_borrow())); + assert!(!object_vec_b.contains(&object!("d".to_string()).as_borrow())); + assert!(object_vec_b.contains(&object!(vec!["a".to_string()]).as_borrow())); + assert!(!object_vec_b.contains(&object!(vec!["a".to_string(), "d".to_string()]).as_borrow())); + } + + #[test] + fn test_object_compare() { + // vector + let object_vec1 = object!(vec![1, 2, 3]); + let object_vec2 = object!(vec![1, 2, 3]); + let object_vec3 = object!(vec![3, 2]); + + assert_eq!(object_vec1, object_vec2); + assert_ne!(object_vec1, object_vec3); + assert!(object_vec1 < object_vec3); + assert!(object_vec3 > object_vec2); + + // kv + let object_kv1 = object!(vec![("a".to_string(), 1_u64), ("b".to_string(), 2_u64)]); + let object_kv2 = object!(vec![("a".to_string(), 1_u64), ("b".to_string(), 2_u64)]); + let object_kv3 = object!(vec![("a".to_string(), 2_u64), ("b".to_string(), 3_u64)]); + let object_kv4 = object!(vec![("c".to_string(), 1_u64), ("d".to_string(), 2_u64)]); + + assert_eq!(object_kv1, object_kv2); + assert_ne!(object_kv1, object_kv3); + assert!(object_kv1 < object_kv3); + assert!(object_kv3 > object_kv2); + assert!(object_kv1 < object_kv4); + assert!(object_kv4 > object_kv2); } #[test] @@ -72,26 +132,6 @@ mod tests { let vec_borrow = vec_obj.as_borrow(); let vec_borrow_to_owned = vec_borrow.try_to_owned().unwrap(); assert_eq!(vec_borrow_to_owned.get::>().unwrap(), vec); - - /* - let mut map = HashMap::new(); - // Review some books. - map.insert("Adventures of Huckleberry Finn".to_string(), "My favorite book.".to_string()); - map.insert("Grimms' Fairy Tales".to_string(), "Masterpiece.".to_string()); - map.insert("Pride and Prejudice".to_string(), "Very enjoyable.".to_string()); - map.insert( - "The Adventures of Sherlock Holmes".to_string(), - "Eye lyked it alot.".to_string(), - ); - - let map_obj = Object::DynOwned(Box::new(map.clone())); - let map_recovered = map_obj.get::>().unwrap(); - assert!(is_map_eq(&map, &(*map_recovered))); - - let map_borrow = map_obj.as_borrow(); - let map_borrow_to_owned = map_borrow.try_to_owned().unwrap(); - assert!(is_map_eq(&map, &(*map_borrow_to_owned.get::>().unwrap()))); - */ } #[test] diff --git a/research/engine/pegasus/clients/java/client/pom.xml b/research/engine/pegasus/clients/java/client/pom.xml new file mode 100644 index 000000000000..bd9c211c7b76 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + + com.alibaba.pegasus + pegasus-client + 1.0-SNAPSHOT + + + 3.18.0 + 1.42.1 + 1.42.1 + + + + + javax.annotation + javax.annotation-api + 1.2 + + + io.grpc + grpc-netty-shaded + ${protoc.grpc.version} + + + io.grpc + grpc-protobuf + ${protoc.grpc.version} + + + io.grpc + grpc-stub + ${protoc.grpc.version} + + + org.slf4j + slf4j-api + 1.7.30 + + + org.slf4j + slf4j-simple + 1.7.30 + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + + + + + kr.motd.maven + os-maven-plugin + 1.6.2 + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.0.0 + + + + src/main/generated + false + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc.grpc.version}:exe:${os.detected.classifier} + ${project.basedir}/../../../server-v0/proto/ + src/main/generated/ + false + + + + + compile + compile-custom + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.4 + + + generate-sources + + add-source + + + + src/main/generated/ + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + + + diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/ClientExample.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/ClientExample.java new file mode 100644 index 000000000000..4319e642f47f --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/ClientExample.java @@ -0,0 +1,159 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus; + +import com.alibaba.pegasus.builder.JobBuilder; +import com.alibaba.pegasus.intf.CloseableIterator; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobConfig; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobRequest; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobResponse; +import com.google.protobuf.ByteString; + +import io.grpc.Status; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ClientExample { + private static final Logger logger = LoggerFactory.getLogger(ClientExample.class); + + private static void process(JobResponse response) { + ByteString data = response.getData(); + ArrayList res = toLongArray(data.toByteArray(), data.size()); + logger.info( + "got one response: job id {}, array size {}, job data {}", + response.getJobId(), + res.size(), + res.toString()); + } + + private static void finish() { + logger.info("finish process"); + } + + private static void error(Status status) { + logger.error("on error {}", status.toString()); + } + + private static ArrayList toLongArray(byte[] bytes, int size) { + ArrayList res = new ArrayList(); + for (int i = 0; i < size; i = i + 8) { + long l = fromByteArray(Arrays.copyOfRange(bytes, i, i + 8)); + res.add(l); + } + return res; + } + + private static byte[] toByteArray(long value) { + byte[] result = new byte[8]; + + for (int i = 0; i < 8; ++i) { + result[i] = (byte) ((int) (value & 255L)); + value >>= 8; + } + + return result; + } + + private static long fromByteArray(byte[] bytes) { + return ((long) bytes[7] & 255L) << 56 + | ((long) bytes[6] & 255L) << 48 + | ((long) bytes[5] & 255L) << 40 + | ((long) bytes[4] & 255L) << 32 + | ((long) bytes[3] & 255L) << 24 + | ((long) bytes[2] & 255L) << 16 + | ((long) bytes[1] & 255L) << 8 + | (long) bytes[0] & 255L; + } + + private static ByteString getSeed(long a) { + return ByteString.copyFrom(toByteArray(a)); + } + + private static ByteString getRoute() { + return ByteString.EMPTY; + } + + private static ByteString add(long a) { + return ByteString.copyFrom(toByteArray(a)); + } + + private static ByteString copy(long a) { + return ByteString.copyFrom(toByteArray(a)); + } + + private static ByteString getSink() { + return ByteString.EMPTY; + } + + public static void main(String[] args) throws Exception { + + RpcChannel rpcChannel0 = new RpcChannel("localhost", 1234); + RpcChannel rpcChannel1 = new RpcChannel("localhost", 1235); + List channels = new ArrayList<>(); + channels.add(rpcChannel0); + channels.add(rpcChannel1); + RpcClient rpcClient = new RpcClient(channels); + + logger.info("Will try to send request"); + JobConfig confPb = + JobConfig.newBuilder() + .setJobId(2) + .setJobName("ping_pong_example") + .setWorkers(2) + .addServers(0) + .addServers(1) + .build(); + // for job build + JobBuilder jobBuilder = new JobBuilder(confPb); + // for nested task + JobBuilder start = new JobBuilder(); + + // construct job + jobBuilder + .addSource(getSeed(0)) + .repeat(3, start.exchange(getRoute()).map(add(1)).flatMap(copy(8))) + .sink(getSink()); + + JobRequest req = jobBuilder.build(); + CloseableIterator iterator = rpcClient.submit(req); + // process response + try { + while (iterator.hasNext()) { + JobResponse response = iterator.next(); + process(response); + } + } catch (Exception e) { + if (iterator != null) { + try { + iterator.close(); + } catch (IOException ioe) { + // Ignore + } + } + error(Status.fromThrowable(e)); + throw e; + } + finish(); + + rpcClient.shutdown(); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/RpcChannel.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/RpcChannel.java new file mode 100644 index 000000000000..4b8ccafdc32e --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/RpcChannel.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +public class RpcChannel { + private static final Logger logger = LoggerFactory.getLogger(RpcChannel.class); + + private final ManagedChannel channel; + + public RpcChannel(ManagedChannel channel) { + this.channel = channel; + } + + public RpcChannel(String host, int port) { + this.channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); + } + + public ManagedChannel getChannel() { + return channel; + } + + public void shutdown() throws InterruptedException { + this.channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/RpcClient.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/RpcClient.java new file mode 100644 index 000000000000..bc99fc9f3e27 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/RpcClient.java @@ -0,0 +1,117 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus; + +import com.alibaba.pegasus.common.StreamIterator; +import com.alibaba.pegasus.intf.CloseableIterator; +import com.alibaba.pegasus.service.protocol.JobServiceGrpc; +import com.alibaba.pegasus.service.protocol.JobServiceGrpc.JobServiceStub; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobRequest; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobResponse; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +public class RpcClient { + private static final Logger logger = LoggerFactory.getLogger(RpcClient.class); + + private List channels; + + public RpcClient(List channels) { + this.channels = channels; + } + + public CloseableIterator submit(JobRequest jobRequest) + throws InterruptedException { + StreamIterator responseIterator = new StreamIterator<>(); + AtomicInteger counter = new AtomicInteger(this.channels.size()); + AtomicBoolean finished = new AtomicBoolean(false); + for (RpcChannel rpcChannel : channels) { + JobServiceStub asyncStub = JobServiceGrpc.newStub(rpcChannel.getChannel()); + // todo: make timeout configurable + asyncStub + .withDeadlineAfter(600000, TimeUnit.MILLISECONDS) + .submit( + jobRequest, + new JobResponseObserver(responseIterator, finished, counter)); + } + return responseIterator; + } + + public void shutdown() throws InterruptedException { + for (RpcChannel rpcChannel : channels) { + rpcChannel.shutdown(); + } + } + + private static class JobResponseObserver implements StreamObserver { + private final StreamIterator iterator; + private final AtomicBoolean finished; + private final AtomicInteger counter; + + public JobResponseObserver( + StreamIterator iterator, + AtomicBoolean finished, + AtomicInteger counter) { + this.iterator = iterator; + this.finished = finished; + this.counter = counter; + } + + @Override + public void onNext(JobResponse jobResponse) { + if (finished.get()) { + return; + } + try { + this.iterator.putData(jobResponse); + } catch (InterruptedException e) { + onError(e); + } + } + + @Override + public void onError(Throwable throwable) { + if (finished.getAndSet(true)) { + return; + } + Status status = Status.fromThrowable(throwable); + logger.error("get job response error: {}", status); + this.iterator.fail(throwable); + } + + @Override + public void onCompleted() { + logger.info("finish get job response from one server"); + if (counter.decrementAndGet() == 0) { + logger.info("finish get job response from all servers"); + try { + this.iterator.finish(); + } catch (InterruptedException e) { + onError(e); + } + } + } + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/AbstractBuilder.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/AbstractBuilder.java new file mode 100644 index 000000000000..0f170fe00b29 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/AbstractBuilder.java @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.builder; + +import com.alibaba.pegasus.service.protocol.PegasusClient.JobConfig; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobRequest; +import com.alibaba.pegasus.service.protocol.PegasusClient.Sink; +import com.alibaba.pegasus.service.protocol.PegasusClient.Source; +import com.alibaba.pegasus.service.protocol.PegasusClient.TaskPlan; +import com.google.protobuf.ByteString; + +public abstract class AbstractBuilder { + protected JobConfig conf; + protected ByteString source; + protected Plan plan; + protected Sink sink; + + public AbstractBuilder(JobConfig conf, ByteString source, Plan plan, Sink sink) { + this.conf = conf; + this.source = source; + this.plan = plan; + this.sink = sink; + } + + public AbstractBuilder(JobConfig conf, ByteString source, Plan plan) { + this.conf = conf; + this.source = source; + this.plan = plan; + this.sink = Sink.newBuilder().build(); + } + + public AbstractBuilder(JobConfig conf, ByteString source) { + this.conf = conf; + this.source = source; + this.plan = new Plan(); + this.sink = Sink.newBuilder().build(); + } + + public AbstractBuilder(JobConfig conf) { + this.conf = conf; + this.plan = new Plan(); + this.sink = Sink.newBuilder().build(); + } + + public AbstractBuilder() { + this.plan = new Plan(); + this.sink = Sink.newBuilder().build(); + } + + public JobConfig getConf() { + return conf; + } + + public void setConf(JobConfig conf) { + this.conf = conf; + } + + public ByteString getSource() { + return source; + } + + public Plan getPlan() { + return plan; + } + + public void setPlan(Plan plan) { + this.plan = plan; + } + + public JobRequest build() { + Sink sink = this.sink; + if (this.plan.endReduce()) { + sink = this.plan.genSink(); + } + return JobRequest.newBuilder() + .setConf(this.conf) + .setSource(Source.newBuilder().setResource(this.source).build()) + .setPlan(TaskPlan.newBuilder().addAllPlan(this.plan.getPlan())) + .setSink(sink) + .build(); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/JobBuilder.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/JobBuilder.java new file mode 100644 index 000000000000..11ac545adce4 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/JobBuilder.java @@ -0,0 +1,208 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.builder; + +import com.alibaba.pegasus.intf.NestedFunc; +import com.alibaba.pegasus.service.protocol.PegasusClient.AccumKind; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobConfig; +import com.alibaba.pegasus.service.protocol.PegasusClient.JobRequest; +import com.alibaba.pegasus.service.protocol.PegasusClient.Sink; +import com.google.protobuf.ByteString; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class JobBuilder extends AbstractBuilder { + private static final Logger logger = LoggerFactory.getLogger(JobBuilder.class); + + public JobBuilder(JobConfig conf, ByteString source, Plan plan, Sink sink) { + super(conf, source, plan, sink); + } + + public JobBuilder(JobConfig conf, ByteString source, Plan plan) { + super(conf, source, plan); + } + + public JobBuilder(JobConfig conf, ByteString source) { + super(conf, source); + } + + public JobBuilder(JobConfig conf) { + super(conf); + } + + public JobBuilder() { + super(); + } + + public JobBuilder addSource(ByteString source) { + this.source = source; + return this; + } + + public JobBuilder exchange(ByteString route) { + this.plan.exchange(route); + return this; + } + + public JobBuilder broadcast() { + this.plan.broadcast(); + return this; + } + + public JobBuilder broadcastBy(ByteString route) { + this.plan.broadcastBy(route); + return this; + } + + public JobBuilder aggregate(int target) { + this.plan.aggregate(target); + return this; + } + + public JobBuilder map(ByteString func) { + this.plan.map(func); + return this; + } + + public JobBuilder flatMap(ByteString func) { + this.plan.flatMap(func); + return this; + } + + public JobBuilder filter(ByteString func) { + this.plan.filter(func); + return this; + } + + public JobBuilder limit(int n) { + this.plan.limit(n); + return this; + } + + public JobBuilder dedup() { + this.plan.dedup(); + return this; + } + + public JobBuilder repeat(int times, JobBuilder subPlan) { + this.plan.repeat(times, subPlan.plan); + return this; + } + + public JobBuilder repeat(int times, NestedFunc func) { + this.plan.repeat(times, func); + return this; + } + + public JobBuilder repeatUntil(int times, ByteString until, JobBuilder subPlan) { + this.plan.repeateUntil(times, until, subPlan.plan); + return this; + } + + public JobBuilder repeatUntil(int times, ByteString until, NestedFunc func) { + this.plan.repeateUntil(times, until, func); + return this; + } + + public JobBuilder fork(JobBuilder subPlan) { + this.plan.fork(subPlan.getPlan()); + return this; + } + + public JobBuilder fork(NestedFunc func) { + this.plan.fork(func); + return this; + } + + public JobBuilder forkJoin(ByteString joiner, JobBuilder subPlan) { + this.plan.forkJoin(joiner, subPlan.getPlan()); + return this; + } + + public JobBuilder forkJoin(ByteString joiner, NestedFunc func) { + this.plan.forkJoin(joiner, func); + return this; + } + + public JobBuilder union(List subPlans) { + List plans = new ArrayList<>(); + subPlans.forEach(builder -> plans.add(builder.getPlan())); + this.plan.union(plans); + return this; + } + + public JobBuilder sortBy(ByteString cmp) { + this.plan.sortBy(cmp); + return this; + } + + public JobBuilder topBy(int n, ByteString cmp) { + this.plan.topBy(n, cmp); + return this; + } + + // reduce api + public ReduceBuilder count() { + this.plan.count(); + return new ReduceBuilder(this.conf, this.source, this.plan, this.sink); + } + + // reduce api + public ReduceBuilder fold(AccumKind accumKind) { + this.plan.fold(accumKind); + return new ReduceBuilder(this.conf, this.source, this.plan, this.sink); + } + + // reduce api + public ReduceBuilder foldCustom(AccumKind accumKind, ByteString accumFunc) { + this.plan.foldCustom(accumKind, accumFunc); + return new ReduceBuilder(this.conf, this.source, this.plan, this.sink); + } + + // reduce api + public ReduceBuilder groupBy(AccumKind accumKind, ByteString keySelector) { + this.plan.groupBy(accumKind, keySelector); + return new ReduceBuilder(this.conf, this.source, this.plan, this.sink); + } + + public void sink(ByteString output) { + this.sink = this.plan.sink(output); + } + + public static void main(String[] args) { + JobConfig confPb = + JobConfig.newBuilder().setJobId(1).setJobName("test").setWorkers(1).build(); + ByteString opBody = ByteString.EMPTY; // should be converted from pb to bytes + JobBuilder jobBuilder = new JobBuilder(confPb, opBody); + // for nested task + JobBuilder plan = new JobBuilder(); + // build JobReq + JobRequest jobReq = + jobBuilder + .map(opBody) + .flatMap(opBody) + .repeat(3, plan.flatMap(opBody).flatMap(opBody)) + .count() + .unfold(opBody) + .count() + .build(); + logger.info("send job req: {}", jobReq.toString()); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/Plan.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/Plan.java new file mode 100644 index 000000000000..1d934969f93b --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/Plan.java @@ -0,0 +1,300 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.builder; + +import com.alibaba.pegasus.intf.NestedFunc; +import com.alibaba.pegasus.service.protocol.PegasusClient.AccumKind; +import com.alibaba.pegasus.service.protocol.PegasusClient.Aggregate; +import com.alibaba.pegasus.service.protocol.PegasusClient.Apply; +import com.alibaba.pegasus.service.protocol.PegasusClient.Broadcast; +import com.alibaba.pegasus.service.protocol.PegasusClient.Communicate; +import com.alibaba.pegasus.service.protocol.PegasusClient.Dedup; +import com.alibaba.pegasus.service.protocol.PegasusClient.Filter; +import com.alibaba.pegasus.service.protocol.PegasusClient.FlatMap; +import com.alibaba.pegasus.service.protocol.PegasusClient.Fold; +import com.alibaba.pegasus.service.protocol.PegasusClient.GroupBy; +import com.alibaba.pegasus.service.protocol.PegasusClient.Iteration; +import com.alibaba.pegasus.service.protocol.PegasusClient.LeftJoin; +import com.alibaba.pegasus.service.protocol.PegasusClient.Limit; +import com.alibaba.pegasus.service.protocol.PegasusClient.Map; +import com.alibaba.pegasus.service.protocol.PegasusClient.Merge; +import com.alibaba.pegasus.service.protocol.PegasusClient.OperatorDef; +import com.alibaba.pegasus.service.protocol.PegasusClient.OperatorDef.Builder; +import com.alibaba.pegasus.service.protocol.PegasusClient.OperatorDef.OpKindCase; +import com.alibaba.pegasus.service.protocol.PegasusClient.Repartition; +import com.alibaba.pegasus.service.protocol.PegasusClient.Sink; +import com.alibaba.pegasus.service.protocol.PegasusClient.SortBy; +import com.alibaba.pegasus.service.protocol.PegasusClient.TaskPlan; +import com.google.protobuf.ByteString; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class Plan { + private static final Logger logger = LoggerFactory.getLogger(Plan.class); + + private List plan; + + public Plan(ArrayList plan) { + this.plan = plan; + } + + public Plan() { + this.plan = new ArrayList(); + } + + public List getPlan() { + return plan; + } + + public void setPlan(List plan) { + this.plan = plan; + } + + public void exchange(ByteString route) { + Repartition exchange = Repartition.newBuilder().setResource(route).build(); + Communicate communicate = Communicate.newBuilder().setToAnother(exchange).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setComm(communicate).build(); + this.plan.add(operatorDef); + } + + public void broadcast() { + Broadcast broadcast = Broadcast.newBuilder().build(); + Communicate communicate = Communicate.newBuilder().setToOthers(broadcast).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setComm(communicate).build(); + this.plan.add(operatorDef); + } + + public void broadcastBy(ByteString route) { + Broadcast broadcast = Broadcast.newBuilder().setResource(route).build(); + Communicate communicate = Communicate.newBuilder().setToOthers(broadcast).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setComm(communicate).build(); + this.plan.add(operatorDef); + } + + public void aggregate(int target) { + Aggregate aggregate = Aggregate.newBuilder().setTarget(target).build(); + Communicate communicate = Communicate.newBuilder().setToOne(aggregate).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setComm(communicate).build(); + this.plan.add(operatorDef); + } + + public void map(ByteString func) { + Map map = Map.newBuilder().setResource(func).build(); + Builder builder = OperatorDef.newBuilder().setMap(map); + OperatorDef operatorDef = builder.build(); + this.plan.add(operatorDef); + } + + public void flatMap(ByteString func) { + FlatMap flatMap = FlatMap.newBuilder().setResource(func).build(); + Builder builder = OperatorDef.newBuilder().setFlatMap(flatMap); + OperatorDef operatorDef = builder.build(); + this.plan.add(operatorDef); + } + + public void filter(ByteString func) { + Filter filter = Filter.newBuilder().setResource(func).build(); + Builder builder = OperatorDef.newBuilder().setFilter(filter); + OperatorDef operatorDef = builder.build(); + this.plan.add(operatorDef); + } + + public void limit(int n) { + Limit limitInfo = Limit.newBuilder().setLimit(n).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setLimit(limitInfo).build(); + this.plan.add(operatorDef); + } + + public void count() { + Fold fold = Fold.newBuilder().setAccum(AccumKind.CNT).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setFold(fold).build(); + this.plan.add(operatorDef); + } + + public void fold(AccumKind accumKind) { + Fold fold = Fold.newBuilder().setAccum(accumKind).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setFold(fold).build(); + this.plan.add(operatorDef); + } + + public void foldCustom(AccumKind accumKind, ByteString accumFunc) { + Fold fold = Fold.newBuilder().setAccum(accumKind).setResource(accumFunc).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setFold(fold).build(); + this.plan.add(operatorDef); + } + + public void dedup() { + Dedup dedup = Dedup.newBuilder().build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setDedup(dedup).build(); + this.plan.add(operatorDef); + } + + public void repeat(int times, Plan subPlan) { + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(subPlan.getPlan()).build(); + Iteration iteration = Iteration.newBuilder().setMaxIters(times).setBody(taskPlan).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setIterate(iteration).build(); + this.plan.add(operatorDef); + } + + public void repeat(int times, NestedFunc func) { + Plan repeatedPlan = new Plan(); + func.nestedFunc(repeatedPlan); + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(repeatedPlan.getPlan()).build(); + Iteration iteration = Iteration.newBuilder().setMaxIters(times).setBody(taskPlan).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setIterate(iteration).build(); + this.plan.add(operatorDef); + } + + public void repeateUntil(int times, ByteString until, Plan subPlan) { + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(subPlan.getPlan()).build(); + Filter filterUntil = Filter.newBuilder().setResource(until).build(); + Iteration iteration = + Iteration.newBuilder() + .setMaxIters(times) + .setBody(taskPlan) + .setUntil(filterUntil) + .build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setIterate(iteration).build(); + this.plan.add(operatorDef); + } + + public void repeateUntil(int times, ByteString until, NestedFunc func) { + Plan repeatedPlan = new Plan(); + func.nestedFunc(repeatedPlan); + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(repeatedPlan.getPlan()).build(); + Filter filterUntil = Filter.newBuilder().setResource(until).build(); + Iteration iteration = + Iteration.newBuilder() + .setMaxIters(times) + .setBody(taskPlan) + .setUntil(filterUntil) + .build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setIterate(iteration).build(); + this.plan.add(operatorDef); + } + + public void fork(Plan subPlan) { + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(subPlan.getPlan()).build(); + Apply subtask = Apply.newBuilder().setTask(taskPlan).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setApply(subtask).build(); + this.plan.add(operatorDef); + } + + public void fork(NestedFunc func) { + Plan subPlan = new Plan(); + func.nestedFunc(subPlan); + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(subPlan.getPlan()).build(); + Apply subtask = Apply.newBuilder().setTask(taskPlan).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setApply(subtask).build(); + this.plan.add(operatorDef); + } + + public void forkJoin(ByteString joiner, Plan subPlan) { + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(subPlan.getPlan()).build(); + LeftJoin leftJoin = LeftJoin.newBuilder().setResource(joiner).build(); + Apply subtask = Apply.newBuilder().setTask(taskPlan).setJoin(leftJoin).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setApply(subtask).build(); + this.plan.add(operatorDef); + } + + public void forkJoin(ByteString joiner, NestedFunc func) { + Plan subPlan = new Plan(); + func.nestedFunc(subPlan); + TaskPlan taskPlan = TaskPlan.newBuilder().addAllPlan(subPlan.getPlan()).build(); + LeftJoin leftJoin = LeftJoin.newBuilder().setResource(joiner).build(); + Apply subtask = Apply.newBuilder().setTask(taskPlan).setJoin(leftJoin).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setApply(subtask).build(); + this.plan.add(operatorDef); + } + + public void union(List subPlans) { + List unionTasks = new ArrayList<>(); + subPlans.forEach( + plan -> unionTasks.add(TaskPlan.newBuilder().addAllPlan(plan.getPlan()).build())); + Merge union = Merge.newBuilder().addAllTasks(unionTasks).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setMerge(union).build(); + this.plan.add(operatorDef); + } + + public void sortBy(ByteString cmp) { + int noLimit = -1; + SortBy orderBy = SortBy.newBuilder().setLimit(noLimit).setCompare(cmp).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setSort(orderBy).build(); + this.plan.add(operatorDef); + } + + public void topBy(int n, ByteString cmp) { + SortBy orderBy = SortBy.newBuilder().setLimit(n).setCompare(cmp).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setSort(orderBy).build(); + this.plan.add(operatorDef); + } + + public void groupBy(AccumKind accumKind, ByteString keySelector) { + GroupBy groupBy = GroupBy.newBuilder().setAccum(accumKind).setResource(keySelector).build(); + OperatorDef operatorDef = OperatorDef.newBuilder().setGroup(groupBy).build(); + this.plan.add(operatorDef); + } + + public Sink sink(ByteString output) { + return Sink.newBuilder().setResource(output).build(); + } + + public void chainUnfold(ByteString func) { + if (plan.size() > 0) { + OperatorDef pre = plan.remove(plan.size() - 1); + OperatorDef.Builder builder = pre.toBuilder(); + if (pre.getOpKindCase() == OpKindCase.GROUP) { + builder.getGroupBuilder().getUnfoldBuilder().setResource(func); + } else if (pre.getOpKindCase() == OpKindCase.FOLD) { + builder.getFoldBuilder().getUnfoldBuilder().setResource(func); + } + OperatorDef reduceChain = builder.build(); + plan.add(reduceChain); + } + } + + public boolean endReduce() { + int len = plan.size(); + if (len == 0) { + return false; + } + OpKindCase opKind = plan.get(len - 1).getOpKindCase(); + return opKind == OpKindCase.GROUP || opKind == OpKindCase.FOLD; + } + + public Sink genSink() { + Sink.Builder sinkBuilder = Sink.newBuilder(); + if (endReduce()) { + OperatorDef pre = plan.remove(plan.size() - 1); + if (pre.getOpKindCase() == OpKindCase.GROUP) { + sinkBuilder.setGroup(pre.getGroup()); + } else if (pre.getOpKindCase() == OpKindCase.FOLD) { + sinkBuilder.setFold(pre.getFold()); + } + } + return sinkBuilder.build(); + } + + private ByteString toBytes(boolean raw) { + byte[] bytes = new byte[1]; + bytes[0] = (byte) (raw ? 1 : 0); + return ByteString.copyFrom(bytes); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/ReduceBuilder.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/ReduceBuilder.java new file mode 100644 index 000000000000..65e9b22402ce --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/builder/ReduceBuilder.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.builder; + +import com.alibaba.pegasus.service.protocol.PegasusClient.JobConfig; +import com.alibaba.pegasus.service.protocol.PegasusClient.Sink; +import com.google.protobuf.ByteString; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReduceBuilder extends AbstractBuilder { + private static final Logger logger = LoggerFactory.getLogger(ReduceBuilder.class); + + public ReduceBuilder(JobConfig conf, ByteString source, Plan plan, Sink sink) { + super(conf, source, plan, sink); + } + + public JobBuilder unfold(ByteString func) { + this.plan.chainUnfold(func); + return new JobBuilder(this.conf, this.source, this.plan, this.sink); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/common/StreamIterator.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/common/StreamIterator.java new file mode 100644 index 000000000000..6fea23c22a52 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/common/StreamIterator.java @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.common; + +import com.alibaba.pegasus.intf.CloseableIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.NoSuchElementException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +public class StreamIterator implements CloseableIterator { + private static final Logger logger = LoggerFactory.getLogger(StreamIterator.class); + + private static final Object PILL = new Object(); + private BlockingQueue buffer; + private Object head; + private volatile boolean closed = false; + private AtomicReference exception = new AtomicReference<>(); + + public StreamIterator() { + this.buffer = new LinkedBlockingQueue<>(); + + this.head = null; + } + + @Override + public boolean hasNext() { + Throwable t = this.exception.get(); + if (t != null) { + throw new RuntimeException(t); + } + if (head == null) { + try { + head = buffer.take(); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } + if (head == PILL) { + return false; + } + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + if (head instanceof Throwable) { + throw new RuntimeException((Throwable) head); + } + T res = (T) head; + head = null; + return res; + } + + // From common + public void putData(T data) throws InterruptedException { + if (closed) { + return; + } + this.buffer.put(data); + } + + // From common + public void fail(Throwable t) { + if (closed) { + return; + } + boolean suc = this.exception.compareAndSet(null, t); + if (!suc) { + return; + } + logger.error("iterator failed", t); + buffer.offer(t); + } + + // From common + public void finish() throws InterruptedException { + if (closed) { + return; + } + buffer.put(PILL); + } + + public int size() { + return this.buffer.size(); + } + + // From iterator consumer + @Override + public void close() { + this.closed = true; + this.buffer.clear(); + } +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/CloseableIterator.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/CloseableIterator.java new file mode 100644 index 000000000000..1fe17328bd66 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/CloseableIterator.java @@ -0,0 +1,21 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.intf; + +import java.io.Closeable; +import java.util.Iterator; + +public interface CloseableIterator extends Iterator, Closeable {} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/NestedFunc.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/NestedFunc.java new file mode 100644 index 000000000000..f71ac0e43116 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/NestedFunc.java @@ -0,0 +1,22 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.intf; + +import com.alibaba.pegasus.builder.Plan; + +public interface NestedFunc { + void nestedFunc(Plan plan); +} diff --git a/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/ResultProcessor.java b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/ResultProcessor.java new file mode 100644 index 000000000000..79ec088a5a39 --- /dev/null +++ b/research/engine/pegasus/clients/java/client/src/main/java/com/alibaba/pegasus/intf/ResultProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.pegasus.intf; + +import com.alibaba.pegasus.service.protocol.PegasusClient.JobResponse; + +import io.grpc.Status; + +public interface ResultProcessor { + void process(JobResponse response); + + void finish(); + + void error(Status status); +} diff --git a/research/engine/pegasus/clients/rust/client/Cargo.toml b/research/engine/pegasus/clients/rust/client/Cargo.toml new file mode 100644 index 000000000000..5d298e3d30e5 --- /dev/null +++ b/research/engine/pegasus/clients/rust/client/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "pegasus_client" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +pegasus_server = { path = "../../../server-v0" } +pegasus = { path = "../../../pegasus" } +tonic = "0.6" +prost = "0.9" +tokio = { version = "1.0", features = ["macros", "sync", "rt-multi-thread"] } +tokio-stream = { version = "0.1.3", features = ["net"] } \ No newline at end of file diff --git a/research/engine/pegasus/clients/rust/client/src/builder.rs b/research/engine/pegasus/clients/rust/client/src/builder.rs new file mode 100644 index 000000000000..0acfd02c6bb2 --- /dev/null +++ b/research/engine/pegasus/clients/rust/client/src/builder.rs @@ -0,0 +1,623 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use pegasus::{BuildJobError, JobConf}; +use pegasus_server::pb; +use std::fmt; +use std::ops::{Deref, DerefMut}; + +#[derive(Clone, Debug, PartialEq)] +pub struct Plan { + plan: Vec, +} + +impl Default for Plan { + fn default() -> Self { + Plan { plan: vec![] } + } +} + +impl Deref for Plan { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.plan + } +} + +impl DerefMut for Plan { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.plan + } +} + +pub type BinaryResource = Vec; + +impl Plan { + pub fn repartition(&mut self, route: BinaryResource) -> &mut Self { + let repartition = pb::Repartition { resource: route }; + let comm = pb::Communicate { ch_kind: Some(pb::communicate::ChKind::ToAnother(repartition)) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Comm(comm)) }; + self.plan.push(op); + self + } + + pub fn broadcast(&mut self) -> &mut Self { + let broadcast = pb::Broadcast { resource: vec![] }; + let comm = pb::Communicate { ch_kind: Some(pb::communicate::ChKind::ToOthers(broadcast)) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Comm(comm)) }; + self.plan.push(op); + self + } + + pub fn aggregate(&mut self, target: u32) -> &mut Self { + let aggregate = pb::Aggregate { target }; + let comm = pb::Communicate { ch_kind: Some(pb::communicate::ChKind::ToOne(aggregate)) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Comm(comm)) }; + self.plan.push(op); + self + } + + pub fn map(&mut self, func: BinaryResource) -> &mut Self { + let map = pb::Map { resource: func }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Map(map)) }; + self.plan.push(op); + self + } + + pub fn flat_map(&mut self, func: BinaryResource) -> &mut Self { + let flat_map = pb::FlatMap { resource: func }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::FlatMap(flat_map)) }; + self.plan.push(op); + self + } + + pub fn filter(&mut self, func: BinaryResource) -> &mut Self { + let filter = pb::Filter { resource: func }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Filter(filter)) }; + self.plan.push(op); + self + } + + pub fn limit(&mut self, size: u32) -> &mut Self { + let limit = pb::Limit { limit: size }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Limit(limit)) }; + self.plan.push(op); + self + } + + pub fn count(&mut self) -> &mut Self { + let fold = pb::Fold { accum: pb::AccumKind::Cnt as i32, resource: vec![], unfold: None }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Fold(fold)) }; + self.plan.push(op); + self + } + + pub fn fold(&mut self, accum_kind: pb::AccumKind) -> &mut Self { + let fold = pb::Fold { accum: accum_kind as i32, resource: vec![], unfold: None }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Fold(fold)) }; + self.plan.push(op); + self + } + + pub fn fold_custom(&mut self, accum_kind: pb::AccumKind, func: BinaryResource) -> &mut Self { + let fold = pb::Fold { accum: accum_kind as i32, resource: func, unfold: None }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Fold(fold)) }; + self.plan.push(op); + self + } + + pub fn dedup(&mut self, res: BinaryResource) -> &mut Self { + let dedup = pb::Dedup { resource: res }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Dedup(dedup)) }; + self.plan.push(op); + self + } + + pub fn iterate(&mut self, times: u32, mut func: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + func(&mut sub_plan); + let iteration = pb::Iteration { + max_iters: times, + until: None, + body: Some(pb::TaskPlan { plan: sub_plan.take() }), + }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Iterate(iteration)) }; + self.plan.push(op); + self + } + + pub fn iterate_emit(&mut self, times: u32, mut func: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + func(&mut sub_plan); + let iteration_emit = pb::IterationEmit { + max_iters: times, + until: None, + body: Some(pb::TaskPlan { plan: sub_plan.take() }), + }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::IterateEmit(iteration_emit)) }; + self.plan.push(op); + self + } + + pub fn iterate_until(&mut self, times: u32, until: BinaryResource, mut func: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + func(&mut sub_plan); + let filter = pb::Filter { resource: until }; + let iteration = pb::Iteration { + max_iters: times, + until: Some(filter), + body: Some(pb::TaskPlan { plan: sub_plan.take() }), + }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Iterate(iteration)) }; + self.plan.push(op); + self + } + + pub fn iterate_emit_until(&mut self, times: u32, until: BinaryResource, mut func: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + func(&mut sub_plan); + let filter = pb::Filter { resource: until }; + let iteration_emit = pb::IterationEmit { + max_iters: times, + until: Some(filter), + body: Some(pb::TaskPlan { plan: sub_plan.take() }), + }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::IterateEmit(iteration_emit)) }; + self.plan.push(op); + self + } + + pub fn apply(&mut self, mut subtask: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + subtask(&mut sub_plan); + let subtask = pb::Apply { join: None, task: Some(pb::TaskPlan { plan: sub_plan.take() }) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Apply(subtask)) }; + self.plan.push(op); + self + } + + pub fn apply_join(&mut self, joiner: BinaryResource, mut subtask: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + subtask(&mut sub_plan); + let left_join = pb::LeftJoin { resource: joiner }; + let subtask = + pb::Apply { join: Some(left_join), task: Some(pb::TaskPlan { plan: sub_plan.take() }) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Apply(subtask)) }; + self.plan.push(op); + self + } + + pub fn segment_apply(&mut self, mut subtask: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + subtask(&mut sub_plan); + let subtask = pb::SegmentApply { join: None, task: Some(pb::TaskPlan { plan: sub_plan.take() }) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::SegApply(subtask)) }; + self.plan.push(op); + self + } + + pub fn segment_apply_join(&mut self, joiner: BinaryResource, mut subtask: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + let mut sub_plan = Plan::default(); + subtask(&mut sub_plan); + let left_join = pb::LeftJoin { resource: joiner }; + let subtask = + pb::SegmentApply { join: Some(left_join), task: Some(pb::TaskPlan { plan: sub_plan.take() }) }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::SegApply(subtask)) }; + self.plan.push(op); + self + } + + pub fn merge(&mut self, mut plans: Vec) -> &mut Self { + let mut tasks = vec![]; + for plan in plans.drain(..) { + tasks.push(pb::TaskPlan { plan: plan.take() }); + } + let merge = pb::Merge { tasks }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Merge(merge)) }; + self.plan.push(op); + self + } + + pub fn join_func( + &mut self, join_kind: pb::join::JoinKind, mut left_task: FL, mut right_task: FR, + res: BinaryResource, + ) -> &mut Self + where + FL: FnMut(&mut Plan), + FR: FnMut(&mut Plan), + { + let mut left_plan = Plan::default(); + left_task(&mut left_plan); + let mut right_plan = Plan::default(); + right_task(&mut right_plan); + let join = pb::Join { + kind: join_kind as i32, + resource: res, + left_task: Some(pb::TaskPlan { plan: left_plan.take() }), + right_task: Some(pb::TaskPlan { plan: right_plan.take() }), + }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Join(join)) }; + self.plan.push(op); + self + } + + pub fn join( + &mut self, join_kind: pb::join::JoinKind, left_plan: Plan, right_plan: Plan, res: BinaryResource, + ) -> &mut Self { + let join = pb::Join { + kind: join_kind as i32, + resource: res, + left_task: Some(pb::TaskPlan { plan: left_plan.take() }), + right_task: Some(pb::TaskPlan { plan: right_plan.take() }), + }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Join(join)) }; + self.plan.push(op); + self + } + + pub fn sort_by(&mut self, cmp: BinaryResource) -> &mut Self { + let sort = pb::SortBy { limit: -1, compare: cmp }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Sort(sort)) }; + self.plan.push(op); + self + } + + pub fn sort_limit_by(&mut self, n: i64, cmp: BinaryResource) -> &mut Self { + let sort = pb::SortBy { limit: n, compare: cmp }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Sort(sort)) }; + self.plan.push(op); + self + } + + pub fn group_by(&mut self, accum_kind: pb::AccumKind, key_selector: BinaryResource) -> &mut Self { + let group = pb::GroupBy { accum: accum_kind as i32, resource: key_selector, unfold: None }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::Group(group)) }; + self.plan.push(op); + self + } + + pub fn key_by(&mut self, key_selector: BinaryResource) -> &mut Self { + let key_by = pb::KeyBy { key_selector }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::KeyBy(key_by)) }; + self.plan.push(op); + self + } + + pub fn filter_map(&mut self, resource: BinaryResource) -> &mut Self { + let filter_map = pb::FilterMap { resource }; + let op = pb::OperatorDef { op_kind: Some(pb::operator_def::OpKind::FilterMap(filter_map)) }; + self.plan.push(op); + self + } + + pub fn take(self) -> Vec { + self.plan + } +} + +#[derive(Default)] +pub struct JobBuilder { + pub conf: JobConf, + source: BinaryResource, + plan: Plan, + sink: BinaryResource, +} + +impl fmt::Debug for JobBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("JobBuilder") + .field("source", &self.source) + .field("plan", &self.plan) + .field("sink", &self.sink) + .finish() + } +} + +impl PartialEq for JobBuilder { + fn eq(&self, other: &JobBuilder) -> bool { + self.source == other.source && self.plan == other.plan && self.sink == other.sink + } +} + +impl JobBuilder { + pub fn new(conf: JobConf) -> Self { + JobBuilder { conf, source: vec![], plan: Default::default(), sink: vec![] } + } + + pub fn add_source(&mut self, src: BinaryResource) -> &mut Self { + self.source = src; + self + } + + pub fn repartition(&mut self, route: BinaryResource) -> &mut Self { + self.plan.repartition(route); + self + } + + pub fn broadcast(&mut self) -> &mut Self { + self.plan.broadcast(); + self + } + + pub fn aggregate(&mut self, target: u32) -> &mut Self { + self.plan.aggregate(target); + self + } + + pub fn map(&mut self, func: BinaryResource) -> &mut Self { + self.plan.map(func); + self + } + + pub fn flat_map(&mut self, func: BinaryResource) -> &mut Self { + self.plan.flat_map(func); + self + } + + pub fn filter(&mut self, func: BinaryResource) -> &mut Self { + self.plan.filter(func); + self + } + + pub fn limit(&mut self, size: u32) -> &mut Self { + self.plan.limit(size); + self + } + + pub fn dedup(&mut self, res: BinaryResource) -> &mut Self { + self.plan.dedup(res); + self + } + + pub fn iterate(&mut self, times: u32, func: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + self.plan.iterate(times, func); + self + } + + pub fn iterate_until(&mut self, times: u32, until: BinaryResource, func: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + self.plan.iterate_until(times, until, func); + self + } + + pub fn apply(&mut self, subtask: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + self.plan.apply(subtask); + self + } + + pub fn apply_join(&mut self, subtask: F, joiner: BinaryResource) -> &mut Self + where + F: FnMut(&mut Plan), + { + self.plan.apply_join(joiner, subtask); + self + } + + pub fn segment_apply(&mut self, subtask: F) -> &mut Self + where + F: FnMut(&mut Plan), + { + self.plan.segment_apply(subtask); + self + } + + pub fn segment_apply_join(&mut self, subtask: F, joiner: BinaryResource) -> &mut Self + where + F: FnMut(&mut Plan), + { + self.plan.segment_apply_join(joiner, subtask); + self + } + + pub fn merge(&mut self, plans: Vec) -> &mut Self { + self.plan.merge(plans); + self + } + + pub fn join_func( + &mut self, join_kind: pb::join::JoinKind, left_task: FL, right_task: FR, res: BinaryResource, + ) -> &mut Self + where + FL: FnMut(&mut Plan), + FR: FnMut(&mut Plan), + { + self.plan + .join_func(join_kind, left_task, right_task, res); + self + } + + pub fn join( + &mut self, join_kind: pb::join::JoinKind, left_plan: Plan, right_plan: Plan, res: BinaryResource, + ) -> &mut Self { + self.plan + .join(join_kind, left_plan, right_plan, res); + self + } + + pub fn sort_by(&mut self, cmp: BinaryResource) -> &mut Self { + self.plan.sort_by(cmp); + self + } + + pub fn sort_limit_by(&mut self, n: i64, cmp: BinaryResource) -> &mut Self { + self.plan.sort_limit_by(n, cmp); + self + } + + pub fn count(&mut self) -> &mut Self { + self.plan.count(); + self + } + + pub fn fold(&mut self, accum_kind: pb::AccumKind) -> &mut Self { + self.plan.fold(accum_kind); + self + } + + pub fn fold_custom(&mut self, accum_kind: pb::AccumKind, func: BinaryResource) -> &mut Self { + self.plan.fold_custom(accum_kind, func); + self + } + + pub fn key_by(&mut self, key_selector: BinaryResource) -> &mut Self { + self.plan.key_by(key_selector); + self + } + + pub fn filter_map(&mut self, udf: BinaryResource) -> &mut Self { + self.plan.filter_map(udf); + self + } + + pub fn sink(&mut self, output: BinaryResource) { + self.sink = output; + } + + pub fn take_plan(self) -> Plan { + self.plan + } + + pub fn build(self) -> Result { + let conf = pb::JobConfig { + job_id: self.conf.job_id, + job_name: self.conf.job_name.clone(), + workers: self.conf.workers, + time_limit: self.conf.time_limit, + batch_size: self.conf.batch_size, + output_capacity: self.conf.batch_capacity, + memory_limit: self.conf.memory_limit, + plan_print: self.conf.plan_print, + servers: self.conf.servers().get_servers(), + }; + let source = pb::Source { resource: self.source }; + let plan = pb::TaskPlan { plan: self.plan.take() }; + let sink = pb::Sink { sinker: Some(pb::sink::Sinker::Resource(self.sink)) }; + + Ok(pb::JobRequest { conf: Some(conf), source: Some(source), plan: Some(plan), sink: Some(sink) }) + } +} + +impl Deref for JobBuilder { + type Target = Plan; + + fn deref(&self) -> &Self::Target { + &self.plan + } +} + +impl DerefMut for JobBuilder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.plan + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_job_build_00() { + let mut builder = JobBuilder::new(JobConf::new("test_build_00")); + builder + .add_source(vec![0u8; 32]) + .map(vec![1u8; 32]) + .repartition(vec![2u8; 32]) + .map(vec![3u8; 32]) + .limit(1) + .iterate(3, |start| { + start + .repartition(vec![4u8; 32]) + .map(vec![5u8; 32]); + }) + .sink(vec![6u8; 32]); + let job_req = builder.build().unwrap(); + assert_eq!(&job_req.source.unwrap().resource, &vec![0u8; 32]); + assert_eq!(&job_req.sink.unwrap().resource, &vec![6u8; 32]); + assert_eq!(&job_req.plan.unwrap().plan.len(), &5); + } + + #[test] + fn test_job_build_01() { + let mut builder = JobBuilder::new(JobConf::new("test_build_01")); + builder + .add_source(vec![0u8; 32]) + .join_func( + pb::join::JoinKind::Inner, + |src1| { + src1.map(vec![]).join_func( + pb::join::JoinKind::Inner, + |src1_1| { + src1_1.map(vec![]); + }, + |src1_2| { + src1_2.map(vec![]); + }, + vec![], + ); + }, + |src2| { + src2.map(vec![]).join_func( + pb::join::JoinKind::Inner, + |src2_1| { + src2_1.map(vec![]); + }, + |src2_2| { + src2_2.map(vec![]); + }, + vec![], + ); + }, + vec![], + ) + .sink(vec![6u8; 32]); + let job_req = builder.build().unwrap(); + assert_eq!(&job_req.source.unwrap().resource, &vec![0u8; 32]); + assert_eq!(&job_req.sink.unwrap().resource, &vec![6u8; 32]); + assert_eq!(&job_req.plan.unwrap().plan.len(), &1); + } +} diff --git a/research/engine/pegasus/clients/rust/client/src/lib.rs b/research/engine/pegasus/clients/rust/client/src/lib.rs new file mode 100644 index 000000000000..1b8929be5fb0 --- /dev/null +++ b/research/engine/pegasus/clients/rust/client/src/lib.rs @@ -0,0 +1,52 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use pegasus_server::pb; +use std::error::Error; +use tonic::transport::Channel; +use tonic::{Request, Streaming}; + +pub mod builder; + +pub struct JobRpcClient { + stub: pb::job_service_client::JobServiceClient, +} + +impl JobRpcClient { + pub async fn from_str(addr: &'static str) -> Result> { + let stub = pb::job_service_client::JobServiceClient::connect(addr).await?; + Ok(JobRpcClient { stub }) + } + + pub fn new(ch: Channel) -> Self { + let stub = pb::job_service_client::JobServiceClient::new(ch); + JobRpcClient { stub } + } + + pub async fn submit( + &mut self, job_req: pb::JobRequest, + ) -> Result, Box> { + Ok(self + .stub + .submit(Request::new(job_req)) + .await? + .into_inner()) + } +} + +#[allow(dead_code)] +pub struct JobClient { + stubs: Vec, +} diff --git a/research/engine/pegasus/server-v0/Cargo.toml b/research/engine/pegasus/server-v0/Cargo.toml new file mode 100644 index 000000000000..77cb6bb702c1 --- /dev/null +++ b/research/engine/pegasus/server-v0/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "pegasus_server" +version = "0.1.0" +authors = ["chenqiang.mcq "] +edition = "2018" + +[dependencies] +pegasus_common = { path = "../common" } +pegasus_network = { path = "../network"} +pegasus_memory = { path = "../memory"} +pegasus = { path = "../pegasus" } +log = "0.4" +crossbeam-utils = "0.6" +#crossbeam-channel = "0.3.6" +tonic = "0.6" +prost = "0.9" +tokio = { version = "1.0", features = ["macros", "sync", "rt-multi-thread"] } +tokio-stream = { version = "0.1.3", features = ["net"] } +toml = "0.5" +serde = { version = "1.0", features = ["derive"] } +structopt = "0.2" + +[dev-dependencies] +libloading = "0.7" + +[build-dependencies] +tonic-build = "0.5" + +[features] +default = [] +# set to generate code in place(generated codes are in current codebase); +gcip = [] + diff --git a/research/engine/pegasus/server-v0/build.rs b/research/engine/pegasus/server-v0/build.rs new file mode 100644 index 000000000000..d7e5fda4d878 --- /dev/null +++ b/research/engine/pegasus/server-v0/build.rs @@ -0,0 +1,41 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed=proto/job_service.proto"); + codegen_inplace() +} + +#[cfg(feature = "gcip")] +fn codegen_inplace() -> Result<(), Box> { + let dir = "src/generated"; + if std::path::Path::new(&dir).exists() { + std::fs::remove_dir_all(&dir).unwrap(); + } + std::fs::create_dir(&dir).unwrap(); + tonic_build::configure() + .build_server(true) + .out_dir("src/generated") + .compile(&["proto/job_service.proto"], &["proto"])?; + Ok(()) +} + +#[cfg(not(feature = "gcip"))] +fn codegen_inplace() -> Result<(), Box> { + tonic_build::configure() + .build_server(true) + .compile(&["proto/job_service.proto"], &["proto"])?; + Ok(()) +} diff --git a/research/engine/pegasus/server-v0/conf/pegasus_config.toml b/research/engine/pegasus/server-v0/conf/pegasus_config.toml new file mode 100644 index 000000000000..c7cbd5757373 --- /dev/null +++ b/research/engine/pegasus/server-v0/conf/pegasus_config.toml @@ -0,0 +1,3 @@ +nonblocking = false +read_timeout_ms = 8 +write_timeout_ms = 8 \ No newline at end of file diff --git a/research/engine/pegasus/server-v0/proto/job_service.proto b/research/engine/pegasus/server-v0/proto/job_service.proto new file mode 100644 index 000000000000..4738054da508 --- /dev/null +++ b/research/engine/pegasus/server-v0/proto/job_service.proto @@ -0,0 +1,213 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; +package protocol; +option java_package = "com.alibaba.pegasus.service.protocol"; +option java_outer_classname = "PegasusClient"; + +message Repartition { + bytes resource = 1; +} + +message Broadcast { + bytes resource = 1; +} + +message Aggregate { + uint32 target = 1; +} + +message Communicate { + oneof ch_kind { + Repartition to_another = 1; + Broadcast to_others = 2; + Aggregate to_one = 3; + } +} + +message Source { + bytes resource = 1; +} + +message Map { + bytes resource = 1; +} + +message FlatMap { + bytes resource = 1; +} + +message Filter { + bytes resource = 1; +} + +message Sink { + oneof sinker { + bytes resource = 1; + Fold fold = 2; + GroupBy group = 3; + } +} + +message LeftJoin { + bytes resource = 1; +} + +message Limit { + uint32 limit = 2; +} + +message Dedup { + bytes resource = 1; +} + +message SortBy { + int64 limit = 1; + bytes compare = 2; +} + +enum AccumKind { + CNT = 0; + SUM = 1; + MAX = 2; + MIN = 3; + TO_LIST = 4; + TO_SET = 5; + CUSTOM = 6; +} + +message Fold { + AccumKind accum = 1; + bytes resource = 2; + FlatMap unfold = 3; +} + +message GroupBy { + AccumKind accum = 1; + bytes resource = 2; + FlatMap unfold = 3; +} + +message Merge { + repeated TaskPlan tasks = 1; +} + +message Iteration { + uint32 max_iters = 1; + Filter until = 2; + TaskPlan body = 3; +} + +message IterationEmit { + uint32 max_iters = 1; + Filter until = 2; + TaskPlan body = 3; +} + +message Apply { + LeftJoin join = 1; + TaskPlan task = 2; +} + +message SegmentApply { + LeftJoin join = 1; + TaskPlan task = 2; +} + +message Join { + enum JoinKind { + // Inner join + INNER = 0; + // Left outer join + LEFT_OUTER = 1; + // Right outer join + RIGHT_OUTER = 2; + // Full outer join + FULL_OUTER = 3; + // Left semi-join, right alternative can be naturally adapted + SEMI = 4; + // Left anti-join, right alternative can be naturally adapted + ANTI = 5; + // aka. Cartesian product + TIMES = 6; + } + JoinKind kind = 1; + bytes resource = 2; + TaskPlan left_task = 3; + TaskPlan right_task = 4; +} + +message KeyBy { + bytes key_selector = 2; +} + +message FilterMap { + bytes resource = 1; +} + +message OperatorDef { + oneof op_kind { + Communicate comm = 1; + Map map = 3; + FlatMap flat_map = 4; + Filter filter = 5; + Limit limit = 6; + SortBy sort = 7; + Fold fold = 8; + GroupBy group = 9; + Merge merge = 10; + Iteration iterate = 11; + Apply apply = 12; + Dedup dedup = 13; + SegmentApply seg_apply = 14; + Join join = 15; + KeyBy key_by = 16; + FilterMap filter_map = 17; + IterationEmit iterate_emit = 18; + } +} + +message TaskPlan { + repeated OperatorDef plan = 1; +} + +message JobConfig { + uint64 job_id = 1; + string job_name = 2; + uint32 workers = 3; + uint64 time_limit = 4; + uint32 batch_size = 5; + uint32 output_capacity = 6; + uint32 memory_limit = 7; + bool plan_print = 8; + repeated uint64 servers = 9; +} + +message JobRequest { + JobConfig conf = 1; + Source source = 2; + TaskPlan plan = 3; + Sink sink = 4; +} + +message JobResponse { + uint64 job_id = 1; + bytes data = 2; +} + +service JobService { + rpc Submit(JobRequest) returns(stream JobResponse) {} +} \ No newline at end of file diff --git a/research/engine/pegasus/server-v0/src/config.rs b/research/engine/pegasus/server-v0/src/config.rs new file mode 100644 index 000000000000..5e767ac7fb9d --- /dev/null +++ b/research/engine/pegasus/server-v0/src/config.rs @@ -0,0 +1,94 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +#![allow(dead_code)] +#![allow(unused_imports)] +#![allow(unused_variables)] +use std::fmt::Debug; +use std::path::Path; + +use pegasus::{Configuration, StartupError}; +use pegasus_network::config::{NetworkConfig, ServerAddr}; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +pub struct HostsConfig { + pub peers: Vec, +} + +impl HostsConfig { + pub fn parse(content: &str) -> Result { + toml::from_str(&content) + } + + pub fn read_from>(path: P) -> Result { + let config_str = std::fs::read_to_string(path)?; + Ok(HostsConfig::parse(&config_str)?) + } +} + +#[derive(Debug, Deserialize)] +pub struct CommonConfig { + pub max_pool_size: Option, + pub nonblocking: Option, + pub read_timeout_ms: Option, + pub write_timeout_ms: Option, + pub read_slab_size: Option, + pub no_delay: Option, + pub send_buffer: Option, + pub heartbeat_sec: Option, +} + +impl CommonConfig { + pub fn parse(content: &str) -> Result { + toml::from_str(&content) + } + + pub fn read_from>(path: P) -> Result { + let config_str = std::fs::read_to_string(path)?; + Ok(CommonConfig::parse(&config_str)?) + } +} + +pub fn combine_config( + server_id: u64, host_config: Option, common_config: Option, +) -> Option { + if let Some(host_config) = host_config { + let local_host = &host_config.peers[server_id as usize]; + let ip = local_host.get_ip().to_owned(); + let port = local_host.get_port(); + let config = if let Some(common_config) = common_config { + let mut network_config = NetworkConfig::with(server_id, host_config.peers); + network_config.nonblocking(common_config.nonblocking) + .read_timeout_ms(common_config.read_timeout_ms) + .write_timeout_ms(common_config.write_timeout_ms) + .read_slab_size(common_config.read_slab_size) + .no_delay(common_config.no_delay) + .send_buffer(common_config.send_buffer) + .heartbeat_sec(common_config.heartbeat_sec); + Configuration { network: Some(network_config), max_pool_size: common_config.max_pool_size } + } else { + let network_config = NetworkConfig::with(server_id, host_config.peers); + Configuration { network: Some(network_config), max_pool_size: None } + }; + Some(config) + } else { + if let Some(common_config) = common_config { + Some(Configuration { network: None, max_pool_size: common_config.max_pool_size }) + } else { + None + } + } +} diff --git a/research/engine/pegasus/server-v0/src/lib.rs b/research/engine/pegasus/server-v0/src/lib.rs new file mode 100644 index 000000000000..6827db7bec8c --- /dev/null +++ b/research/engine/pegasus/server-v0/src/lib.rs @@ -0,0 +1,72 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +#[macro_use] +extern crate log; + +pub use config::{CommonConfig, HostsConfig}; +use pegasus::Data; + +#[cfg(not(feature = "gcip"))] +mod generated { + pub mod protocol { + tonic::include_proto!("protocol"); + } +} + +#[cfg(feature = "gcip")] +mod generated { + #[path = "protocol.rs"] + pub mod protocol; +} + +pub use generated::protocol as pb; + +pub trait AnyData: Data + Eq {} + +// pub mod client; +pub mod config; +pub mod rpc; +pub mod service; + +pub use generated::protocol::{JobRequest, JobResponse}; + +#[allow(dead_code)] +pub fn report_memory(job_id: u64) -> Option> { + let g = std::thread::Builder::new() + .name(format!("memory-reporter {}", job_id)) + .spawn(move || { + let mut max_usage = 0; + let mut count_zero_times = 50; + loop { + if let Some(usage) = pegasus_memory::alloc::check_task_memory(job_id as usize) { + if usage > max_usage { + max_usage = usage; + } + } else if max_usage > 0 { + break; + } else if max_usage == 0 { + count_zero_times -= 1; + if count_zero_times <= 0 { + break; + } + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + info!("Job {} memory usage: {:.4} MB;", job_id, max_usage as f64 / 1_000_000.0); + }) + .unwrap(); + Some(g) +} diff --git a/research/engine/pegasus/server-v0/src/rpc.rs b/research/engine/pegasus/server-v0/src/rpc.rs new file mode 100644 index 000000000000..9da80b6441f1 --- /dev/null +++ b/research/engine/pegasus/server-v0/src/rpc.rs @@ -0,0 +1,199 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::error::Error; +use std::fmt::Debug; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; + +use pegasus::api::function::FnResult; +use pegasus::api::FromStream; +use pegasus::result::{FromStreamExt, ResultSink}; +use pegasus::{Data, JobConf, ServerConf}; +use prost::Message; +use tokio::sync::mpsc::UnboundedSender; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tonic::transport::Server; +use tonic::{Code, Request, Response, Status}; + +use crate::generated::protocol as pb; +use crate::service::{JobParser, Service}; +use tokio::net::TcpListener; + +pub struct RpcSink { + pub job_id: u64, + had_error: Arc, + peers: Arc, + tx: UnboundedSender>, +} + +impl RpcSink { + pub fn new(job_id: u64, tx: UnboundedSender>) -> Self { + RpcSink { + tx, + had_error: Arc::new(AtomicBool::new(false)), + peers: Arc::new(AtomicUsize::new(1)), + job_id, + } + } +} + +impl FromStream for RpcSink { + fn on_next(&mut self, next: T) -> FnResult<()> { + let bytes = next.encode_to_vec(); + let res = pb::JobResponse { job_id: self.job_id, data: bytes }; + self.tx.send(Ok(res)).ok(); + Ok(()) + } +} + +impl Clone for RpcSink { + fn clone(&self) -> Self { + self.peers.fetch_add(1, Ordering::SeqCst); + RpcSink { + job_id: self.job_id, + had_error: self.had_error.clone(), + peers: self.peers.clone(), + tx: self.tx.clone(), + } + } +} + +impl FromStreamExt for RpcSink { + fn on_error(&mut self, error: Box) { + self.had_error.store(true, Ordering::SeqCst); + let status = Status::unknown(format!("execution_error: {}", error)); + self.tx.send(Err(status)).ok(); + } +} + +impl Drop for RpcSink { + fn drop(&mut self) { + let before_sub = self.peers.fetch_sub(1, Ordering::SeqCst); + if before_sub == 1 { + if !self.had_error.load(Ordering::SeqCst) { + self.tx.send(Err(Status::ok("ok"))).ok(); + } + } + } +} + +#[derive(Clone)] +pub struct RpcService { + inner: Service, + report: bool, +} + +impl RpcService { + pub fn new(service: Service, report: bool) -> RpcService { + RpcService { inner: service, report } + } +} + +#[tonic::async_trait] +impl pb::job_service_server::JobService for RpcService +where + I: Data, + O: Send + Debug + Message + 'static, + P: JobParser, +{ + type SubmitStream = UnboundedReceiverStream>; + + async fn submit(&self, req: Request) -> Result, Status> { + let mut job_req = req.into_inner(); + if job_req.conf.is_none() { + return Err(Status::new(Code::InvalidArgument, "job configuration not found")); + } + + let conf_req = job_req.conf.take().unwrap(); + let conf = parse_conf_req(conf_req); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let rpc_sink = RpcSink::new(conf.job_id, tx); + let sink = ResultSink::::with(rpc_sink); + let service = self.inner.clone(); + let submitted = + pegasus::run_opt(conf, sink, move |worker| worker.dataflow(service.accept(&job_req))); + + if let Err(e) = submitted { + return Err(Status::invalid_argument(format!("submit job error {}", e))); + } + + Ok(Response::new(UnboundedReceiverStream::new(rx))) + } +} + +pub struct RpcServer { + service: S, + addr: SocketAddr, +} + +pub async fn start_rpc_server( + addr: SocketAddr, service: RpcService, blocking: bool, +) -> Result> +where + I: Data, + O: Send + Debug + Message + 'static, + P: JobParser, +{ + let server = RpcServer::new(addr, service); + let local_addr = server.run(blocking).await?; + Ok(local_addr) +} + +impl RpcServer { + pub fn new(addr: SocketAddr, service: S) -> Self { + RpcServer { service, addr } + } + + pub async fn run(self, blocking: bool) -> Result> { + let RpcServer { service, addr } = self; + let listener = TcpListener::bind(addr).await?; + let local_addr = listener.local_addr()?; + info!("Rpc server started on {}", local_addr); + let serve = Server::builder() + .add_service(pb::job_service_server::JobServiceServer::new(service)) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)); + if blocking { + serve.await?; + } else { + tokio::spawn(async move { + serve.await.expect("Rpc server start error"); + }); + } + Ok(local_addr) + } +} + +fn parse_conf_req(conf: pb::JobConfig) -> JobConf { + let mut job_conf = JobConf::with_id(conf.job_id, conf.job_name, conf.workers); + if conf.time_limit != 0 { + job_conf.time_limit = conf.time_limit; + } + if conf.batch_size != 0 { + job_conf.batch_size = conf.batch_size; + } + if conf.output_capacity != 0 { + job_conf.batch_capacity = conf.output_capacity; + } + if conf.memory_limit != 0 { + job_conf.memory_limit = conf.memory_limit; + } + job_conf.plan_print = conf.plan_print; + if !conf.servers.is_empty() { + job_conf.reset_servers(ServerConf::Partial(conf.servers.clone())); + } + job_conf +} diff --git a/research/engine/pegasus/server-v0/src/service.rs b/research/engine/pegasus/server-v0/src/service.rs new file mode 100644 index 000000000000..d253e64b4501 --- /dev/null +++ b/research/engine/pegasus/server-v0/src/service.rs @@ -0,0 +1,58 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +#![allow(dead_code)] + +use std::fmt::Debug; +use std::sync::Arc; + +use pegasus::api::Source; +use pegasus::result::ResultSink; +use pegasus::{BuildJobError, Data}; +use prost::Message; + +use crate::generated::protocol as pb; + +pub trait JobParser: Send + Sync + 'static { + fn parse( + &self, plan: &pb::JobRequest, input: &mut Source, output: ResultSink, + ) -> Result<(), BuildJobError>; +} + +pub struct Service { + parser: Arc

, + _ph: std::marker::PhantomData<(I, O)>, +} + +impl Clone for Service { + fn clone(&self) -> Self { + Service { parser: self.parser.clone(), _ph: std::marker::PhantomData } + } +} + +unsafe impl Sync for Service {} + +impl> Service { + pub fn new(parser: P) -> Self { + Service { parser: Arc::new(parser), _ph: std::marker::PhantomData } + } + + pub fn accept<'a>( + &'a self, req: &'a pb::JobRequest, + ) -> impl FnOnce(&mut Source, ResultSink) -> Result<(), BuildJobError> + 'a { + move |input, output| self.parser.parse(req, input, output) + } +} diff --git a/research/graph_store/src/graph_db.rs b/research/graph_store/src/graph_db.rs index 0d661f25eae3..f80e44a28575 100644 --- a/research/graph_store/src/graph_db.rs +++ b/research/graph_store/src/graph_db.rs @@ -196,6 +196,10 @@ impl<'a, G: IndexType, I: IndexType> LocalEdge<'a, G, I> { } } + pub fn is_from_start(&self) -> bool { + self.from_start + } + pub fn get_label(&self) -> LabelId { self.label } diff --git a/research/graph_store/src/graph_db_impl.rs b/research/graph_store/src/graph_db_impl.rs index d138e0f99b24..ffb64258ff25 100644 --- a/research/graph_store/src/graph_db_impl.rs +++ b/research/graph_store/src/graph_db_impl.rs @@ -549,24 +549,20 @@ where } fn get_edge(&self, edge_id: EdgeId) -> Option> { - if self.is_vertex_local(edge_id.0) { - let ei = edge_index::(edge_id.1); - if let Some((src, dst)) = self.graph.edge_endpoints(ei.clone()) { - let _src_v = self.index_data.get_global_id(src); - let _dst_v = self.index_data.get_global_id(dst); - let label = *self.graph.edge_weight(ei).unwrap(); - if _src_v.is_some() && _dst_v.is_some() { - let mut local_edge = LocalEdge::new(_src_v.unwrap(), _dst_v.unwrap(), label, ei); - if let Some(properties) = self.get_all_edge_property(&ei) { - local_edge = local_edge.with_properties(RowWithSchema::new( - Some(properties), - self.graph_schema.get_edge_schema(label), - )); - } - Some(local_edge) - } else { - None + let ei = edge_index::(edge_id.1); + if let Some((src, dst)) = self.graph.edge_endpoints(ei.clone()) { + let _src_v = self.index_data.get_global_id(src); + let _dst_v = self.index_data.get_global_id(dst); + let label = *self.graph.edge_weight(ei).unwrap(); + if _src_v.is_some() && _dst_v.is_some() { + let mut local_edge = LocalEdge::new(_src_v.unwrap(), _dst_v.unwrap(), label, ei); + if let Some(properties) = self.get_all_edge_property(&ei) { + local_edge = local_edge.with_properties(RowWithSchema::new( + Some(properties), + self.graph_schema.get_edge_schema(label), + )); } + Some(local_edge) } else { None } diff --git a/research/query_service/benchmark/README.md b/research/query_service/benchmark/README.md new file mode 100644 index 000000000000..4d499e51f75b --- /dev/null +++ b/research/query_service/benchmark/README.md @@ -0,0 +1,63 @@ +## Benchmark Tool Usage + +In this directory is a tool that can be used to benchmark GAIA. It serves as multiple clients to send +queries to gremlin server through the gremlin endpoint exposed by the engine, and report the performance numbers +(e.g., latency, throughput, query results). +The benchmark program sends mixed queries to the server by reading query templates from [queries](queries) with filling the parameters in the query templates +using [substitution_parameters](data/substitution_parameters). +The program uses a round-robin strategy to iterate all the **enabled** queries with corresponding parameters. + +### Repository contents +``` +- config + - interactive-benchmark.properties // configurations for running benchmark +- data + - substitution_parameters // query parameter files using to fill the query templates +- queries // qurery templates including LDBC queries, K-hop queries and user-defined queries +- shell + - benchmark.sh // script for running benchmark +- src // source code of benchmark program +``` +_Note:_ the queries here with the prefix _ldbc_query_ are implementations of LDBC official interactive complex reads, +and the corresponding parameters (factor 1) are generated by [LDBC official tools](http://github.com/ldbc/ldbc_snb_datagen). + +### Building + +Build benchmark program using Maven: +```bash +mvn clean package +``` +All the binary and queries would be packed into _target/benchmark-0.0.1-SNAPSHOT-dist.tar.gz_, +and you can use deploy the package to anywhere could connect to the gremlin endpoint. + +### Running the benchmark + +```bash +cd target +tar -xvf gaia-benchmark-0.0.1-SNAPSHOT-dist.tar.gz +cd gaia-benchmark-0.0.1-SNAPSHOT +vim config/interactive-benchmark.properties # specify the gremlin endpoint of your server and modify running configurations +chmod +x ./shell/benchmark.sh +./shell/benchmark.sh # run the benchmark program +``` + +Benchmark reports numbers as following: +``` +QueryName[LDBC_QUERY_1], Parameter[{firstName=John, personId=17592186223433}], ResultCount[87], ExecuteTimeMS[ 1266 ]. +QueryName[LDBC_QUERY_12], Parameter[{tagClassName=Judge, personId=19791209469071}], ResultCount[0], ExecuteTimeMS[ 259 ]. +QueryName[LDBC_QUERY_11], Parameter[{workFromYear=2001, personId=32985348901156, countryName=Bolivia}], ResultCount[0], ExecuteTimeMS[ 60 ]. +QueryName[LDBC_QUERY_9], Parameter[{personId=10995116420051, maxDate=20121128080000000}], ResultCount[20], ExecuteTimeMS[ 55755 ]. +QueryName[LDBC_QUERY_8], Parameter[{personId=67523}], ResultCount[20], ExecuteTimeMS[ 148 ]. +QueryName[LDBC_QUERY_7], Parameter[{personId=26388279199350}], ResultCount[0], ExecuteTimeMS[ 10 ]. +QueryName[LDBC_QUERY_6], Parameter[{personId=26388279148519, tagName=Vallabhbhai_Patel}], ResultCount[0], ExecuteTimeMS[ 12837 ]. +QueryName[LDBC_QUERY_5], Parameter[{minDate=20120814080000000, personId=2199023436754}], ResultCount[0], ExecuteTimeMS[ 11268 ]. +QueryName[LDBC_QUERY_3], Parameter[{durationDays=30, endDate=20110701080000000, countryXName=Mongolia, countryYName=Namibia, personId=8796093204429, startDate=20110601080000000}], ResultCount[20] +, ExecuteTimeMS[ 21474 ]. +QueryName[LDBC_QUERY_2], Parameter[{personId=28587302394490, maxDate=20121128080000000}], ResultCount[20], ExecuteTimeMS[ 331 ]. +query count: 10; execute time(ms): ...; qps: ... +``` + +### User-defined Benchmarking Queries +Users can add their own benchmarking queries to [queries](queries) as well as adding substitution parameters of queries to [substitution_parameters](data/substitution_parameters). +Note that the file name of user-defined query templates should follow the prefix _custom_query_ or _custom_constant_query_. The difference between custom_query and +custom_constant_query is that the latter has no corresponding parameters. diff --git a/research/query_service/benchmark/assembly.xml b/research/query_service/benchmark/assembly.xml new file mode 100644 index 000000000000..694efbdfa1c0 --- /dev/null +++ b/research/query_service/benchmark/assembly.xml @@ -0,0 +1,58 @@ + + + dist + + tar.gz + + + + /lib + false + runtime + + + + + config/ + /config + + * + + + + shell/ + /shell + + * + + + + queries/ + /queries + + * + + + + queries/ir_ldbc + /queries/ir_ldbc + + * + + + + queries/gaiax_ldbc + /queries/gaiax_ldbc + + * + + + + data/substitution_parameters + /data/substitution_parameters + + * + + + + \ No newline at end of file diff --git a/research/query_service/benchmark/config/interactive-benchmark.properties b/research/query_service/benchmark/config/interactive-benchmark.properties new file mode 100644 index 000000000000..d27c76ecf8c7 --- /dev/null +++ b/research/query_service/benchmark/config/interactive-benchmark.properties @@ -0,0 +1,51 @@ +# gremlin server endpoint +endpoint=127.0.0.1:8182 +# the number of threads sending queries +thread_count=1 +# the number of warmup queries each kind of query +warmup.every.query=0 +# the number of queries sending by each thread +operation.count.every.query=11 +# the directory of query templates +queryDir=./queries/ir_ldbc/ +# the director of query parameters +interactive.parameters.dir=./data/substitution_parameters/ + +# enable print info +printQueryNames=true +printQueryStrings=true +printQueryResults=false + +# specify which kind of queries are sent +# ldbc interactive complex read queries +ldbc_query_1.enable=true +ldbc_query_2.enable=true +ldbc_query_3.enable=true +ldbc_query_4.enable=true +ldbc_query_5.enable=true +ldbc_query_6.enable=true +ldbc_query_7.enable=true +ldbc_query_8.enable=true +ldbc_query_9.enable=true +ldbc_query_11.enable=true +ldbc_query_12.enable=true + +# k-hop queries +1_hop_query.enable=false +2_hop_query.enable=false +3_hop_query.enable=false +4_hop_query.enable=false + +# early stop queries +early_stop_query.enable=false + +# subtask queries +subtask_query.enable=false + +# custom queries without parameters +custom_constant_query_1.enable=false +custom_constant_query_2.enable=false + +# custom queries +custom_query_1.enable=false +custom_query_2.enable=false diff --git a/research/query_service/benchmark/data/substitution_parameters/1_hop_query.param b/research/query_service/benchmark/data/substitution_parameters/1_hop_query.param new file mode 100644 index 000000000000..f1d05073ffd8 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/1_hop_query.param @@ -0,0 +1,11 @@ +vertexId +216174019064365056 +76 +72057594037932075 +216174843698085892 +86 +504403158265497622 +504403158265496359 +504403158265496331 +72090579386766311 +72064191107705176 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/2_hop_query.param b/research/query_service/benchmark/data/substitution_parameters/2_hop_query.param new file mode 100644 index 000000000000..80c5295d8156 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/2_hop_query.param @@ -0,0 +1,11 @@ +vertexId +72068589154212744 +144117249660157961 +504403158265506123 +504403158265498631 +504403158265498523 +504403158265497606 +504403158265497323 +504403158265495610 +50 +504403158265504715 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/3_hop_query.param b/research/query_service/benchmark/data/substitution_parameters/3_hop_query.param new file mode 100644 index 000000000000..6d95f3ce571f --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/3_hop_query.param @@ -0,0 +1,11 @@ +vertexId +144117249660157961 +504403158265498631 +504403158265498523 +50 +216174019064365056 +216174843698085892 +86 +504403158265496331 +72090579386766311 +72064191107705176 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/4_hop_query.param b/research/query_service/benchmark/data/substitution_parameters/4_hop_query.param new file mode 100644 index 000000000000..72e9a93ad368 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/4_hop_query.param @@ -0,0 +1,11 @@ +vertexId +504403158265496359 +504403158265496331 +72090579386766311 +144117249660157961 +76 +216174843698085892 +86 +504403158265496331 +72090579386766311 +72064191107705176 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/custom_query_1.param b/research/query_service/benchmark/data/substitution_parameters/custom_query_1.param new file mode 100644 index 000000000000..ec6096b0abd2 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/custom_query_1.param @@ -0,0 +1,11 @@ +vertexId +144117249660157961 +504403158265506123 +504403158265498631 +504403158265498523 +504403158265497606 +504403158265497323 +504403158265497622 +504403158265496331 +72090579386766311 +72064191107705176 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/custom_query_2.param b/research/query_service/benchmark/data/substitution_parameters/custom_query_2.param new file mode 100644 index 000000000000..45eeb9467a60 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/custom_query_2.param @@ -0,0 +1,11 @@ +vertexId +504403158265506123 +504403158265495610 +216174019064365056 +76 +86 +504403158265497622 +504403158265496359 +504403158265496331 +72090579386766311 +72064191107705176 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/early_stop_query.param b/research/query_service/benchmark/data/substitution_parameters/early_stop_query.param new file mode 100644 index 000000000000..d30110d49fab --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/early_stop_query.param @@ -0,0 +1,11 @@ +vertexId +216174019064365056 +86 +504403158265497622 +504403158265496359 +504403158265496331 +72090579386766311 +72064191107705176 +72068589154212744 +144117249660157961 +76 \ No newline at end of file diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_1.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_1.param new file mode 100644 index 000000000000..856d7aa1bb64 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_1.param @@ -0,0 +1,99 @@ +personId|firstName +30786325583618|Chau +17592186053645|John +4398046512818|John +30786325578911|John +6597069766659|John +26388279075172|Jun +19791209309138|John +24189255822062|Mark +10995116287298|Baruch +30786325578963|John +21990232557314|John +2199023264241|John +15393162790365|John +6597069769100|Jun +6597069773997|Jimmy +17592186045019|John +30786325587888|Kenji +17592186045360|John +10995116280597|John +15393162790770|John +9782|John +8796093026933|Ganesh +28587302322817|John +13194139542834|Denis +17592186051169|John +4398046513464|John +13194139535318|Klaus +4398046517060|John +13194139538142|John +19791209300404|Hsin +19791209300882|John +6597069770264|Gabriel +2199023260581|Babar +4398046516880|John +13194139542764|John +768|John +26388279074461|John +32985348835137|John +8796093032135|Haim +26388279074072|Cesar +35184372095658|John +13194139539724|Jun +15393162789349|John +17592186054325|Laurent +21990232562091|John +30786325587843|John +30786325587276|John +17592186047707|Ian +30786325585356|John +13194139542130|Jun +15393162795979|John +24189255811315|John +15393162791567|Amrozi Bin +32985348844096|John +13194139541713|John +17592186051933|John +13194139536557|Barry +21990232565814|Babar +26388279068701|Jun +19791209306192|John +30786325581911|Jun +2199023256125|Eun-Hye +4398046518845|Jorge +19791209300153|Francisco +4398046521818|David +17592186053195|John +13194139542945|Helen +32985348836952|John +6597069776117|Joseph +2199023255940|John +28587302327943|John +24189255820909|Jimmy +26388279077068|Jun +10995116283556|John +15393162796757|John +17592186052877|Ali +2199023265470|John +24189255814005|Jun +15393162789189|Jan +32985348842593|Jose +13194139535090|Jose +4398046517753|Karl +24189255813156|Francisco +4398046514583|John +17592186049987|Jimmy +10995116283196|John +19791209308316|John +35184372094078|Stephen +19791209302972|Albaye Papa +15393162790921|John +32985348837345|Jun +21990232562252|John +2199023257044|Gabriel +13194139533606|John +10995116281419|John +17592186050886|John +21990232556833|Helen +6597069767674|John diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_11.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_11.param new file mode 100644 index 000000000000..60206791e8ec --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_11.param @@ -0,0 +1,99 @@ +personId|countryName|workFromYear +30786325583618|Laos|2010 +17592186053645|Uruguay|2005 +4398046512818|Scotland|2004 +30786325578911|Norway|2009 +6597069766659|Bosnia_and_Herzegovina|1998 +26388279075172|Papua_New_Guinea|2006 +19791209309138|Honduras|2014 +24189255822062|Slovakia|2009 +10995116287298|Mauritania|2004 +30786325578963|Lithuania|2009 +21990232557314|Liberia|2008 +2199023264241|Norway|2001 +15393162790365|Honduras|2001 +6597069769100|Bosnia_and_Herzegovina|2013 +6597069773997|Norway|2002 +17592186045019|Laos|1999 +30786325587888|Liberia|2012 +17592186045360|Liberia|2006 +10995116280597|Liberia|2004 +15393162790770|Uruguay|2006 +9782|Scotland|2010 +8796093026933|Honduras|2000 +28587302322817|Liberia|2009 +13194139542834|Papua_New_Guinea|2010 +17592186051169|Slovakia|2011 +4398046513464|Slovakia|2002 +13194139535318|Scotland|2008 +4398046517060|Papua_New_Guinea|2001 +13194139538142|Norway|2007 +19791209300404|Scotland|2000 +19791209300882|Laos|2011 +6597069770264|Norway|2012 +2199023260581|Liberia|2003 +4398046516880|Bosnia_and_Herzegovina|2001 +13194139542764|Slovakia|2014 +768|Slovakia|2010 +26388279074461|Liberia|2012 +32985348835137|Bosnia_and_Herzegovina|1998 +8796093032135|Norway|2013 +26388279074072|Norway|2008 +35184372095658|Norway|2003 +13194139539724|Lithuania|2005 +15393162789349|Lithuania|2010 +17592186054325|Norway|2011 +21990232562091|Norway|2001 +30786325587843|Papua_New_Guinea|2008 +30786325587276|Honduras|2000 +17592186047707|Honduras|2014 +30786325585356|Scotland|2005 +13194139542130|Laos|2013 +15393162795979|Scotland|2010 +24189255811315|Laos|2008 +15393162791567|Honduras|2002 +32985348844096|Laos|2006 +13194139541713|Laos|2000 +17592186051933|Honduras|2000 +13194139536557|Norway|2010 +21990232565814|Mauritania|2004 +26388279068701|Papua_New_Guinea|2010 +19791209306192|Lithuania|2002 +30786325581911|Uruguay|2010 +2199023256125|Papua_New_Guinea|2010 +4398046518845|Mauritania|2003 +19791209300153|Laos|1999 +4398046521818|Liberia|2004 +17592186053195|Uruguay|2006 +13194139542945|Mauritania|1999 +32985348836952|Liberia|2001 +6597069776117|Lithuania|1998 +2199023255940|Norway|2008 +28587302327943|Uruguay|2013 +24189255820909|Papua_New_Guinea|2001 +26388279077068|Liberia|1998 +10995116283556|Norway|2009 +15393162796757|Lithuania|2011 +17592186052877|Uruguay|2014 +2199023265470|Liberia|2008 +24189255814005|Laos|2003 +15393162789189|Norway|2012 +32985348842593|Lithuania|2000 +13194139535090|Mauritania|2009 +4398046517753|Lithuania|1999 +24189255813156|Lithuania|2004 +4398046514583|Mauritania|2006 +17592186049987|Slovakia|2004 +10995116283196|Uruguay|2000 +19791209308316|Bosnia_and_Herzegovina|2001 +35184372094078|Uruguay|2011 +19791209302972|Slovakia|2005 +15393162790921|Slovakia|2007 +32985348837345|Scotland|2001 +21990232562252|Laos|2010 +2199023257044|Liberia|2003 +13194139533606|Mauritania|2008 +10995116281419|Honduras|2013 +17592186050886|Papua_New_Guinea|2014 +21990232556833|Lithuania|1998 +6597069767674|Bosnia_and_Herzegovina|2011 diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_12.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_12.param new file mode 100644 index 000000000000..8ab423b40cce --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_12.param @@ -0,0 +1,99 @@ +personId|tagClassName +17592186052613|BasketballPlayer +4398046514238|MilitaryUnit +4398046513906|Chancellor +10995116287298|GolfPlayer +6597069777312|Criminal +24189255819351|MilitaryUnit +17592186045019|Chancellor +10976|MilitaryUnit +15393162792273|Chancellor +21990232566199|MilitaryUnit +19791209306192|GolfPlayer +8796093032135|GolfPlayer +24189255820909|Criminal +3781|Criminal +10995116280426|Chancellor +8796093026933|Chancellor +6597069767679|Criminal +30786325583425|GolfPlayer +19791209300882|Chancellor +24189255820428|MilitaryUnit +32985348844096|MilitaryUnit +21990232557314|BasketballPlayer +32985348837345|Criminal +4398046511941|BasketballPlayer +24189255811654|GolfPlayer +4398046516895|Chancellor +24189255820924|Criminal +17592186053195|GolfPlayer +10995116287061|Criminal +4398046517138|BasketballPlayer +24189255815321|Chancellor +26388279072666|Chancellor +2199023256552|MilitaryUnit +8796093022963|Chancellor +28587302332884|MilitaryUnit +2199023255940|Chancellor +19791209309138|BasketballPlayer +28587302322817|Chancellor +32985348843190|Chancellor +17592186054325|MilitaryUnit +28587302332187|MilitaryUnit +6597069771814|GolfPlayer +15393162799483|GolfPlayer +2199023259917|Chancellor +28587302327029|GolfPlayer +13194139542130|MilitaryUnit +2199023265945|MilitaryUnit +32985348842593|BasketballPlayer +2199023263001|MilitaryUnit +6597069776117|Chancellor +17592186045360|Criminal +2199023260581|Criminal +4398046521818|Chancellor +2199023255871|Criminal +6597069766659|Chancellor +2199023265470|Criminal +4398046514583|Chancellor +17592186049682|GolfPlayer +13194139542834|Criminal +30786325578963|MilitaryUnit +26388279075172|BasketballPlayer +10995116288293|GolfPlayer +8796093026900|Chancellor +10995116283556|MilitaryUnit +17592186054490|BasketballPlayer +24189255811315|MilitaryUnit +21990232556833|Chancellor +30786325583618|Chancellor +24189255813156|Criminal +21990232562671|Chancellor +28587302322286|GolfPlayer +24189255818526|Criminal +32985348839614|GolfPlayer +4398046516880|Chancellor +30786325587843|GolfPlayer +32985348843447|GolfPlayer +4398046513464|GolfPlayer +9782|GolfPlayer +15393162789349|Chancellor +8796093024749|GolfPlayer +4398046517060|GolfPlayer +15393162793546|Criminal +768|GolfPlayer +30786325581911|Criminal +15393162796638|GolfPlayer +2199023258130|Criminal +26388279071471|GolfPlayer +19791209305199|MilitaryUnit +13194139539724|MilitaryUnit +17592186052877|GolfPlayer +17592186049987|Criminal +10009|Chancellor +15393162790770|BasketballPlayer +13194139538142|Criminal +26388279077068|Chancellor +21990232562252|Chancellor +32985348839526|BasketballPlayer +13194139533606|Chancellor diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_2.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_2.param new file mode 100644 index 000000000000..26d89bd7259e --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_2.param @@ -0,0 +1,99 @@ +personId|maxDate +17592186052613|1354060800000 +4398046514238|1354060800000 +4398046513906|1323561600000 +10995116287298|1323648000000 +6597069777312|1347494400000 +24189255819351|1330300800000 +17592186045019|1334707200000 +10976|1334448000000 +15393162792273|1354060800000 +21990232566199|1320364800000 +19791209306192|1323388800000 +8796093032135|1337990400000 +24189255820909|1354060800000 +3781|1314230400000 +10995116280426|1323993600000 +8796093026933|1354060800000 +6597069767679|1337385600000 +30786325583425|1337472000000 +19791209300882|1341792000000 +24189255820428|1323734400000 +32985348844096|1346457600000 +21990232557314|1329004800000 +32985348837345|1345334400000 +4398046511941|1291507200000 +24189255811654|1354060800000 +4398046516895|1339718400000 +24189255820924|1333238400000 +17592186053195|1354060800000 +10995116287061|1354060800000 +4398046517138|1323475200000 +24189255815321|1332374400000 +26388279072666|1345248000000 +2199023256552|1353456000000 +8796093022963|1344038400000 +28587302332884|1331251200000 +2199023255940|1341100800000 +19791209309138|1354060800000 +28587302322817|1354060800000 +32985348843190|1354060800000 +17592186054325|1354060800000 +28587302332187|1334880000000 +6597069771814|1348704000000 +15393162799483|1309651200000 +2199023259917|1354060800000 +28587302327029|1330128000000 +13194139542130|1336348800000 +2199023265945|1339632000000 +32985348842593|1354060800000 +2199023263001|1345507200000 +6597069776117|1354060800000 +17592186045360|1354060800000 +2199023260581|1334016000000 +4398046521818|1354060800000 +2199023255871|1323648000000 +6597069766659|1354060800000 +2199023265470|1354060800000 +4398046514583|1354060800000 +17592186049682|1343779200000 +13194139542834|1343174400000 +30786325578963|1350604800000 +26388279075172|1324598400000 +10995116288293|1354060800000 +8796093026900|1354060800000 +10995116283556|1354060800000 +17592186054490|1346457600000 +24189255811315|1329091200000 +21990232556833|1354060800000 +30786325583618|1354060800000 +24189255813156|1354060800000 +21990232562671|1339459200000 +28587302322286|1354060800000 +24189255818526|1344384000000 +32985348839614|1349222400000 +4398046516880|1354060800000 +30786325587843|1354060800000 +32985348843447|1337817600000 +4398046513464|1354060800000 +9782|1354060800000 +15393162789349|1354060800000 +8796093024749|1337644800000 +4398046517060|1354060800000 +15393162793546|1345680000000 +768|1346025600000 +30786325581911|1354060800000 +15393162796638|1354060800000 +2199023258130|1323561600000 +26388279071471|1322697600000 +19791209305199|1321747200000 +13194139539724|1333670400000 +17592186052877|1354060800000 +17592186049987|1354060800000 +10009|1330819200000 +15393162790770|1354060800000 +13194139538142|1354060800000 +26388279077068|1354060800000 +21990232562252|1354060800000 +32985348839526|1345852800000 +13194139533606|1354060800000 diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_3.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_3.param new file mode 100644 index 000000000000..c037402b56ea --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_3.param @@ -0,0 +1,99 @@ +personId|startDate|durationDays|countryXName|countryYName +17592186055119|1306886400000|42|Laos|Scotland +8796093030404|1298937600000|28|Uruguay|Scotland +2199023266354|1301616000000|30|Scotland|Slovakia +6597069774475|1301616000000|31|Norway|Scotland +21990232560302|1306886400000|30|Bosnia_and_Herzegovina|Laos +6597069767470|1298937600000|41|Papua_New_Guinea|Honduras +32985348839760|1298937600000|28|Honduras|Mauritania +28587302331227|1291161600000|29|Slovakia|Honduras +6597069775210|1296518400000|32|Mauritania|Slovakia +6597069777205|1306886400000|28|Lithuania|Norway +6597069768320|1298937600000|36|Liberia|Slovakia +19791209307492|1306886400000|32|Uruguay|Slovakia +21990232558067|1298937600000|29|Slovakia|Uruguay +28587302322515|1298937600000|32|Norway|Scotland +8796093025131|1296518400000|28|Papua_New_Guinea|Mauritania +21990232560628|1296518400000|28|Norway|Bosnia_and_Herzegovina +32985348837736|1296518400000|28|Lithuania|Mauritania +32985348840452|1296518400000|29|Lithuania|Laos +32985348834961|1301616000000|40|Laos|Uruguay +17592186047698|1298937600000|31|Scotland|Uruguay +2199023261819|1298937600000|31|Norway|Laos +15393162792270|1296518400000|32|Liberia|Bosnia_and_Herzegovina +24189255821418|1296518400000|33|Mauritania|Liberia +24189255819548|1298937600000|31|Norway|Lithuania +15393162791355|1298937600000|29|Scotland|Norway +30786325578316|1301616000000|29|Slovakia|Lithuania +6597069776966|1298937600000|33|Lithuania|Honduras +2199023260581|1301616000000|30|Liberia|Mauritania +21990232560233|1306886400000|33|Norway|Honduras +4398046516880|1298937600000|42|Lithuania|Liberia +30786325586428|1301616000000|31|Slovakia|Liberia +28587302325306|1298937600000|34|Papua_New_Guinea|Lithuania +28587302324102|1298937600000|33|Liberia|Laos +30786325587163|1298937600000|41|Scotland|Bosnia_and_Herzegovina +2199023258040|1298937600000|32|Slovakia|Honduras +8796093032964|1298937600000|32|Laos|Uruguay +4398046514525|1291161600000|30|Uruguay|Norway +28587302328025|1306886400000|32|Liberia|Uruguay +32985348834523|1296518400000|29|Scotland|Liberia +10995116278636|1296518400000|30|Mauritania|Slovakia +21990232566347|1301616000000|30|Honduras|Liberia +13194139542764|1298937600000|28|Lithuania|Scotland +4398046520770|1298937600000|37|Bosnia_and_Herzegovina|Uruguay +17592186051070|1291161600000|30|Norway|Papua_New_Guinea +24189255811500|1306886400000|28|Norway|Laos +6597069777346|1306886400000|32|Lithuania|Honduras +8796093031741|1298937600000|33|Honduras|Scotland +17592186052400|1325376000000|35|Liberia|Bosnia_and_Herzegovina +6597069768934|1306886400000|31|Lithuania|Slovakia +30786325578737|1306886400000|38|Uruguay|Mauritania +30786325584591|1306886400000|36|Honduras|Lithuania +21990232557143|1298937600000|33|Uruguay|Scotland +24189255818799|1298937600000|31|Laos|Liberia +4398046517049|1298937600000|29|Laos|Mauritania +13194139542115|1298937600000|34|Lithuania|Uruguay +10995116282665|1298937600000|36|Mauritania|Papua_New_Guinea +28587302330379|1296518400000|31|Lithuania|Honduras +4398046515300|1301616000000|29|Norway|Mauritania +30786325586109|1296518400000|30|Honduras|Slovakia +32985348842142|1298937600000|29|Mauritania|Honduras +17592186052168|1298937600000|29|Bosnia_and_Herzegovina|Lithuania +10995116285640|1298937600000|29|Honduras|Lithuania +6597069777539|1298937600000|30|Scotland|Norway +6597069777438|1296518400000|28|Laos|Liberia +6597069768429|1306886400000|34|Scotland|Honduras +13194139542253|1296518400000|33|Lithuania|Liberia +21990232563680|1306886400000|29|Honduras|Slovakia +30786325578795|1301616000000|31|Liberia|Norway +17592186051356|1298937600000|35|Papua_New_Guinea|Lithuania +26388279072405|1298937600000|33|Norway|Uruguay +24189255819887|1298937600000|30|Mauritania|Scotland +32985348835335|1296518400000|29|Lithuania|Papua_New_Guinea +4398046521107|1306886400000|29|Laos|Norway +4398046511779|1306886400000|28|Slovakia|Norway +19791209308155|1298937600000|33|Uruguay|Laos +30786325578370|1298937600000|28|Uruguay|Liberia +21990232562366|1296518400000|30|Lithuania|Scotland +28587302323855|1298937600000|37|Laos|Bosnia_and_Herzegovina +28587302322565|1298937600000|29|Scotland|Lithuania +30786325580833|1301616000000|33|Liberia|Papua_New_Guinea +4398046518685|1298937600000|42|Bosnia_and_Herzegovina|Slovakia +13194139543486|1298937600000|37|Uruguay|Liberia +768|1296518400000|29|Uruguay|Slovakia +26388279074461|1306886400000|30|Scotland|Uruguay +4398046512568|1301616000000|29|Mauritania|Honduras +4398046517042|1296518400000|31|Uruguay|Honduras +32985348835137|1306886400000|30|Liberia|Uruguay +17592186049742|1298937600000|33|Bosnia_and_Herzegovina|Norway +32985348840202|1306886400000|32|Liberia|Scotland +21990232558484|1301616000000|30|Liberia|Slovakia +17592186049357|1298937600000|29|Norway|Laos +2199023257753|1298937600000|28|Scotland|Laos +2199023257371|1298937600000|28|Papua_New_Guinea|Bosnia_and_Herzegovina +32985348842449|1296518400000|30|Uruguay|Scotland +8796093024620|1291161600000|29|Slovakia|Honduras +28587302325201|1298937600000|30|Laos|Mauritania +28587302329937|1296518400000|29|Laos|Uruguay +32985348844221|1296518400000|29|Liberia|Norway diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_4.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_4.param new file mode 100644 index 000000000000..7f68f2545b32 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_4.param @@ -0,0 +1,99 @@ +personId|startDate|durationDays +21990232559429|1335830400000|37 +24189255814337|1343779200000|29 +2199023261318|1343779200000|29 +13194139543810|1343779200000|29 +15393162790846|1343779200000|29 +26388279069465|1343779200000|29 +10995116288363|1343779200000|29 +28587302324427|1328054400000|33 +17592186052034|1312156800000|30 +28587302328699|1314835200000|52 +13194139543769|1301616000000|34 +30786325580099|1298937600000|51 +8796093027093|1298937600000|28 +32985348834171|1333238400000|30 +10995116284060|1320105600000|36 +28587302325624|1335830400000|46 +26388279073335|1320105600000|31 +24189255815697|1346457600000|34 +4398046513523|1304208000000|61 +17592186055312|1335830400000|34 +17592186049335|1335830400000|39 +28587302330832|1312156800000|69 +24189255817229|1285891200000|35 +17592186051615|1346457600000|29 +32985348835971|1304208000000|36 +4398046513291|1333238400000|42 +19791209304304|1328054400000|28 +17592186053370|1301616000000|37 +32985348844230|1312156800000|32 +6274|1317427200000|42 +19791209308978|1293840000000|56 +26388279071253|1333238400000|51 +6597069768560|1314835200000|61 +28587302328912|1333238400000|50 +32985348838954|1349049600000|36 +4398046518699|1320105600000|37 +1829|1325376000000|30 +26388279076932|1293840000000|28 +26388279074522|1320105600000|36 +8796093032246|1301616000000|36 +8796093031069|1296518400000|30 +19791209301760|1317427200000|43 +30786325588437|1341100800000|35 +15393162790819|1341100800000|35 +26388279069231|1341100800000|35 +6597069770596|1341100800000|35 +15393162795032|1341100800000|35 +8796093028978|1341100800000|35 +6597069773746|1341100800000|35 +13194139542594|1341100800000|35 +10289|1335830400000|33 +21990232558510|1333238400000|30 +24189255811643|1320105600000|37 +2609|1328054400000|32 +8796093028937|1330560000000|29 +8796093032345|1296518400000|28 +8796093028388|1306886400000|57 +13194139543357|1285891200000|40 +6597069773726|1291161600000|32 +30786325586289|1328054400000|37 +4398046512362|1285891200000|55 +21990232555771|1320105600000|28 +30786325578629|1346457600000|42 +28587302327911|1343779200000|31 +35184372095093|1328054400000|31 +30786325578887|1306886400000|92 +15393162790043|1306886400000|31 +4398046511322|1293840000000|41 +26388279073429|1312156800000|40 +2199023265718|1335830400000|31 +19791209306504|1335830400000|53 +30786325583940|1306886400000|41 +32985348842365|1338508800000|31 +24189255811922|1320105600000|30 +8796093025371|1320105600000|30 +2199023264826|1320105600000|30 +26388279068427|1320105600000|30 +26388279068636|1320105600000|30 +15393162791859|1320105600000|30 +28587302326480|1320105600000|30 +30786325582883|1320105600000|30 +19791209306732|1320105600000|30 +6597069774777|1320105600000|30 +19791209310885|1320105600000|30 +10995116286055|1304208000000|42 +4398046518645|1312156800000|48 +13194139536362|1349049600000|39 +15393162789226|1349049600000|46 +30786325581118|1320105600000|51 +8796093027338|1335830400000|32 +13194139540417|1298937600000|29 +32985348837706|1306886400000|29 +4398046520915|1301616000000|35 +30786325588084|1346457600000|38 +17592186046824|1320105600000|28 +32985348834867|1335830400000|31 +6597069775170|1314835200000|35 +28587302325401|1325376000000|39 diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_5.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_5.param new file mode 100644 index 000000000000..cfd371b51acc --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_5.param @@ -0,0 +1,99 @@ +personId|minDate +17592186055119|1348704000000 +8796093030404|1347062400000 +2199023266354|1344556800000 +6597069774475|1346976000000 +21990232560302|1346457600000 +6597069767470|1348617600000 +32985348839760|1346630400000 +28587302331227|1344384000000 +6597069775210|1346889600000 +6597069777205|1345680000000 +6597069768320|1347753600000 +19791209307492|1346889600000 +28587302322515|1345507200000 +8796093025131|1345680000000 +21990232558067|1345852800000 +21990232560628|1344816000000 +32985348837736|1346544000000 +32985348840452|1345852800000 +32985348834961|1348444800000 +17592186047698|1344038400000 +2199023261819|1346889600000 +15393162792270|1346544000000 +24189255821418|1346803200000 +24189255819548|1346803200000 +15393162791355|1346716800000 +30786325578316|1345075200000 +6597069776966|1347580800000 +2199023260581|1345766400000 +21990232560233|1347148800000 +4398046516880|1348531200000 +30786325586428|1344556800000 +28587302325306|1343088000000 +28587302324102|1347840000000 +30786325587163|1348444800000 +2199023258040|1346803200000 +8796093032964|1343779200000 +28587302328025|1345075200000 +4398046514525|1344902400000 +32985348834523|1345075200000 +10995116278636|1344902400000 +21990232566347|1346025600000 +13194139542764|1346457600000 +4398046520770|1347494400000 +17592186051070|1347235200000 +24189255811500|1346544000000 +6597069777346|1347235200000 +17592186052400|1345248000000 +8796093031741|1347148800000 +6597069768934|1344643200000 +30786325578737|1348444800000 +21990232557143|1347148800000 +30786325584591|1347321600000 +4398046517049|1346457600000 +24189255818799|1344729600000 +13194139542115|1344816000000 +28587302330379|1344902400000 +10995116282665|1347494400000 +4398046515300|1346630400000 +30786325586109|1346630400000 +32985348842142|1346025600000 +17592186052168|1346630400000 +10995116285640|1345334400000 +6597069777539|1346889600000 +6597069777438|1344988800000 +13194139542253|1343865600000 +6597069768429|1343865600000 +21990232563680|1345420800000 +30786325578795|1346803200000 +17592186051356|1347580800000 +26388279072405|1344729600000 +32985348835335|1347148800000 +24189255819887|1344729600000 +4398046521107|1346025600000 +4398046511779|1346457600000 +19791209308155|1345334400000 +30786325578370|1346025600000 +21990232562366|1347062400000 +28587302323855|1347840000000 +28587302322565|1345852800000 +30786325580833|1343001600000 +4398046518685|1348444800000 +13194139543486|1347753600000 +768|1345507200000 +26388279074461|1345420800000 +4398046517042|1346976000000 +4398046512568|1346025600000 +32985348835137|1345420800000 +17592186049742|1347148800000 +32985348840202|1344902400000 +21990232558484|1344643200000 +17592186049357|1344988800000 +2199023257753|1345161600000 +2199023257371|1346630400000 +32985348842449|1344643200000 +8796093024620|1345161600000 +28587302325201|1346457600000 +28587302329937|1345766400000 +32985348844221|1345334400000 diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_6.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_6.param new file mode 100644 index 000000000000..bf91ed608511 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_6.param @@ -0,0 +1,99 @@ +personId|tagName +30786325583618|Angola +17592186053645|Eric_Clapton +4398046512818|Sergei_Prokofiev +30786325578911|Sergei_Eisenstein +6597069766659|Bermuda +26388279075172|Batman +19791209309138|Austria-Hungary +24189255822062|Robert_De_Niro +10995116287298|Confederate_States_of_America +30786325578963|Agatha_Christie +21990232557314|Michel_Foucault +2199023264241|Niccolò_Machiavelli +15393162790365|Samuel_Taylor_Coleridge +6597069769100|Monaco +6597069773997|James_Stewart +17592186045019|Robin_Williams +30786325587888|Jonathan_Swift +17592186045360|Prince_Philip,_Duke_of_Edinburgh +10995116280597|Louis_Armstrong +15393162790770|Louis_XVI_of_France +9782|Lewis_Carroll +8796093026933|Bono +28587302322817|Henry_Kissinger +13194139542834|F._Scott_Fitzgerald +17592186051169|H._P._Lovecraft +4398046513464|Jules_Verne +13194139535318|Napoleon_III +4398046517060|Mustafa_Kemal_Atatürk +13194139538142|Graham_Greene +19791209300404|Humphrey_Bogart +19791209300882|Robert_E._Lee +6597069770264|Walter_Scott +2199023260581|Alice_Cooper +4398046516880|Paraguay +13194139542764|Cary_Grant +768|Democratic_Republic_of_the_Congo +26388279074461|Henrik_Ibsen +32985348835137|British_Empire +8796093032135|Anton_Chekhov +26388279074072|Ralph_Waldo_Emerson +35184372095658|James_Madison +13194139539724|East_Germany +15393162789349|El_Salvador +17592186054325|William_S._Burroughs +21990232562091|Harrison_Ford +30786325587843|Kuwait +30786325587276|Raphael +17592186047707|James_II_of_England +30786325585356|Marlon_Brando +13194139542130|William_III_of_England +15393162795979|Dalai_Lama +24189255811315|Nepal +15393162791567|Herbert_Hoover +32985348844096|Rembrandt +13194139541713|Leon_Trotsky +17592186051933|Billie_Holiday +13194139536557|Byzantine_Empire +21990232565814|Juan_Carlos_I_of_Spain +26388279068701|Henry_James +19791209306192|Cole_Porter +30786325581911|George_Lucas +2199023256125|Gamal_Abdel_Nasser +4398046518845|Leonard_Bernstein +19791209300153|Robert_Louis_Stevenson +4398046521818|Superman +17592186053195|Pope_Paul_VI +13194139542945|Katarina_Srebotnik +32985348836952|Dustin_Hoffman +6597069776117|Hirohito +2199023255940|Ringo_Starr +28587302327943|J._K._Rowling +24189255820909|Sean_Connery +26388279077068|Yasser_Arafat +10995116283556|Calvin_Coolidge +15393162796757|Federico_Fellini +17592186052877|John_Kerry +2199023265470|Jack_Kerouac +24189255814005|Martin_Scorsese +15393162789189|Paul_Newman +32985348842593|Fiji +13194139535090|D._H._Lawrence +4398046517753|John_Steinbeck +24189255813156|Herodotus +4398046514583|Honduras +17592186049987|Walt_Whitman +10995116283196|Barbados +19791209308316|Azerbaijan +35184372094078|Monty_Python +19791209302972|Henry_Wadsworth_Longfellow +15393162790921|George_S._Patton +32985348837345|Bette_Davis +21990232562252|Jimi_Hendrix +2199023257044|Arthur_C._Clarke +13194139533606|Hermann_Göring +10995116281419|Carl_Jung +17592186050886|Julie_Andrews +21990232556833|Vatican_City +6597069767674|Robert_Burns diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_7.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_7.param new file mode 100644 index 000000000000..1b8afbc571ca --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_7.param @@ -0,0 +1,99 @@ +personId +17592186053137 +6597069777240 +2199023262543 +4398046513018 +19791209302403 +24189255819727 +21990232563274 +8796093029267 +15393162795439 +10995116285387 +2199023259756 +4398046519372 +26388279074032 +8796093030364 +24189255817517 +15393162790796 +19791209304051 +13194139535025 +13194139543218 +13194139535632 +15393162796861 +30786325581208 +8796093029854 +21990232558990 +17592186046293 +8796093025074 +4139 +2783 +7725 +26388279072730 +19791209303491 +10995116284808 +30786325585162 +4398046512826 +24189255821300 +30786325578932 +8796093029002 +26388279076187 +28587302332758 +24189255811663 +30786325583918 +17592186052746 +26388279067534 +13194139537478 +26388279070362 +9116 +24189255814889 +24189255820751 +8796093029365 +8796093031506 +26388279068863 +15393162798801 +32985348834375 +26388279075417 +26388279075075 +13194139540684 +17592186052552 +26388279074356 +13194139538575 +24189255817294 +28587302326035 +17592186048261 +32985348841922 +4555 +4398046520894 +4398046515585 +17592186052272 +6597069777302 +10995116283243 +28587302330430 +28587302327496 +19791209304023 +15393162799074 +8796093023907 +6597069767242 +6597069771199 +24189255815044 +10995116280854 +30786325578585 +6597069774643 +2199023256816 +6597069773262 +19791209310429 +24189255811566 +17592186054401 +6597069771886 +8796093030527 +6597069777107 +30786325588227 +5274 +28587302322537 +32985348835374 +24189255817201 +30786325579845 +2199023261462 +21990232560132 +19791209303550 +4398046520887 diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_8.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_8.param new file mode 100644 index 000000000000..7aa56a9cc573 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_8.param @@ -0,0 +1,99 @@ +personId +24189255818757 +30786325585496 +13194139543594 +4398046519908 +9479 +6597069772289 +4398046514115 +2199023262863 +13194139541453 +15393162798825 +6597069767013 +8796093024648 +15393162798070 +10995116286325 +17592186049188 +26388279069006 +21990232563585 +19791209300159 +26388279071726 +21990232561209 +21990232565617 +24189255816625 +26388279067147 +26388279076341 +24189255812103 +26388279071839 +28587302324631 +24189255821959 +28587302330273 +21990232565453 +28587302326271 +26388279070248 +24189255819511 +30786325582299 +24189255811922 +21990232565912 +26388279077232 +28587302332677 +30786325583618 +28587302328958 +28587302325895 +28587302332226 +28587302330066 +30786325579117 +28587302332037 +30786325583551 +32985348839699 +28587302324619 +32985348836997 +32985348836635 +32985348838177 +32985348834886 +30786325580376 +26388279069766 +24189255818646 +4398046520281 +32985348833314 +28587302323238 +28587302323276 +30786325582236 +30786325584895 +28587302332391 +21990232556376 +21990232557705 +21990232558064 +15393162793342 +26388279072962 +30786325584456 +32985348840215 +8796093029402 +15393162797017 +28587302331366 +15393162798250 +30786325577834 +13194139534376 +26388279068783 +10995116281483 +24189255815047 +32985348838449 +15393162794170 +4398046516663 +28587302328175 +28587302328644 +30786325585356 +10995116286636 +8796093032185 +30786325587843 +8796093032875 +13194139534816 +28587302325012 +21990232559158 +13194139537109 +32985348839308 +32985348839337 +32985348839710 +30786325584565 +32985348840866 +10995116288720 diff --git a/research/query_service/benchmark/data/substitution_parameters/ldbc_query_9.param b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_9.param new file mode 100644 index 000000000000..a52203911ebf --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/ldbc_query_9.param @@ -0,0 +1,99 @@ +personId|maxDate +13194139542834|1324080000000 +15393162790770|1329350400000 +17592186045019|1332720000000 +13194139535318|1323302400000 +8796093026933|1329004800000 +28587302322817|1328745600000 +17592186051169|1337904000000 +4398046513464|1335052800000 +13194139542764|1341619200000 +6597069770264|1330646400000 +9782|1339545600000 +19791209300882|1344384000000 +35184372095658|1317513600000 +8796093032135|1317686400000 +4398046517060|1333411200000 +13194139538142|1333756800000 +2199023260581|1341360000000 +19791209300404|1340582400000 +13194139539724|1322956800000 +26388279074461|1348099200000 +4398046516880|1305590400000 +21990232562091|1346630400000 +17592186054325|1336003200000 +32985348835137|1346716800000 +13194139542130|1346716800000 +17592186051933|1343174400000 +15393162789349|1333411200000 +26388279068701|1347235200000 +768|1340236800000 +30786325587276|1323129600000 +15393162795979|1333324800000 +26388279074072|1331510400000 +21990232565814|1347235200000 +15393162791567|1342742400000 +24189255811315|1340496000000 +17592186047707|1336608000000 +30786325587843|1338595200000 +19791209306192|1350086400000 +13194139541713|1347062400000 +30786325585356|1354060800000 +30786325581911|1351728000000 +19791209300153|1345593600000 +13194139536557|1354060800000 +32985348844096|1340236800000 +2199023255940|1340496000000 +13194139542945|1354060800000 +28587302327943|1347235200000 +4398046517753|1349049600000 +4398046518845|1354060800000 +32985348836952|1338854400000 +17592186052877|1354060800000 +4398046521818|1354060800000 +17592186053195|1335916800000 +32985348842593|1348444800000 +24189255813156|1351123200000 +6597069776117|1347321600000 +15393162796757|1341273600000 +2199023256125|1353283200000 +24189255820909|1333411200000 +24189255814005|1354060800000 +2199023265470|1354060800000 +15393162789189|1354060800000 +13194139535090|1341446400000 +19791209308316|1354060800000 +10995116283196|1354060800000 +4398046514583|1330041600000 +10995116283556|1334361600000 +17592186049987|1345852800000 +26388279077068|1354060800000 +15393162790921|1354060800000 +2199023257044|1354060800000 +19791209302972|1344729600000 +13194139533606|1347062400000 +21990232562252|1350000000000 +17592186050886|1354060800000 +35184372094078|1354060800000 +6597069767674|1354060800000 +10995116281419|1321574400000 +21990232556833|1354060800000 +15393162792273|1342828800000 +32985348837345|1354060800000 +10995116284769|1354060800000 +2199023258980|1354060800000 +30786325577834|1354060800000 +13194139540753|1338595200000 +8796093022963|1354060800000 +17592186052613|1354060800000 +15393162796638|1348617600000 +8796093026900|1354060800000 +28587302329064|1337817600000 +3755|1354060800000 +2199023255953|1337558400000 +24189255818885|1354060800000 +2199023259917|1332115200000 +21990232563675|1354060800000 +24189255811455|1354060800000 +32985348842860|1354060800000 +10995116287061|1354060800000 diff --git a/research/query_service/benchmark/data/substitution_parameters/subtask_query.param b/research/query_service/benchmark/data/substitution_parameters/subtask_query.param new file mode 100644 index 000000000000..1690500a6511 --- /dev/null +++ b/research/query_service/benchmark/data/substitution_parameters/subtask_query.param @@ -0,0 +1,11 @@ +vertexId +72068589154212744 +144117249660157961 +504403158265506123 +504403158265498631 +504403158265497622 +504403158265496359 +504403158265496331 +72090579386766311 +72064191107705176 +504403158265497323 \ No newline at end of file diff --git a/research/query_service/benchmark/pom.xml b/research/query_service/benchmark/pom.xml new file mode 100644 index 000000000000..d17653197eba --- /dev/null +++ b/research/query_service/benchmark/pom.xml @@ -0,0 +1,47 @@ + + + 4.0.0 + + com.alibaba.graphscope.gaia + gaia-benchmark + 0.0.1-SNAPSHOT + + + + org.apache.tinkerpop + gremlin-driver + 3.4.2 + + + + + + + maven-assembly-plugin + + assembly.xml + + + + make-assembly + package + + single + + + + + + maven-compiler-plugin + 3.1 + + 8 + 8 + + + + + + \ No newline at end of file diff --git a/research/query_service/benchmark/queries/1_hop_query.gremlin b/research/query_service/benchmark/queries/1_hop_query.gremlin new file mode 100644 index 000000000000..7c9f69775297 --- /dev/null +++ b/research/query_service/benchmark/queries/1_hop_query.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().count() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/2_hop_query.gremlin b/research/query_service/benchmark/queries/2_hop_query.gremlin new file mode 100644 index 000000000000..214bf242436a --- /dev/null +++ b/research/query_service/benchmark/queries/2_hop_query.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().out().count() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/3_hop_query.gremlin b/research/query_service/benchmark/queries/3_hop_query.gremlin new file mode 100644 index 000000000000..a7f83b949fad --- /dev/null +++ b/research/query_service/benchmark/queries/3_hop_query.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().out().out().count() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/4_hop_query.gremlin b/research/query_service/benchmark/queries/4_hop_query.gremlin new file mode 100644 index 000000000000..c7f35ea73910 --- /dev/null +++ b/research/query_service/benchmark/queries/4_hop_query.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().out().out().out().count() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/custom_constant_query_1.gremlin b/research/query_service/benchmark/queries/custom_constant_query_1.gremlin new file mode 100644 index 000000000000..6df222b655cc --- /dev/null +++ b/research/query_service/benchmark/queries/custom_constant_query_1.gremlin @@ -0,0 +1 @@ +g.V().limit(1) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/custom_constant_query_2.gremlin b/research/query_service/benchmark/queries/custom_constant_query_2.gremlin new file mode 100644 index 000000000000..b5cfc3020a07 --- /dev/null +++ b/research/query_service/benchmark/queries/custom_constant_query_2.gremlin @@ -0,0 +1 @@ +g.V().limit(10) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/custom_query_1.gremlin b/research/query_service/benchmark/queries/custom_query_1.gremlin new file mode 100644 index 000000000000..237d20240ded --- /dev/null +++ b/research/query_service/benchmark/queries/custom_query_1.gremlin @@ -0,0 +1 @@ +g.V($vertexId).both().limit(1) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/custom_query_2.gremlin b/research/query_service/benchmark/queries/custom_query_2.gremlin new file mode 100644 index 000000000000..7c9f69775297 --- /dev/null +++ b/research/query_service/benchmark/queries/custom_query_2.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().count() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/early_stop_query.gremlin b/research/query_service/benchmark/queries/early_stop_query.gremlin new file mode 100644 index 000000000000..6946b62f56ce --- /dev/null +++ b/research/query_service/benchmark/queries/early_stop_query.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().out().limit(1) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_1.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_1.gremlin new file mode 100644 index 000000000000..95d6151e9258 --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_1.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('KNOWS').union(identity(),both('KNOWS').union(identity(),both('KNOWS'))).dedup().has('id', neq($personId)).has('firstName',eq('$firstName')).as('a').path().count(local).as('b').select('a').order().by(select('b'),asc).by('lastName').by('id').limit(20).select('a', 'b') \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_11.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_11.gremlin new file mode 100644 index 000000000000..21910e4dca7b --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_11.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).as('root').both('KNOWS').union(identity(), both('KNOWS')).dedup().has('id', neq($personId)).as('friends').outE('WORKAT').has('workFrom', lt($workFromYear)).as('startWork').values('workFrom').as('works').select('startWork').inV().as('comp').values('name').as('orgname').select('comp').out('ISLOCATEDIN').has('name', '$countryName').select('friends').order().by(select('works'), asc).by('id', asc).by(select('orgname'), desc).limit(10).select('friends', 'orgname', 'works').by(valueMap('id', 'firstName', 'lastName')).by().by() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_12.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_12.gremlin new file mode 100644 index 000000000000..73bceca60662 --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_12.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('KNOWS').as('friend').in('HASCREATOR').hasLabel('COMMENT').filter(out('REPLYOF').hasLabel('POST').out('HASTAG').out('HASTYPE').has('name',eq('$tagClassName'))).select('friend').groupCount().unfold().order().by(values, desc).by(select(keys).values('id'), asc).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_2.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_2.gremlin new file mode 100644 index 000000000000..d9eedd777f2a --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_2.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('KNOWS').as('p').in('HASCREATOR').has('creationDate',lte($maxDate)).order().by('creationDate',desc).by('id',asc).limit(20).as('m').select('p', 'm').by(valueMap('id', 'firstName', 'lastName')).by(valueMap('id', 'imageFile', 'creationDate', 'content')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_3.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_3.gremlin new file mode 100644 index 000000000000..84a73c74539c --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_3.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('KNOWS').union(identity(), both('KNOWS')).dedup().filter(__.out('ISLOCATEDIN').has('name',neq('$countryXName').and(neq('$countryYName')))).as('friend').filter(__.in('HASCREATOR').has('creationDate',inside($startDate,$endDate)).out('ISLOCATEDIN').has('name', eq('$countryXName').or(eq('$countryYName'))).values("name").dedup().count().unfold().is(2)).group().by().by(__.in('HASCREATOR').has('creationDate',inside($startDate,$endDate)).out('ISLOCATEDIN').has('name', '$countryXName').count()).unfold().order().by(select(values), desc).by(select(keys).values('id')).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_4.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_4.gremlin new file mode 100644 index 000000000000..465ae9d42d90 --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_4.gremlin @@ -0,0 +1 @@ +g.V().has('PERSON','id',$personId).out('KNOWS').fold().store('friends').unfold().in('HASCREATOR').hasLabel('POST').has('creationDate',inside($startDate,$endDate)).out('HASTAG').filter(__.in('HASTAG').hasLabel('POST').has('creationDate', lt($startDate)).out('HASCREATOR').where(within('friends')).count().is(0)).groupCount().unfold().order().by(select(values),decr).by(select(keys).values('name')).limit(10) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_5.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_5.gremlin new file mode 100644 index 000000000000..91c4de48025c --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_5.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('KNOWS').union(identity(), both('KNOWS')).dedup().as('p').inE('HASMEMBER').has('joinDate', gt($minDate)).outV().group().by().by(out('CONTAINEROF').out('HASCREATOR').where(eq('p')).count()).unfold().order().by(select(values), desc).by(select(keys).values('id'), asc).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_5.gremlin.nosubtask b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_5.gremlin.nosubtask new file mode 100644 index 000000000000..71d8a4cee960 --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_5.gremlin.nosubtask @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('KNOWS').union(identity(), both('KNOWS')).dedup().as('p').inE('HASMEMBER').has('joinDate', gt($minDate)).outV().as('forum').out('CONTAINEROF').out('HASCREATOR').where(eq('p')).select('forum').groupCount().unfold().order().by(select(values),desc).by(select(keys).values('id'), asc).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_6.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_6.gremlin new file mode 100644 index 000000000000..ada066c2fe0d --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_6.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('KNOWS').union(identity(), both('KNOWS')).dedup().has('id', neq($personId)).in('HASCREATOR').hasLabel('POST').filter(out('HASTAG').has('name', eq('$tagName'))).out('HASTAG').has('name', neq('$tagName')).groupCount().unfold().order().by(select(values), desc).by(select(keys).values('name'), asc).limit(10) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_6.gremlin.nosubtask b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_6.gremlin.nosubtask new file mode 100644 index 000000000000..e8addcf9185f --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_6.gremlin.nosubtask @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('KNOWS').union(identity(), both('KNOWS')).dedup().has('id', neq($personId)).in('HASCREATOR').hasLabel('POST').as('_t').out('HASTAG').has('name', eq('$tagName')).select('_t').dedup().out('HASTAG').has('name', neq('$tagName')).groupCount().unfold().order().by(select(values), desc).by(select(keys).values('name'), asc).limit(10) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_7.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_7.gremlin new file mode 100644 index 000000000000..f0e09242ae7a --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_7.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).in('HASCREATOR').as('message').inE('LIKES').as('like').values('creationDate').as('likedate').select('like').outV().as('liker').order().by(select('likedate'), desc).by('id', asc).limit(20).select('message', 'likedate', 'liker').by(valueMap('id', 'content', 'imageFile')).by().by(valueMap('id', 'firstName', 'lastName')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_8.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_8.gremlin new file mode 100644 index 000000000000..0ef7d7463969 --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_8.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).in('HASCREATOR').in('REPLYOF').hasLabel('COMMENT').as('comment').order().by('creationDate', desc).by('id', asc).limit(20).out('HASCREATOR').as('commenter').select('commenter','comment').by(valueMap('id', 'firstName', 'lastName')).by(valueMap('creationDate', 'id', 'content')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_9.gremlin b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_9.gremlin new file mode 100644 index 000000000000..7150ab957034 --- /dev/null +++ b/research/query_service/benchmark/queries/gaiax_ldbc/ldbc_query_9.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('KNOWS').union(identity(), both('KNOWS')).dedup().has('id', neq($personId)).as('friends').in('HASCREATOR').hasLabel('POST').has('creationDate',lt($maxDate)).as('post').order().by('creationDate', desc).by('id', asc).limit(20).select('friends','post').by(valueMap('id', 'firstName', 'lastName')).by(valueMap('id', 'content', 'imageFile', 'creationDate')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_1.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_1.gremlin new file mode 100644 index 000000000000..06432b97ca99 --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_1.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('1..4', 'KNOWS').as('p').inV().has('id', neq($personId)).has('firstName',eq('$firstName')).as('a').dedup().limit(20).order().by(select('p').by('~len'), asc).by(select('a').by('id')).by(select('a').by('lastName')).select('a', 'p').by(valueMap('id', 'firstName', 'lastName')).by('~len') \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_11.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_11.gremlin new file mode 100644 index 000000000000..7f56a9bc2855 --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_11.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).as('root').both('1..3', 'KNOWS').inV().dedup().has('id', neq($personId)).as('friends').outE('WORKAT').has('workFrom', lt($workFromYear)).as('startWork').values('workFrom').as('works').select('startWork').inV().as('comp').values('name').as('orgname').select('comp').out('ISLOCATEDIN').has('name', '$countryName').select('friends').order().by(select('works'), asc).by('id', asc).by(select('orgname'), desc).limit(10).select('friends', 'orgname', 'works').by(valueMap('id', 'firstName', 'lastName')).by().by() \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_12.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_12.gremlin new file mode 100644 index 000000000000..0b7c8f72e34d --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_12.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('KNOWS').as('friend').in('HASCREATOR').hasLabel('COMMENT').where(out('REPLYOF').hasLabel('POST').out('HASTAG').out('HASTYPE').has('name',eq('$tagClassName'))).select('friend').groupCount().order().by(select(values), desc).by(select(keys).values('id'), asc).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_2.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_2.gremlin new file mode 100644 index 000000000000..d9eedd777f2a --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_2.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).both('KNOWS').as('p').in('HASCREATOR').has('creationDate',lte($maxDate)).order().by('creationDate',desc).by('id',asc).limit(20).as('m').select('p', 'm').by(valueMap('id', 'firstName', 'lastName')).by(valueMap('id', 'imageFile', 'creationDate', 'content')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_3.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_3.gremlin new file mode 100644 index 000000000000..87be066b2be0 --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_3.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).union(both('KNOWS'), both('KNOWS').both('KNOWS')).dedup().where(__.out('ISLOCATEDIN').has('name',without('$countryXName','$countryYName'))).where(__.in('HASCREATOR').has('creationDate',gt($startDate).and(lt($endDate))).out('ISLOCATEDIN').has('name', eq('$countryXName').or(eq('$countryYName'))).values("name").dedup().count().is(2)).order().by(__.in('HASCREATOR').has('creationDate',gt($startDate).and(lt($endDate))).out('ISLOCATEDIN').has('name', '$countryXName').count(), desc).by('id',asc).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_4.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_4.gremlin new file mode 100644 index 000000000000..80520b602700 --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_4.gremlin @@ -0,0 +1 @@ +g.V().hasLabel("PERSON").has("id", $personId).as('person').both("KNOWS").in("HASCREATOR").hasLabel("POST").as('post').has("creationDate", gte($startDate).and(lt($endDate))).out("HASTAG").as('tag').select("person").not(both("KNOWS").in("HASCREATOR").hasLabel("POST").has("creationDate", lt($startDate)).out("HASTAG").where(eq('tag'))).select("tag").groupCount().order().by(select(values), Order.desc).by(select(keys).values('name'), Order.asc).limit(10).as("ordered").select(keys).values("name").as("tagName").select("ordered").select(values).as("postCount").select("tagName","postCount") \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_5.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_5.gremlin new file mode 100644 index 000000000000..54f9b22c0aae --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_5.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('1..3', 'KNOWS').inV().dedup().as('p').inE('HASMEMBER').has('joinDate', gt($minDate)).outV().as('forum').out('CONTAINEROF').hasLabel('POST').out('HASCREATOR').where(eq('p')).select('forum').groupCount().order().by(select(values),desc).by(select(keys).values('id'), asc).limit(20) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_6.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_6.gremlin new file mode 100644 index 000000000000..5ab0986e3f1b --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_6.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('1..3', 'KNOWS').inV().dedup().has('id', neq($personId)).in('HASCREATOR').hasLabel('POST').where(out('HASTAG').has('name', eq('$tagName'))).out('HASTAG').has('name', neq('$tagName')).groupCount().order().by(select(values), desc).by(select(keys).values('name'), asc).limit(10) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_6_no_subtask.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_6_no_subtask.gremlin new file mode 100644 index 000000000000..18cb38c5129c --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_6_no_subtask.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).union(both('KNOWS'), both('KNOWS').both('KNOWS')).dedup().has('id', neq($personId)).in('HASCREATOR').hasLabel('POST').as('_t').out('HASTAG').has('name', eq('$tagName')).select('_t').dedup().out('HASTAG').has('name', neq('$tagName')).groupCount().order().by(select(values), desc).by(select(keys).values('name'), asc).limit(10) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_7.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_7.gremlin new file mode 100644 index 000000000000..f0e09242ae7a --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_7.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id',$personId).in('HASCREATOR').as('message').inE('LIKES').as('like').values('creationDate').as('likedate').select('like').outV().as('liker').order().by(select('likedate'), desc).by('id', asc).limit(20).select('message', 'likedate', 'liker').by(valueMap('id', 'content', 'imageFile')).by().by(valueMap('id', 'firstName', 'lastName')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_8.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_8.gremlin new file mode 100644 index 000000000000..0ef7d7463969 --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_8.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).in('HASCREATOR').in('REPLYOF').hasLabel('COMMENT').as('comment').order().by('creationDate', desc).by('id', asc).limit(20).out('HASCREATOR').as('commenter').select('commenter','comment').by(valueMap('id', 'firstName', 'lastName')).by(valueMap('creationDate', 'id', 'content')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_9.gremlin b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_9.gremlin new file mode 100644 index 000000000000..c571fbae1b06 --- /dev/null +++ b/research/query_service/benchmark/queries/ir_ldbc/ldbc_query_9.gremlin @@ -0,0 +1 @@ +g.V().hasLabel('PERSON').has('id', $personId).both('1..3', 'KNOWS').inV().dedup().has('id', neq($personId)).as('friends').in('HASCREATOR').has('creationDate',lt($maxDate)).as('post').order().by('creationDate', desc).by('id', asc).limit(20).select('friends','post').by(valueMap('id', 'firstName', 'lastName')).by(valueMap('id', 'content', 'imageFile', 'creationDate')) \ No newline at end of file diff --git a/research/query_service/benchmark/queries/subtask_query.gremlin b/research/query_service/benchmark/queries/subtask_query.gremlin new file mode 100644 index 000000000000..e3df766b1ef1 --- /dev/null +++ b/research/query_service/benchmark/queries/subtask_query.gremlin @@ -0,0 +1 @@ +g.V($vertexId).out().as('a').select('a').by(out().count()) \ No newline at end of file diff --git a/research/query_service/benchmark/shell/benchmark.sh b/research/query_service/benchmark/shell/benchmark.sh new file mode 100755 index 000000000000..297231156814 --- /dev/null +++ b/research/query_service/benchmark/shell/benchmark.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Copyright 2020 Alibaba Group Holding Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CURR_DIR=$(pwd) + +BIN_DIR=$(dirname $0) +cd $BIN_DIR/../ +BASE_DIR=$(pwd) + +CONF_DIR=$BASE_DIR/config/interactive-benchmark.properties +LIB_DIR=$BASE_DIR/lib + +JAVA_CLASSPATH="." +for libfile in $LIB_DIR; do + JAVA_CLASSPATH=$JAVA_CLASSPATH":$LIB_DIR/$libfile" +done + +java -cp $JAVA_CLASSPATH com.alibaba.graphscope.gaia.benchmark.InteractiveBenchmark $CONF_DIR + +cd $CURR_DIR diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/benchmark/InteractiveBenchmark.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/benchmark/InteractiveBenchmark.java new file mode 100644 index 000000000000..ceea2695a6a3 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/benchmark/InteractiveBenchmark.java @@ -0,0 +1,148 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.benchmark; + +import com.alibaba.graphscope.gaia.common.CommonQuery; +import com.alibaba.graphscope.gaia.common.Configuration; +import com.alibaba.graphscope.gaia.utils.PropertyUtil; +import com.alibaba.graphscope.gaia.utils.QueryUtil; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.driver.Client; +import org.apache.tinkerpop.gremlin.driver.Cluster; +import org.apache.tinkerpop.gremlin.driver.MessageSerializer; +import org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +public class InteractiveBenchmark { + public static void main(String[] args) throws Exception { + if (args.length != 1) { + System.out.println("Error, Usage: "); + return; + } + + Properties properties = PropertyUtil.getProperties(args[0], false); + Configuration configuration = new Configuration(properties); + + String gremlinServerEndpoint = + configuration.getString(Configuration.GREMLIN_SERVER_ENDPOINT); + int threadCount = configuration.getInt(Configuration.THREAD_COUNT, 1); + int warmUpCount = configuration.getInt(Configuration.WARMUP_EVERY_QUERY, 0); + int operationCount = configuration.getInt(Configuration.OPERATION_COUNT_EVERY_QUERY, 10); + boolean printQueryName = configuration.getBoolean(Configuration.PRINT_QUERY_NAME, true); + boolean printQueryResult = configuration.getBoolean(Configuration.PRINT_QUERY_RESULT, true); + String username = configuration.getString(Configuration.GREMLIN_USERNAME, ""); + String password = configuration.getString(Configuration.GREMLIN_PASSWORD, ""); + + List ldbcQueryList = QueryUtil.initQueryList(configuration); + + AtomicInteger atomicQueryCount = new AtomicInteger(operationCount * threadCount); + AtomicInteger atomicParameterIndex = new AtomicInteger(0); + + class MyRunnable implements Runnable { + private Client client; + + public MyRunnable(String endpoint, String username, String password) { + String[] address = endpoint.split(":"); + try { + Cluster.Builder cluster = + Cluster.build() + .addContactPoint(address[0]) + .port(Integer.parseInt(address[1])) + .serializer(initializeSerialize()); + if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) { + cluster.credentials(username, password); + } + client = cluster.create().connect(); + + System.out.println("Connect success."); + } catch (Exception e) { + System.err.println("Connect failure, caused by : " + e); + throw new RuntimeException(e); + } + } + + @Override + public void run() { + for (int index = 0; index < warmUpCount; index++) { + System.out.println("Begin Warm up ...."); + CommonQuery commonQuery = ldbcQueryList.get(index % ldbcQueryList.size()); + HashMap queryParameter = commonQuery.getSingleParameter(index); + + commonQuery.processGremlinQuery( + client, queryParameter, printQueryResult, printQueryName); + } + System.out.println("Begin standard test..."); + while (true) { + int currentValue = atomicQueryCount.getAndDecrement(); + if (currentValue > 0) { + int queryIndex = currentValue % ldbcQueryList.size(); + CommonQuery commonQuery = + ldbcQueryList.get(queryIndex % ldbcQueryList.size()); + int parameterIndex = 0; + if (queryIndex == 0) { + parameterIndex = atomicParameterIndex.getAndIncrement(); + } else { + parameterIndex = atomicParameterIndex.get(); + } + HashMap queryParameter = + commonQuery.getSingleParameter(parameterIndex); + commonQuery.processGremlinQuery( + client, queryParameter, printQueryResult, printQueryName); + } else { + break; + } + } + client.close(); + } + } + + ExecutorService threadPool = Executors.newFixedThreadPool(threadCount); + + long startTime = System.currentTimeMillis(); + for (int i = 0; i < threadCount; i++) { + threadPool.submit(new MyRunnable(gremlinServerEndpoint, username, password)); + } + + threadPool.shutdown(); + + while (true) { + if (threadPool.isTerminated()) { + long endTime = System.currentTimeMillis(); + long executeTime = endTime - startTime; + long queryCount = operationCount * threadCount; + float qps = (float) queryCount / executeTime * 1000; + System.out.println( + "query count: " + + queryCount + + "; execute time(ms): " + + executeTime + + "; qps: " + + qps); + System.exit(0); + } + Thread.sleep(10); + } + } + + private static MessageSerializer initializeSerialize() { + return new GryoMessageSerializerV1d0(); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java new file mode 100644 index 000000000000..c4c36d6db874 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/AbstractLdbcWithSubQuery.java @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.tinkerpop.gremlin.driver.Client; +import org.apache.tinkerpop.gremlin.driver.Result; +import org.apache.tinkerpop.gremlin.driver.ResultSet; + +import java.util.HashMap; + +public abstract class AbstractLdbcWithSubQuery extends CommonQuery { + + public AbstractLdbcWithSubQuery(String queryName, String queryFile, String parameterFile) + throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + public void processGremlinQuery( + Client client, + HashMap singleParameter, + boolean printResult, + boolean printQuery) { + + try { + String gremlinQuery = generateGremlinQuery(singleParameter, queryPattern); + + long startTime = System.currentTimeMillis(); + ResultSet resultSet = client.submit(gremlinQuery); + int resultCount = 0; + String resultStr = ""; + + Pair resultPair = processResult(resultSet); + resultCount += resultPair.getLeft(); + if (printResult && !resultPair.getRight().isEmpty()) { + resultStr = String.format("%s%s", resultStr, resultPair.getValue()); + } + + long endTime = System.currentTimeMillis(); + long executeTime = endTime - startTime; + if (printQuery) { + String printInfo = + String.format( + "QueryName[%s], Parameter[%s], ResultCount[%d], ExecuteTimeMS[%d].", + queryName, singleParameter.toString(), resultCount, executeTime); + if (printResult) { + printInfo = String.format("%s Result: { %s }", printInfo, resultStr); + } + System.out.println(printInfo); + } + } catch (Exception e) { + System.out.println( + String.format( + "Timeout or failed: QueryName[%s], Parameter[%s].", + queryName, singleParameter.toString())); + e.printStackTrace(); + } + } + + abstract String buildSubQuery(Result result, HashMap singleParameter); +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/CommonQuery.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/CommonQuery.java new file mode 100644 index 000000000000..6de07e6f531d --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/CommonQuery.java @@ -0,0 +1,171 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.tinkerpop.gremlin.driver.Client; +import org.apache.tinkerpop.gremlin.driver.Result; +import org.apache.tinkerpop.gremlin.driver.ResultSet; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; + +public class CommonQuery { + String queryName; + String queryPattern; + private ArrayList> parameters; + + public CommonQuery(String queryName, String queryFile) throws Exception { + this.queryName = queryName; + this.queryPattern = getGremlinQueryPattern(queryFile); + } + + public CommonQuery(String queryName, String queryFile, String parameterFile) throws Exception { + this.queryName = queryName; + this.queryPattern = getGremlinQueryPattern(queryFile); + this.parameters = getParameters(parameterFile); + } + + public HashMap getSingleParameter(int index) { + return parameters.get(index % parameters.size()); + } + + public void processGremlinQuery( + Client client, + HashMap singleParameter, + boolean printResult, + boolean printQuery) { + try { + String gremlinQuery = generateGremlinQuery(singleParameter, queryPattern); + + long startTime = System.currentTimeMillis(); + ResultSet resultSet = client.submit(gremlinQuery); + Pair result = processResult(resultSet); + long endTime = System.currentTimeMillis(); + long executeTime = endTime - startTime; + if (printQuery) { + String printInfo = + String.format( + "QueryName[%s], Parameter[%s], ResultCount[%d], ExecuteTimeMS[%d].", + queryName, + singleParameter.toString(), + result.getLeft(), + executeTime); + if (printResult) { + printInfo = String.format("%s Result: { %s }", printInfo, result.getRight()); + } + System.out.println(printInfo); + } + + } catch (Exception e) { + System.out.println( + String.format( + "Timeout or failed: QueryName[%s], Parameter[%s].", + queryName, singleParameter.toString())); + e.printStackTrace(); + } + } + + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } + + Pair processResult(ResultSet resultSet) { + Iterator iterator = resultSet.iterator(); + int count = 0; + String result = ""; + while (iterator.hasNext()) { + count += 1; + result = String.format("%s\n%s", result, iterator.next().toString()); + } + return Pair.of(count, result); + } + + private static String getGremlinQueryPattern(String gremlinQueryPath) throws Exception { + FileInputStream fileInputStream = new FileInputStream(gremlinQueryPath); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream)); + return bufferedReader.readLine(); + } + + private static ArrayList> getParameters(String parameterFilePath) + throws Exception { + ArrayList> parameters = new ArrayList<>(); + FileInputStream fileInputStream = new FileInputStream(parameterFilePath); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(fileInputStream)); + String headerStr = bufferedReader.readLine(); + String[] headerSet = headerStr.split("\\|"); + String idStr; + while ((idStr = bufferedReader.readLine()) != null) { + String[] idSet = idStr.split("\\|"); + if (headerSet.length != idSet.length) { + throw new RuntimeException("there is parameter is empty"); + } + HashMap parameter = new HashMap<>(); + for (int i = 0; i < headerSet.length; i++) { + parameter.put(headerSet[i], idSet[i]); + } + parameters.add(parameter); + } + bufferedReader.close(); + fileInputStream.close(); + + return parameters; + } + + protected String getParameterPrefix() { + return "$"; + } + + protected String getParameterPostfix() { + return ""; + } + + protected String getEndDate(String startDate, String durationDays) { + DateFormat format = new SimpleDateFormat("yyyyMMddHHmmssSSS"); // date format + try { + Date sDate = format.parse(startDate); + sDate.after(new Date(Long.parseLong(durationDays) * 24 * 3600 * 1000)); + return format.format( + new Date(sDate.getTime() + (Long.parseLong(durationDays) * 24 * 3600 * 1000))); + } catch (Exception e) { + return String.valueOf( + Long.parseLong(startDate) + (Long.parseLong(durationDays) * 24 * 3600 * 1000)); + } + } + + protected String transformDate(String date) { + DateFormat format = new SimpleDateFormat("yyyyMMddHHmmssSSS"); + TimeZone gmtTime = TimeZone.getTimeZone("UTC"); + format.setTimeZone(gmtTime); + try { + format.parse(date); + return date; + } catch (Exception e) { + return format.format(new Date(Long.parseLong(date))); + } + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/Configuration.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/Configuration.java new file mode 100644 index 000000000000..6aea1df0d394 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/Configuration.java @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; +import java.util.Optional; +import java.util.Properties; + +public class Configuration { + protected HashMap settings = new HashMap<>(); + + public Configuration(Properties properties) { + for (Object key : properties.keySet()) { + settings.put((String) key, properties.getProperty((String) key)); + } + } + + public static final String GREMLIN_SERVER_ENDPOINT = "endpoint"; + public static final String GREMLIN_USERNAME = "username"; + public static final String GREMLIN_PASSWORD = "password"; + + public static final String THREAD_COUNT = "thread_count"; + public static final String QUERY_DIR = "queryDir"; + public static final String PARAMETER_DIR = "interactive.parameters.dir"; + public static final String WARMUP_EVERY_QUERY = "warmup.every.query"; + public static final String OPERATION_COUNT_EVERY_QUERY = "operation.count.every.query"; + public static final String PRINT_QUERY_NAME = "printQueryNames"; + public static final String PRINT_QUERY_RESULT = "printQueryResults"; + + public Optional getOption(String key) { + String value = settings.get(key); + if (value == null) { + return Optional.empty(); + } + return Optional.of(value); + } + + public String getString(String key) { + Optional optional = getOption(key); + if (optional.isPresent()) { + return optional.get(); + } else { + throw new IllegalArgumentException("Required key not found: " + key); + } + } + + public String getString(String key, String defaultValue) { + Optional optional = getOption(key); + return optional.orElse(defaultValue); + } + + public int getInt(String key, int defaultValue) { + Optional optional = getOption(key); + return optional.map(Integer::parseInt).orElse(defaultValue); + } + + public boolean getBoolean(String key, boolean defaultValue) { + Optional optional = getOption(key); + return optional.map(Boolean::parseBoolean).orElse(defaultValue); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery1.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery1.java new file mode 100644 index 000000000000..5dd5ff49ea3e --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery1.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import org.apache.tinkerpop.gremlin.driver.Result; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import java.util.HashMap; +import java.util.Map; + +public class LdbcQuery1 extends AbstractLdbcWithSubQuery { + public LdbcQuery1(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String buildSubQuery(Result result, HashMap singleParameter) { + Vertex v = (Vertex) ((Map) result.getObject()).get("a"); + return String.format( + "g.V(%s).union(out('ISLOCATEDIN').values('name'),outE('STUDYAT').inV().as('u').out('ISLOCATEDIN').as('city').select('u','city'),outE('WORKAT').inV().as('company').out('ISLOCATEDIN').as('country').select('company','country'))", + v.id().toString()); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery12.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery12.java new file mode 100644 index 000000000000..f956fad8c7a2 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery12.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import org.apache.tinkerpop.gremlin.driver.Result; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import java.util.HashMap; +import java.util.Map; + +public class LdbcQuery12 extends AbstractLdbcWithSubQuery { + public LdbcQuery12(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String buildSubQuery(Result result, HashMap singleParameter) { + String tagClassName = singleParameter.get("tagClassName"); + Map map = ((Map) result.getObject()); + return String.format( + "g.V(%s).in('HASCREATOR').hasLabel('COMMENT').out('REPLYOF').hasLabel('POST').out('HASTAG').as('tag').out('HASTYPE').has('name',eq('%s')).select('tag').values('name').dedup()", + map.entrySet().iterator().next().getKey().id().toString(), tagClassName); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery2.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery2.java new file mode 100644 index 000000000000..6e01b8c14b2f --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery2.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; + +public class LdbcQuery2 extends CommonQuery { + public LdbcQuery2(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + singleParameter.put("maxDate", transformDate(singleParameter.get("maxDate"))); + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery3.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery3.java new file mode 100644 index 000000000000..1e9129170864 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery3.java @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import org.apache.tinkerpop.gremlin.driver.Result; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import java.util.HashMap; +import java.util.Map; + +public class LdbcQuery3 extends AbstractLdbcWithSubQuery { + public LdbcQuery3(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + singleParameter.put("startDate", transformDate(singleParameter.get("startDate"))); + String endDate = + getEndDate(singleParameter.get("startDate"), singleParameter.get("durationDays")); + singleParameter.put("endDate", endDate); + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } + + @Override + String buildSubQuery(Result result, HashMap singleParameter) { + Map map = (Map) result.getObject(); + singleParameter.put("startDate", transformDate(singleParameter.get("startDate"))); + String startDate = singleParameter.get("startDate"); + String endDate = getEndDate(startDate, singleParameter.get("durationDays")); + String countryY = singleParameter.get("countryYName"); + + return String.format( + "g.V(%s).in('HASCREATOR').has('creationDate',inside(%s,%s)).filter(__.out('ISLOCATEDIN').has('name',eq('%s'))).count()", + map.entrySet().iterator().next().getKey().id().toString(), + startDate, + endDate, + countryY); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery4.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery4.java new file mode 100644 index 000000000000..8876c382abc1 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery4.java @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; + +public class LdbcQuery4 extends CommonQuery { + public LdbcQuery4(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + singleParameter.put("startDate", transformDate(singleParameter.get("startDate"))); + singleParameter.put( + "endDate", + getEndDate(singleParameter.get("startDate"), singleParameter.get("durationDays"))); + + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery5.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery5.java new file mode 100644 index 000000000000..2d556e02d44e --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery5.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; + +public class LdbcQuery5 extends CommonQuery { + public LdbcQuery5(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + singleParameter.put("minDate", transformDate(singleParameter.get("minDate"))); + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery7.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery7.java new file mode 100644 index 000000000000..328cb469114a --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery7.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import org.apache.tinkerpop.gremlin.driver.Result; + +import java.util.HashMap; +import java.util.Map; + +public class LdbcQuery7 extends AbstractLdbcWithSubQuery { + public LdbcQuery7(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String buildSubQuery(Result result, HashMap singleParameter) { + String personId = singleParameter.get("personId"); + long friendId = (long) ((Map) ((Map) result.getObject()).get("liker")).get("id"); + return String.format( + "g.V().hasLabel('PERSON').has('id',%s).both('KNOWS').hasLabel('PERSON').has('id'," + + " %d)", + personId, friendId); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery9.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery9.java new file mode 100644 index 000000000000..6478231fd67d --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/LdbcQuery9.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; + +public class LdbcQuery9 extends CommonQuery { + public LdbcQuery9(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + singleParameter.put("maxDate", transformDate(singleParameter.get("maxDate"))); + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/QueryWithoutParameter.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/QueryWithoutParameter.java new file mode 100644 index 000000000000..6ac6c5696d18 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/QueryWithoutParameter.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; + +public class QueryWithoutParameter extends CommonQuery { + + public QueryWithoutParameter(String queryName, String queryFile) throws Exception { + super(queryName, queryFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + return gremlinQueryPattern; + } + + @Override + public HashMap getSingleParameter(int index) { + return new HashMap(); + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/SubtaskQuery.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/SubtaskQuery.java new file mode 100644 index 000000000000..fc62261a34ea --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/common/SubtaskQuery.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.common; + +import java.util.HashMap; + +public class SubtaskQuery extends CommonQuery { + public SubtaskQuery(String queryName, String queryFile, String parameterFile) throws Exception { + super(queryName, queryFile, parameterFile); + } + + @Override + String generateGremlinQuery( + HashMap singleParameter, String gremlinQueryPattern) { + + for (String parameter : singleParameter.keySet()) { + gremlinQueryPattern = + gremlinQueryPattern.replace( + getParameterPrefix() + parameter + getParameterPostfix(), + singleParameter.get(parameter)); + } + return gremlinQueryPattern; + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/PropertyUtil.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/PropertyUtil.java new file mode 100644 index 000000000000..85401988081f --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/PropertyUtil.java @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; + +public class PropertyUtil { + public static Properties getProperties(String fileName, boolean resourcePath) { + Properties props = new Properties(); + InputStream is = null; + try { + if (resourcePath) { + is = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName); + } else { + is = new FileInputStream(new File(fileName)); + } + props.load(is); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + try { + if (is != null) { + is.close(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return props; + } +} diff --git a/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java new file mode 100644 index 000000000000..f2adbdc4f4f7 --- /dev/null +++ b/research/query_service/benchmark/src/main/java/com/alibaba/graphscope/gaia/utils/QueryUtil.java @@ -0,0 +1,128 @@ +/** + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gaia.utils; + +import com.alibaba.graphscope.gaia.common.*; + +import java.util.ArrayList; +import java.util.List; + +public class QueryUtil { + public static List initQueryList(Configuration configuration) throws Exception { + String queryDir = configuration.getString(Configuration.QUERY_DIR); + String parameterDir = configuration.getString(Configuration.PARAMETER_DIR); + List queryList = new ArrayList<>(); + + // for ldbc queries + for (int index = 1; index <= 100; index++) { + String enableQuery = String.format("ldbc_query_%d.enable", index); + String queryFileName = String.format("ldbc_query_%d.gremlin", index); + String parameterFileName = String.format("ldbc_query_%d.param", index); + if (configuration.getBoolean(enableQuery, false)) { + String queryFilePath = String.format("%s/%s", queryDir, queryFileName); + String parameterFilePath = String.format("%s/%s", parameterDir, parameterFileName); + String queryName = String.format("LDBC_QUERY_%d", index); + if (index == 1) { + queryList.add(new LdbcQuery1(queryName, queryFilePath, parameterFilePath)); + } else if (index == 2) { + queryList.add(new LdbcQuery2(queryName, queryFilePath, parameterFilePath)); + } else if (index == 3) { + queryList.add(new LdbcQuery3(queryName, queryFilePath, parameterFilePath)); + } else if (index == 4) { + queryList.add(new LdbcQuery4(queryName, queryFilePath, parameterFilePath)); + } else if (index == 5) { + queryList.add(new LdbcQuery5(queryName, queryFilePath, parameterFilePath)); + } else if (index == 7) { + queryList.add(new LdbcQuery7(queryName, queryFilePath, parameterFilePath)); + } else if (index == 9) { + queryList.add(new LdbcQuery9(queryName, queryFilePath, parameterFilePath)); + } else if (index == 12) { + queryList.add(new LdbcQuery12(queryName, queryFilePath, parameterFilePath)); + } else { + queryList.add(new CommonQuery(queryName, queryFilePath, parameterFilePath)); + } + } + } + + // for k hop + for (int index = 1; index < 5; index++) { + String enableQuery = String.format("%d_hop_query.enable", index); + String queryFileName = String.format("%d_hop_query.gremlin", index); + String parameterFileName = String.format("%d_hop_query.param", index); + + if (configuration.getBoolean(enableQuery, false)) { + String queryFilePath = String.format("%s/%s", queryDir, queryFileName); + String parameterFilePath = String.format("%s/%s", parameterDir, parameterFileName); + String queryName = String.format("%d_HOP_QUERY", index); + + queryList.add(new CommonQuery(queryName, queryFilePath, parameterFilePath)); + } + } + + // for benchmarking early-stop queries + String enableEsQuery = "early_stop_query.enable"; + String esQueryFileName = "early_stop_query.gremlin"; + String esParameterFileName = "early_stop_query.param"; + if (configuration.getBoolean(enableEsQuery, false)) { + String queryFilePath = String.format("%s/%s", queryDir, esQueryFileName); + String parameterFilePath = String.format("%s/%s", parameterDir, esParameterFileName); + String queryName = "EARLY_STOP_QUERY"; + queryList.add(new CommonQuery(queryName, queryFilePath, parameterFilePath)); + } + + // for benchmarking subtask queries + String enableSubtaskQuery = "subtask_query.enable"; + String subtaskQueryFileName = "subtask_query.gremlin"; + String subtaskParameterFileName = "subtask_query.param"; + if (configuration.getBoolean(enableSubtaskQuery, false)) { + String queryFilePath = String.format("%s/%s", queryDir, subtaskQueryFileName); + String parameterFilePath = + String.format("%s/%s", parameterDir, subtaskParameterFileName); + String queryName = "SUBTASK_QUERY"; + queryList.add(new SubtaskQuery(queryName, queryFilePath, parameterFilePath)); + } + + // custom queries without parameter + for (int index = 1; index < 100; index++) { + String enableQuery = String.format("custom_constant_query_%d.enable", index); + String queryFileName = String.format("custom_constant_query_%d.gremlin", index); + + if (configuration.getBoolean(enableQuery, false)) { + String queryFilePath = String.format("%s/%s", queryDir, queryFileName); + String queryName = String.format("CUSTOM_QUERY_%d_WITHOUT_PARAMETER", index); + + queryList.add(new QueryWithoutParameter(queryName, queryFilePath)); + } + } + + // custom queries + for (int index = 1; index < 100; index++) { + String enableQuery = String.format("custom_query_%d.enable", index); + String queryFileName = String.format("custom_query_%d.gremlin", index); + String parameterFileName = String.format("custom_query_%d.param", index); + + if (configuration.getBoolean(enableQuery, false)) { + String queryFilePath = String.format("%s/%s", queryDir, queryFileName); + String parameterFilePath = String.format("%s/%s", parameterDir, parameterFileName); + String queryName = String.format("CUSTOM_QUERY_%d", index); + + queryList.add(new CommonQuery(queryName, queryFilePath, parameterFilePath)); + } + } + + return queryList; + } +} diff --git a/research/query_service/ir/Cargo.toml b/research/query_service/ir/Cargo.toml new file mode 100644 index 000000000000..9849ba6a2c20 --- /dev/null +++ b/research/query_service/ir/Cargo.toml @@ -0,0 +1,41 @@ +[workspace] +members = [ + "common", + "core", + "graph_proxy", + "integrated", + "runtime" +] + +[profile.dev] +opt-level = 1 +debug = true +debug-assertions = true +overflow-checks = true +lto = false +panic = 'unwind' +incremental = true +codegen-units = 256 +rpath = false + +[profile.test] +opt-level = 0 +debug = true +debug-assertions = true +overflow-checks = true +lto = false +panic = 'unwind' +incremental = true +codegen-units = 256 +rpath = false + +[profile.release] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = true +panic = 'unwind' +incremental = false +codegen-units = 16 +rpath = false \ No newline at end of file diff --git a/research/query_service/ir/common/Cargo.toml b/research/query_service/ir/common/Cargo.toml new file mode 100644 index 000000000000..db31ead95273 --- /dev/null +++ b/research/query_service/ir/common/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ir_common" +version = "0.1.0" +edition = "2018" + +[dependencies] +dyn_type = {path = "../../../dyn_type"} +pegasus_common = { path = "../../../engine/pegasus/common" } +prost = "0.9" +serde = { version = "1.0", features = ["derive"] } +tonic = "0.4" + +[build-dependencies] +prost-build = "0.9" + +[features] +default = [] +proto_inplace = [] diff --git a/research/query_service/ir/common/build.rs b/research/query_service/ir/common/build.rs new file mode 100644 index 000000000000..e6c9a7c774ad --- /dev/null +++ b/research/query_service/ir/common/build.rs @@ -0,0 +1,70 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +fn main() -> Result<(), Box> { + codegen_inplace() +} + +#[cfg(feature = "proto_inplace")] +use std::path::PathBuf; +#[cfg(feature = "proto_inplace")] +const GEN_DIR: &'static str = "src/generated"; + +#[cfg(feature = "proto_inplace")] +fn codegen_inplace() -> Result<(), Box> { + println!("cargo:rerun-if-changed=../proto/common.proto"); + println!("cargo:rerun-if-changed=../proto/expr.proto"); + println!("cargo:rerun-if-changed=../proto/algebra.proto"); + println!("cargo:rerun-if-changed=../proto/schema.proto"); + println!("cargo:rerun-if-changed=../proto/results.proto"); + let out_dir = PathBuf::from(GEN_DIR); + if out_dir.exists() { + let _ = std::fs::remove_dir_all(GEN_DIR); + } + let _ = std::fs::create_dir(GEN_DIR); + prost_build::Config::new() + .type_attribute(".", "#[derive(Serialize,Deserialize)]") + .out_dir(&out_dir) + .compile_protos( + &[ + "../proto/common.proto", + "../proto/expr.proto", + "../proto/algebra.proto", + "../proto/schema.proto", + "../proto/results.proto", + ], + &["../proto"], + )?; + + Ok(()) +} + +#[cfg(not(feature = "proto_inplace"))] +fn codegen_inplace() -> Result<(), Box> { + prost_build::Config::new() + .type_attribute(".", "#[derive(Serialize,Deserialize)]") + .compile_protos( + &[ + "../proto/common.proto", + "../proto/expr.proto", + "../proto/algebra.proto", + "../proto/schema.proto", + "../proto/results.proto", + ], + &["../proto"], + )?; + + Ok(()) +} diff --git a/research/query_service/ir/common/src/error.rs b/research/query_service/ir/common/src/error.rs new file mode 100644 index 000000000000..020697e2727a --- /dev/null +++ b/research/query_service/ir/common/src/error.rs @@ -0,0 +1,62 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +pub type ParsePbResult = Result; + +/// Errors that occur when parse a pb struct +#[derive(Clone, Debug, PartialEq)] +pub enum ParsePbError { + /// Parse pb structure error + ParseError(String), + /// Caused by any transformation for the pb from `serde_json` + SerdeError(String), + /// Empty pb fields error + EmptyFieldError(String), + /// Not supported + Unsupported(String), +} + +impl std::fmt::Display for ParsePbError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + ParsePbError::ParseError(e) => { + write!(f, "invalid protobuf: {}", e) + } + ParsePbError::SerdeError(e) => { + write!(f, "serde error: {}", e) + } + ParsePbError::EmptyFieldError(e) => { + write!(f, "empty protobuf field: {}", e) + } + ParsePbError::Unsupported(e) => { + write!(f, "Not supported: {}", e) + } + } + } +} + +impl std::error::Error for ParsePbError {} + +impl From for ParsePbError { + fn from(desc: String) -> Self { + ParsePbError::ParseError(desc) + } +} + +impl From<&str> for ParsePbError { + fn from(desc: &str) -> Self { + desc.to_string().into() + } +} diff --git a/research/query_service/ir/common/src/expr_parse/error.rs b/research/query_service/ir/common/src/expr_parse/error.rs new file mode 100644 index 000000000000..2ee9ebe15627 --- /dev/null +++ b/research/query_service/ir/common/src/expr_parse/error.rs @@ -0,0 +1,92 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use std::fmt::Display; + +use crate::error::ParsePbError; +use crate::expr_parse::token::PartialToken; + +pub type ExprResult = Result; + +/// The error cases while parsing and evaluating expressions +#[derive(Clone, Debug, PartialEq)] +pub enum ExprError { + /// The left brace may not be closed by a right brace + UnmatchedLRBraces, + /// The left bracket may not be closed by a right braket + UnmatchedLRBrackets, + /// An escape sequence within a string literal is illegal. + IllegalEscapeSequence(String), + /// A `PartialToken` is unmatched, such that it cannot be combined into a full `Token`. + /// For example, '&' is a partial token, and it can be a full token if there is another + /// '&&' that represents logical and, same applies to '|' ('||'), '=' ('>=', '<=', '=='). + UnmatchedPartialToken { + /// The unmatched partial token. + first: PartialToken, + /// The token that follows the unmatched partial token and that cannot be matched to the + /// partial token, or `None`, if `first` is the last partial token in the stream. + second: Option, + }, + /// Parse from protobuf error + ParsePbError(ParsePbError), + /// Unsupported + Unsupported(String), + /// Other unknown errors that is converted from a error description + OtherErr(String), +} + +impl Display for ExprError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use super::ExprError::*; + match self { + UnmatchedLRBraces => write!(f, "the left and right braces may not be matched"), + UnmatchedLRBrackets => write!(f, "the left and right brackets may not be matched"), + IllegalEscapeSequence(s) => write!(f, "illegal escape sequence {:?}", s), + UnmatchedPartialToken { first: s1, second: s2 } => { + write!(f, "partial token {:?} cannot be completed by {:?}", s1, s2) + } + ParsePbError(e) => { + write!(f, "parse from pb error {:?}", e) + } + Unsupported(e) => write!(f, "unsupported: {}", e), + OtherErr(e) => write!(f, "parse error {}", e), + } + } +} + +impl std::error::Error for ExprError {} + +impl ExprError { + pub fn unmatched_partial_token(first: PartialToken, second: Option) -> Self { + Self::UnmatchedPartialToken { first, second } + } + + pub fn unsupported(string: String) -> Self { + Self::Unsupported(string) + } +} + +impl From for ExprError { + fn from(error: ParsePbError) -> Self { + Self::ParsePbError(error) + } +} + +impl From<&str> for ExprError { + fn from(e: &str) -> Self { + Self::OtherErr(e.into()) + } +} diff --git a/research/query_service/ir/common/src/expr_parse/mod.rs b/research/query_service/ir/common/src/expr_parse/mod.rs new file mode 100644 index 000000000000..d2b9009fd58b --- /dev/null +++ b/research/query_service/ir/common/src/expr_parse/mod.rs @@ -0,0 +1,285 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +pub mod error; +pub mod token; + +use std::convert::{TryFrom, TryInto}; + +use crate::expr_parse::error::{ExprError, ExprResult}; +use crate::expr_parse::token::{tokenize, Token}; +use crate::generated::common as pb; +use crate::VAR_PREFIX; + +fn idents_to_vars(idents: Vec) -> ExprResult { + let mut vars = Vec::with_capacity(idents.len()); + for ident in idents { + if !ident.starts_with(VAR_PREFIX) { + return Err(format!("invalid variable token: {:?}, a variable must start with \"@\"", ident) + .as_str() + .into()); + } else { + let var: pb::Variable = ident.into(); + vars.push(var) + } + } + + Ok(pb::VariableKeys { keys: vars }) +} + +impl TryFrom for pb::ExprOpr { + type Error = ExprError; + + fn try_from(token: Token) -> ExprResult { + match token { + Token::Plus => Ok(pb::Arithmetic::Add.into()), + Token::Minus => Ok(pb::Arithmetic::Sub.into()), + Token::Star => Ok(pb::Arithmetic::Mul.into()), + Token::Slash => Ok(pb::Arithmetic::Div.into()), + Token::Percent => Ok(pb::Arithmetic::Mod.into()), + Token::Hat => Ok(pb::Arithmetic::Exp.into()), + Token::Eq => Ok(pb::Logical::Eq.into()), + Token::Ne => Ok(pb::Logical::Ne.into()), + Token::Gt => Ok(pb::Logical::Gt.into()), + Token::Lt => Ok(pb::Logical::Lt.into()), + Token::Ge => Ok(pb::Logical::Ge.into()), + Token::Le => Ok(pb::Logical::Le.into()), + Token::And => Ok(pb::Logical::And.into()), + Token::Or => Ok(pb::Logical::Or.into()), + Token::Not => Ok(pb::Logical::Not.into()), + Token::Within => Ok(pb::Logical::Within.into()), + Token::Without => Ok(pb::Logical::Without.into()), + Token::Boolean(b) => Ok(pb::Value::from(b).into()), + Token::Int(i) => Ok(pb::Value::from(i).into()), + Token::Float(f) => Ok(pb::Value::from(f).into()), + Token::String(s) => Ok(pb::Value::from(s).into()), + Token::IntArray(v) => Ok(pb::Value::from(v).into()), + Token::FloatArray(v) => Ok(pb::Value::from(v).into()), + Token::StrArray(v) => Ok(pb::Value::from(v).into()), + Token::LBrace => Ok(pb::ExprOpr { item: Some(pb::expr_opr::Item::Brace(0)) }), + Token::RBrace => Ok(pb::ExprOpr { item: Some(pb::expr_opr::Item::Brace(1)) }), + Token::Identifier(ident) => { + if !ident.starts_with(VAR_PREFIX) { + Err(format!("invalid variable token: {:?}, a variable must start with \"@\"", ident) + .as_str() + .into()) + } else { + let var: pb::Variable = ident.into(); + Ok(var.into()) + } + } + Token::IdentArray(idents) => Ok((idents_to_vars(idents)?, false).into()), + Token::IdentMap(idents) => Ok((idents_to_vars(idents)?, true).into()), + } + } +} + +pub trait ExprToken { + fn is_operand(&self) -> bool; + + fn is_left_brace(&self) -> bool; + + fn is_right_brace(&self) -> bool; + + /// Returns the precedence of the operator. + /// A high precedence means that the operator has larger priority to get operated + fn precedence(&self) -> i32; +} + +impl ExprToken for pb::ExprOpr { + fn is_operand(&self) -> bool { + match self.item { + Some(pb::expr_opr::Item::Const(_)) => true, + Some(pb::expr_opr::Item::Var(_)) => true, + _ => false, + } + } + + fn is_left_brace(&self) -> bool { + match self.item { + Some(pb::expr_opr::Item::Brace(i)) => i == 0, + _ => false, + } + } + + fn is_right_brace(&self) -> bool { + match self.item { + Some(pb::expr_opr::Item::Brace(i)) => i == 1, + _ => false, + } + } + + fn precedence(&self) -> i32 { + use pb::expr_opr::Item::*; + if self.item.is_some() { + match self.item.as_ref().unwrap() { + &Arith(a) => { + let arith = unsafe { std::mem::transmute::(a) }; + match arith { + pb::Arithmetic::Add | pb::Arithmetic::Sub => 95, + pb::Arithmetic::Mul | pb::Arithmetic::Div | pb::Arithmetic::Mod => 100, + pb::Arithmetic::Exp => 120, + } + } + &Logical(l) => { + let logical = unsafe { std::mem::transmute::(l) }; + match logical { + pb::Logical::Eq + | pb::Logical::Ne + | pb::Logical::Lt + | pb::Logical::Le + | pb::Logical::Gt + | pb::Logical::Ge + | pb::Logical::Within + | pb::Logical::Without => 80, + pb::Logical::And => 75, + pb::Logical::Or => 70, + pb::Logical::Not => 110, + } + } + &Brace(_) => 0, + _ => 200, + } + } else { + -1 + } + } +} + +#[allow(dead_code)] +/// Turn a sequence of tokens with bracket, into a suffix order +pub fn to_suffix_expr(expr: Vec) -> ExprResult> { + let mut stack: Vec = Vec::with_capacity(expr.len()); + let mut results: Vec = Vec::with_capacity(expr.len()); + + for token in expr { + if token.is_operand() { + results.push(token); + } else if token.is_left_brace() { + stack.push(token); + } else if token.is_right_brace() { + let mut is_left_brace = false; + while !stack.is_empty() { + let recent = stack.pop().unwrap(); + if recent.is_left_brace() { + is_left_brace = true; + break; + } else { + results.push(recent); + } + } + if !is_left_brace { + return Err(ExprError::UnmatchedLRBraces); + } + } else { + // the operator + if stack.is_empty() { + stack.push(token); + } else { + while !stack.is_empty() && stack.last().unwrap().precedence() >= token.precedence() { + results.push(stack.pop().unwrap()); + } + stack.push(token); + } + } + } + + while !stack.is_empty() { + results.push(stack.pop().unwrap()); + } + Ok(results) +} + +pub fn str_to_expr_pb(expr_str: String) -> ExprResult { + let mut operators = vec![]; + for token in tokenize(&expr_str)? { + operators.push(token.try_into()?); + } + + Ok(pb::Expression { operators }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expr_parse::token::tokenize; + + #[test] + fn test_to_suffix_expr() { + // 1 + 2 + let case1 = tokenize("1 + 2").unwrap(); + let expected_case1 = vec![Token::Int(1), Token::Int(2), Token::Plus]; + assert_eq!(to_suffix_expr(case1).unwrap(), expected_case1); + + // 1 + 2 == 3 + let case2 = tokenize("1 + 2 == 3").unwrap(); + let expected_case2 = vec![Token::Int(1), Token::Int(2), Token::Plus, Token::Int(3), Token::Eq]; + assert_eq!(to_suffix_expr(case2).unwrap(), expected_case2); + + // 1 + 2 * 3 + let case3 = tokenize("1 + 2 * 3").unwrap(); + let expected_case3 = vec![Token::Int(1), Token::Int(2), Token::Int(3), Token::Star, Token::Plus]; + assert_eq!(to_suffix_expr(case3).unwrap(), expected_case3); + + // (1 + 2) * 3 + let case4 = tokenize("(1 + 2) * 3").unwrap(); + let expected_case4 = vec![Token::Int(1), Token::Int(2), Token::Plus, Token::Int(3), Token::Star]; + assert_eq!(to_suffix_expr(case4).unwrap(), expected_case4); + + // 1 + 2 ^ 3 > 6 + let case5 = tokenize(" 1 + 2 ^ 3 > 6").unwrap(); + let expected_case5 = vec![ + Token::Int(1), + Token::Int(2), + Token::Int(3), + Token::Hat, + Token::Plus, + Token::Int(6), + Token::Gt, + ]; + assert_eq!(to_suffix_expr(case5).unwrap(), expected_case5); + + // (1 + 2) ^ 3 > 6 + let case6 = tokenize("(1 + 2) ^ 3 > 6").unwrap(); + let expected_case6 = vec![ + Token::Int(1), + Token::Int(2), + Token::Plus, + Token::Int(3), + Token::Hat, + Token::Int(6), + Token::Gt, + ]; + assert_eq!(to_suffix_expr(case6).unwrap(), expected_case6); + + // ((1 + 2) * 2) ^ 3 == 6 ^ 3 + let case7 = tokenize("((1 + 1e-3) * 2) ^ 3 == 6 ^ 3").unwrap(); + let expected_case7 = vec![ + Token::Int(1), + Token::Float(0.001), + Token::Plus, + Token::Int(2), + Token::Star, + Token::Int(3), + Token::Hat, + Token::Int(6), + Token::Int(3), + Token::Hat, + Token::Eq, + ]; + assert_eq!(to_suffix_expr(case7).unwrap(), expected_case7); + } +} diff --git a/research/query_service/ir/common/src/expr_parse/token.rs b/research/query_service/ir/common/src/expr_parse/token.rs new file mode 100644 index 000000000000..0478fb9db68f --- /dev/null +++ b/research/query_service/ir/common/src/expr_parse/token.rs @@ -0,0 +1,643 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use crate::expr_parse::error::{ExprError, ExprResult}; +use crate::expr_parse::ExprToken; + +#[derive(Debug, PartialEq, Clone)] +pub enum Token { + // Arithmetic + Plus, // + + Minus, // - + Star, // * + Slash, // / + Percent, // % + Hat, // ^ + + // Logical + Eq, // == + Ne, // != + Gt, // > + Lt, // < + Ge, // >= + Le, // <= + And, // && + Or, // || + Not, // ! + Within, // Within + Without, // Without + + // Precedence + LBrace, // ( + RBrace, // ) + + // Values and Variables + Identifier(String), // a string-identifier + Boolean(bool), // a boolean value + Int(i64), // an integer value + Float(f64), // a float value + String(String), // a string value + IntArray(Vec), // a integer array + FloatArray(Vec), // a float array + StrArray(Vec), // a string array + IdentArray(Vec), // an identifier array + IdentMap(Vec), // an identifier map +} + +impl ExprToken for Token { + #[inline] + fn is_operand(&self) -> bool { + use crate::expr_parse::token::Token::*; + match self { + Identifier(_) | Float(_) | Int(_) | Boolean(_) | String(_) | IntArray(_) | FloatArray(_) + | StrArray(_) | IdentArray(_) | IdentMap(_) => true, + _ => false, + } + } + + #[inline] + fn is_left_brace(&self) -> bool { + self == &Token::LBrace + } + + #[inline] + fn is_right_brace(&self) -> bool { + self == &Token::RBrace + } + + #[inline] + fn precedence(&self) -> i32 { + use crate::expr_parse::token::Token::*; + match self { + Plus | Minus => 95, + Star | Slash | Percent => 100, + Hat => 120, + + Eq | Ne | Gt | Lt | Ge | Le | Within | Without => 80, + And => 75, + Or => 70, + Not => 110, + + LBrace | RBrace => 0, + _ => 200, + } + } +} + +/// A partial token is an input character whose meaning depends on the characters around it. +#[derive(Clone, Debug, PartialEq)] +pub enum PartialToken { + /// A partial token that unambiguously maps to a single token. + Token(Token), + /// A partial token that is a minus, which may be a Sub, or a Negative sign. + Minus, + /// A partial token that is a literal. + Literal(String), + /// A whitespace character, e.g. ' '. + Whitespace, + /// An equal-to character '='. + Eq, + /// An exclamation mark character '!'. + ExclamationMark, + /// A greater-than character '>'. + Gt, + /// A lower-than character '<'. + Lt, + /// An ampersand character '&'. + Ampersand, + /// A vertical bar character '|'. + VerticalBar, + /// To initiate an array, '[' + LBracket, + /// To terminate an array, ']' + RBracket, + /// To initiate a map, '{' + LCBracket, + /// To terminate a map, '}' + RCBracket, +} + +#[inline] +fn char_to_partial_token(c: char) -> PartialToken { + match c { + '=' => PartialToken::Eq, + '!' => PartialToken::ExclamationMark, + '>' => PartialToken::Gt, + '<' => PartialToken::Lt, + '&' => PartialToken::Ampersand, + '|' => PartialToken::VerticalBar, + '-' => PartialToken::Minus, + '[' => PartialToken::LBracket, + ']' => PartialToken::RBracket, + '{' => PartialToken::LCBracket, + '}' => PartialToken::RCBracket, + '+' => PartialToken::Token(Token::Plus), + '*' => PartialToken::Token(Token::Star), + '/' => PartialToken::Token(Token::Slash), + '%' => PartialToken::Token(Token::Percent), + '^' => PartialToken::Token(Token::Hat), + '(' => PartialToken::Token(Token::LBrace), + ')' => PartialToken::Token(Token::RBrace), + c => { + if c.is_whitespace() { + PartialToken::Whitespace + } else { + PartialToken::Literal(c.to_string()) + } + } + } +} + +/// Parses an escape sequence within a string literal. +fn parse_escape_sequence>(iter: &mut Iter) -> ExprResult { + match iter.next() { + Some('"') => Ok('"'), + Some('\\') => Ok('\\'), + Some(c) => Err(ExprError::IllegalEscapeSequence(format!("\\{}", c))), + None => Err(ExprError::IllegalEscapeSequence("\\".to_string())), + } +} + +/// Parses a string value from the given character iterator. +/// +/// Either in the case of an array, in which the whole literal is enclosed by a pair of brackets [], +/// or it is a typical string literal in a quote, in this case: +/// * The first character from the iterator is interpreted as first character of the string. +/// * The string is terminated by a double quote `"`. +/// * Occurrences of `"` within the string can be escaped with `\`. +/// * The backslash needs to be escaped with another backslash `\`. +fn parse_string_literal>( + mut iter: &mut Iter, is_in_bracket: bool, +) -> ExprResult { + let mut result = String::new(); + if !is_in_bracket { + while let Some(c) = iter.next() { + match c { + '"' => break, + '\\' => result.push(parse_escape_sequence(&mut iter)?), + c => result.push(c), + } + } + } else { + let mut has_right_bracket = false; + // Treat everything as a string + while let Some(c) = iter.next() { + if c == ']' || c == '}' { + has_right_bracket = true; + break; + } else if c == '[' || c == '{' { + return Err(ExprError::unsupported("nested array is not supported".to_string())); + } else { + result.push(c); + } + } + + if !has_right_bracket { + return Err(ExprError::UnmatchedLRBrackets); + } + } + + Ok(PartialToken::Token(Token::String(result))) +} + +/// Converts a string to a vector of partial tokens. +fn str_to_partial_tokens(string: &str) -> ExprResult> { + let mut result = Vec::new(); + let mut iter = string.chars().peekable(); + while let Some(c) = iter.next() { + if c == '"' { + result.push(parse_string_literal(&mut iter, false)?); + } else if c == '[' { + result.push(PartialToken::LBracket); + result.push(parse_string_literal(&mut iter, true)?); + // must have right bracket to escape from `parse_string_literal()` + result.push(PartialToken::RBracket); + } else if c == '{' { + result.push(PartialToken::LCBracket); + result.push(parse_string_literal(&mut iter, true)?); + // must have right bracket to escape from `parse_string_literal()` + result.push(PartialToken::RCBracket); + } else { + let partial_token = char_to_partial_token(c); + + let if_let_successful = + if let (Some(PartialToken::Literal(last)), PartialToken::Literal(literal)) = + (result.last_mut(), &partial_token) + { + last.push_str(literal); + true + } else { + false + }; + + if !if_let_successful { + result.push(partial_token); + } + } + } + Ok(result) +} + +fn token_array_to_token(token_array: Vec) -> ExprResult { + if token_array.is_empty() { + Ok(Token::IdentArray(vec![])) + } else { + // use pivot to regulate the type of all elements + let pivot = token_array.first().unwrap(); + match pivot { + Token::Int(_) => { + let mut vec = Vec::with_capacity(token_array.len()); + for t in token_array { + match t { + Token::Int(i) => vec.push(i), + _ => { + return Err(ExprError::unsupported( + "array of various type unsupported".to_string(), + )) + } + } + } + Ok(Token::IntArray(vec)) + } + Token::Float(_) => { + let mut vec = Vec::with_capacity(token_array.len()); + for t in token_array { + match t { + Token::Float(f) => vec.push(f), + _ => { + return Err(ExprError::unsupported( + "array of various type unsupported".to_string(), + )) + } + } + } + Ok(Token::FloatArray(vec)) + } + Token::String(_) => { + let mut vec = Vec::with_capacity(token_array.len()); + for t in token_array { + match t { + Token::String(s) => vec.push(s), + _ => { + return Err(ExprError::unsupported( + "array of various type unsupported".to_string(), + )) + } + } + } + Ok(Token::StrArray(vec)) + } + Token::Identifier(_) => { + let mut vec = Vec::with_capacity(token_array.len()); + for t in token_array { + match t { + Token::Identifier(s) => vec.push(s), + _ => { + return Err(ExprError::unsupported( + "array of various type unsupported".to_string(), + )) + } + } + } + Ok(Token::IdentArray(vec)) + } + _ => Err(ExprError::unsupported(format!("array of this type: {:?} is not supported", pivot))), + } + } +} + +/// Resolves all partial tokens by converting them to complex tokens. +fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> ExprResult> { + let mut result = Vec::new(); + let mut recent_token: Option = None; + while !tokens.is_empty() { + let first = tokens[0].clone(); + let second = tokens.get(1).cloned(); + let third = tokens.get(2).cloned(); + let mut cutoff = 2; + + let curr_token = match first { + PartialToken::Token(token) => { + cutoff = 1; + Some(token) + } + PartialToken::Literal(literal) => { + cutoff = 1; + if let Ok(number) = literal.parse::() { + Some(Token::Int(number)) + } else if let Ok(number) = literal.parse::() { + Some(Token::Float(number)) + } else if let Ok(boolean) = literal.parse::() { + Some(Token::Boolean(boolean)) + } else if literal.to_lowercase().as_str() == "within" { + Some(Token::Within) + } else if literal.to_lowercase().as_str() == "without" { + Some(Token::Without) + } else { + // To parse the float of the form `e{+,-}`, + // for example [Literal("10e"), Minus, Literal("3")] => "1e-3".parse(). + match (second, third) { + (Some(second), Some(third)) + if second == PartialToken::Minus + || second == PartialToken::Token(Token::Plus) => + { + let second_sign = match second { + PartialToken::Minus => "-", + _ => "+", + }; + let third_num = match third { + PartialToken::Literal(s) => s, + _ => "".to_string(), + }; + if let Ok(number) = + format!("{}{}{}", literal, second_sign, third_num).parse::() + { + cutoff = 3; + Some(Token::Float(number)) + } else { + Some(Token::Identifier(literal.to_string())) + } + } + _ => Some(Token::Identifier(literal.to_string())), + } + } + } + PartialToken::Whitespace => { + cutoff = 1; + None + } + PartialToken::Minus => { + // Should we consider minus as a negative sign + let is_negative_sign = + { recent_token.is_none() || !recent_token.as_ref().unwrap().is_operand() }; + if is_negative_sign { + match &second { + // Be aware that minus can represent both subtraction or negative sign + // if it is a negative sign, it must be directly trailed by a number. + // However, we can not actually tell whether the case "x -y", is actually + // subtracting x by y, or -y must be treated as a number. + Some(PartialToken::Literal(literal)) => { + // Must check whether previous is what + if let Ok(number) = literal.parse::() { + Some(Token::Int(-number)) + } else if let Ok(number) = literal.parse::() { + Some(Token::Float(-number)) + } else { + return Err(ExprError::unmatched_partial_token(first, second)); + } + } + _ => { + cutoff = 1; + Some(Token::Minus) + } + } + } else { + cutoff = 1; + Some(Token::Minus) + } + } + PartialToken::Eq => match second { + Some(PartialToken::Eq) => Some(Token::Eq), + _ => { + return Err(ExprError::unmatched_partial_token(first, second)); + } + }, + PartialToken::ExclamationMark => match second { + Some(PartialToken::Eq) => Some(Token::Ne), + _ => { + cutoff = 1; + Some(Token::Not) + } + }, + PartialToken::Gt => match second { + Some(PartialToken::Eq) => Some(Token::Ge), + _ => { + cutoff = 1; + Some(Token::Gt) + } + }, + PartialToken::Lt => match second { + Some(PartialToken::Eq) => Some(Token::Le), + _ => { + cutoff = 1; + Some(Token::Lt) + } + }, + PartialToken::Ampersand => match second { + Some(PartialToken::Ampersand) => Some(Token::And), + _ => return Err(ExprError::unmatched_partial_token(first, second)), + }, + PartialToken::VerticalBar => match second { + Some(PartialToken::VerticalBar) => Some(Token::Or), + _ => return Err(ExprError::unmatched_partial_token(first, second)), + }, + PartialToken::LBracket | PartialToken::LCBracket => { + let is_bracket = first == PartialToken::LBracket; + cutoff = 3; + if (is_bracket && third != Some(PartialToken::RBracket)) + || (!is_bracket && third != Some(PartialToken::RCBracket)) + { + return Err(ExprError::UnmatchedLRBrackets); + } else { + let mut token_array: Vec = Vec::new(); + match second { + Some(PartialToken::Token(Token::String(ref s))) => { + let elements = s.split(","); + for e in elements { + let t = partial_tokens_to_tokens(&str_to_partial_tokens(e)?)?; + if t.is_empty() { + // do nothing + } else if t.len() == 1 { + token_array.push(t[0].clone()) + } else { + return Err(format!("invalid token: {:?}", second) + .as_str() + .into()); + } + } + } + _ => { + return Err(format!("invalid token: {:?}", second) + .as_str() + .into()) + } + } + let result = token_array_to_token(token_array)?; + if is_bracket { + Some(result) + } else { + if let Token::IdentArray(vec) = result { + Some(Token::IdentMap(vec)) + } else { + unreachable!() + } + } + } + } + _ => { + return Err(format!("invalid token: {:?}", first) + .as_str() + .into()); + } + }; + + if let Some(token) = curr_token.clone() { + result.push(token); + recent_token = curr_token.clone(); + } + + tokens = &tokens[cutoff..]; + } + Ok(result) +} + +pub fn tokenize(string: &str) -> ExprResult> { + partial_tokens_to_tokens(&str_to_partial_tokens(string)?) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tokenize() { + // ((1 + 2) * 2) ^ 3 == 6 ^ 3 + let case1 = tokenize("((1 + 1e-3) * 2) ^ 3 == 6 ^ 3").unwrap(); + let expected_case1 = vec![ + Token::LBrace, + Token::LBrace, + Token::Int(1), + Token::Plus, + Token::Float(0.001), + Token::RBrace, + Token::Star, + Token::Int(2), + Token::RBrace, + Token::Hat, + Token::Int(3), + Token::Eq, + Token::Int(6), + Token::Hat, + Token::Int(3), + ]; + + assert_eq!(case1, expected_case1); + + let case2 = tokenize("1 - 2").unwrap(); + let expected_case2 = vec![Token::Int(1), Token::Minus, Token::Int(2)]; + assert_eq!(case2, expected_case2); + + let case3 = tokenize("1 + (-2)").unwrap(); + let expected_case3 = vec![Token::Int(1), Token::Plus, Token::LBrace, Token::Int(-2), Token::RBrace]; + assert_eq!(case3, expected_case3); + + let case4 = tokenize("1 + -2 + 2").unwrap(); + let expected_case4 = vec![Token::Int(1), Token::Plus, Token::Int(-2), Token::Plus, Token::Int(2)]; + assert_eq!(case4, expected_case4); + } + + #[test] + fn test_tokenize_array() { + let case1 = tokenize("[1, 2, 3, 4]").unwrap(); + let expected_case1 = vec![Token::IntArray(vec![1, 2, 3, 4])]; + assert_eq!(case1, expected_case1); + + let case2 = tokenize("1 within [1, 2, 3, 4]").unwrap(); + let expected_case2 = vec![Token::Int(1), Token::Within, Token::IntArray(vec![1, 2, 3, 4])]; + assert_eq!(case2, expected_case2); + + let case3 = tokenize("[1.0] without [1.0, 2.0, 3.0, 4.0]").unwrap(); + let expected_case3 = + vec![Token::FloatArray(vec![1.0]), Token::Without, Token::FloatArray(vec![1.0, 2.0, 3.0, 4.0])]; + assert_eq!(case3, expected_case3); + + let case4 = tokenize("\"a\" within [\"a\", \"b\"]").unwrap(); + let expected_case4 = vec![ + Token::String("a".to_string()), + Token::Within, + Token::StrArray(vec!["a".to_string(), "b".to_string()]), + ]; + assert_eq!(case4, expected_case4); + + let case5 = tokenize("[1, -2, 3, -4]").unwrap(); + let expected_case5 = vec![Token::IntArray(vec![1, -2, 3, -4])]; + assert_eq!(case5, expected_case5); + + let case6 = tokenize("-4.0 within [1.0, -2.0, 3.0, -4.0]").unwrap(); + let expected_case6 = + vec![Token::Float(-4.0), Token::Within, Token::FloatArray(vec![1.0, -2.0, 3.0, -4.0])]; + assert_eq!(case6, expected_case6); + + let case7 = tokenize("[@a, @a.name, @.age]"); + let expected_case7 = + vec![Token::IdentArray(vec!["@a".to_string(), "@a.name".to_string(), "@.age".to_string()])]; + assert_eq!(case7.unwrap(), expected_case7,); + + let case8 = tokenize("{@a, @a.name, @.age}"); + let expected_case8 = + vec![Token::IdentMap(vec!["@a".to_string(), "@a.name".to_string(), "@.age".to_string()])]; + assert_eq!(case8.unwrap(), expected_case8,); + + let case9 = tokenize("[]"); + assert_eq!(case9.unwrap(), vec![Token::IdentArray(vec![])]); + } + + #[test] + fn test_tokenize_errors() { + // 1 = 1, the partial = must be completed by another = + let case1 = tokenize("1 = 1"); + assert_eq!( + case1.err().unwrap(), + ExprError::unmatched_partial_token(PartialToken::Eq, Some(PartialToken::Whitespace)) + ); + + let case2 = tokenize("1 & 2"); + assert_eq!( + case2.err().unwrap(), + ExprError::unmatched_partial_token(PartialToken::Ampersand, Some(PartialToken::Whitespace)) + ); + + let case3 = tokenize("1 | 2"); + assert_eq!( + case3.err().unwrap(), + ExprError::unmatched_partial_token(PartialToken::VerticalBar, Some(PartialToken::Whitespace)) + ); + + let case4 = tokenize("-a"); + assert_eq!( + case4.err().unwrap(), + ExprError::unmatched_partial_token( + PartialToken::Minus, + Some(PartialToken::Literal("a".to_string())) + ) + ); + + let case5 = tokenize("[1, -2, 3, -4"); + assert_eq!(case5.err().unwrap(), ExprError::UnmatchedLRBrackets); + + let case6 = tokenize("[1, -2, [3], -4]"); + assert_eq!( + case6.err().unwrap(), + ExprError::Unsupported("nested array is not supported".to_string()) + ); + + let case7 = tokenize("[1, 0.5, -4]"); + assert_eq!( + case7.err().unwrap(), + ExprError::unsupported("array of various type unsupported".to_string()) + ); + } +} diff --git a/research/query_service/ir/common/src/lib.rs b/research/query_service/ir/common/src/lib.rs new file mode 100644 index 000000000000..10d9d21fd745 --- /dev/null +++ b/research/query_service/ir/common/src/lib.rs @@ -0,0 +1,725 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::convert::TryFrom; +use std::io; + +use dyn_type::{BorrowObject, Object, Primitives}; +use pegasus_common::codec::{Decode, Encode, ReadExt, WriteExt}; +use prost::Message; + +use crate::error::{ParsePbError, ParsePbResult}; +use crate::generated::algebra as pb; +use crate::generated::common as common_pb; +use crate::generated::results as result_pb; + +pub mod error; +pub mod expr_parse; + +#[macro_use] +extern crate serde; + +#[cfg(feature = "proto_inplace")] +pub mod generated { + #[path = "algebra.rs"] + pub mod algebra; + #[path = "common.rs"] + pub mod common; + #[path = "results.rs"] + pub mod results; + #[path = "schema.rs"] + pub mod schema; +} + +#[cfg(not(feature = "proto_inplace"))] +pub mod generated { + pub mod common { + tonic::include_proto!("common"); + } + pub mod algebra { + tonic::include_proto!("algebra"); + } + pub mod results { + tonic::include_proto!("results"); + } + pub mod schema { + tonic::include_proto!("schema"); + } +} + +pub type KeyId = i32; +pub const SPLITTER: &'static str = "."; +pub const VAR_PREFIX: &'static str = "@"; + +/// Refer to a key of a relation or a graph element, by either a string-type name or an identifier +#[derive(Debug, PartialEq, Eq, Hash, Clone, PartialOrd, Ord)] +pub enum NameOrId { + Str(String), + Id(KeyId), +} + +impl Default for NameOrId { + fn default() -> Self { + Self::Str("".to_string()) + } +} + +impl NameOrId { + pub fn as_object(&self) -> Object { + match self { + NameOrId::Str(s) => s.to_string().into(), + NameOrId::Id(id) => (*id as i32).into(), + } + } + + pub fn as_borrow_object(&self) -> BorrowObject { + match self { + NameOrId::Str(s) => BorrowObject::String(s.as_str()), + NameOrId::Id(id) => (*id as i32).into(), + } + } +} + +impl From for NameOrId { + fn from(id: KeyId) -> Self { + Self::Id(id) + } +} + +impl From for NameOrId { + fn from(str: String) -> Self { + Self::Str(str) + } +} + +impl From<&str> for NameOrId { + fn from(str: &str) -> Self { + Self::Str(str.to_string()) + } +} + +impl Encode for NameOrId { + fn write_to(&self, writer: &mut W) -> io::Result<()> { + match self { + NameOrId::Id(id) => { + writer.write_u8(0)?; + writer.write_i32(*id)?; + } + NameOrId::Str(str) => { + writer.write_u8(1)?; + str.write_to(writer)?; + } + } + Ok(()) + } +} + +impl Decode for NameOrId { + fn read_from(reader: &mut R) -> io::Result { + let e = reader.read_u8()?; + match e { + 0 => { + let id = reader.read_i32()?; + Ok(NameOrId::Id(id)) + } + 1 => { + let str = ::read_from(reader)?; + Ok(NameOrId::Str(str)) + } + _ => Err(io::Error::new(io::ErrorKind::Other, "unreachable")), + } + } +} + +impl TryFrom for NameOrId { + type Error = ParsePbError; + + fn try_from(t: common_pb::NameOrId) -> ParsePbResult + where + Self: Sized, + { + use common_pb::name_or_id::Item; + + if let Some(item) = t.item { + match item { + Item::Name(name) => Ok(NameOrId::Str(name)), + Item::Id(id) => { + if id < 0 { + Err(ParsePbError::from("key id must be positive number")) + } else { + Ok(NameOrId::Id(id as KeyId)) + } + } + } + } else { + Err(ParsePbError::from("empty content provided")) + } + } +} + +impl From for common_pb::NameOrId { + fn from(tag: NameOrId) -> Self { + let name_or_id = match tag { + NameOrId::Str(name) => common_pb::name_or_id::Item::Name(name), + NameOrId::Id(id) => common_pb::name_or_id::Item::Id(id), + }; + common_pb::NameOrId { item: Some(name_or_id) } + } +} + +impl From for common_pb::ExprOpr { + fn from(arith: common_pb::Arithmetic) -> Self { + common_pb::ExprOpr { + item: Some(common_pb::expr_opr::Item::Arith(unsafe { + std::mem::transmute::(arith) + })), + } + } +} + +impl From for common_pb::ExprOpr { + fn from(logical: common_pb::Logical) -> Self { + common_pb::ExprOpr { + item: Some(common_pb::expr_opr::Item::Logical(unsafe { + std::mem::transmute::(logical) + })), + } + } +} + +impl From for common_pb::ExprOpr { + fn from(const_val: common_pb::Value) -> Self { + common_pb::ExprOpr { item: Some(common_pb::expr_opr::Item::Const(const_val)) } + } +} + +impl From for common_pb::ExprOpr { + fn from(var: common_pb::Variable) -> Self { + common_pb::ExprOpr { item: Some(common_pb::expr_opr::Item::Var(var)) } + } +} + +/// An indicator for whether it is a map +impl From<(common_pb::VariableKeys, bool)> for common_pb::ExprOpr { + fn from(vars: (common_pb::VariableKeys, bool)) -> Self { + if !vars.1 { + // not a map + common_pb::ExprOpr { item: Some(common_pb::expr_opr::Item::Vars(vars.0)) } + } else { + // is a map + common_pb::ExprOpr { item: Some(common_pb::expr_opr::Item::VarMap(vars.0)) } + } + } +} + +impl From for common_pb::Value { + fn from(b: bool) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::Boolean(b)) } + } +} + +impl From for common_pb::Value { + fn from(f: f64) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::F64(f)) } + } +} + +impl From for common_pb::Value { + fn from(i: i32) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::I32(i)) } + } +} + +impl From for common_pb::Value { + fn from(i: i64) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::I64(i)) } + } +} + +impl From for common_pb::Value { + fn from(s: String) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::Str(s)) } + } +} + +impl From> for common_pb::Value { + fn from(item: Vec) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::I64Array(common_pb::I64Array { item })) } + } +} + +impl From> for common_pb::Value { + fn from(item: Vec) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::F64Array(common_pb::DoubleArray { item })) } + } +} + +impl From> for common_pb::Value { + fn from(item: Vec) -> Self { + common_pb::Value { item: Some(common_pb::value::Item::StrArray(common_pb::StringArray { item })) } + } +} + +impl From for common_pb::NameOrId { + fn from(i: i32) -> Self { + common_pb::NameOrId { item: Some(common_pb::name_or_id::Item::Id(i)) } + } +} + +impl From<&str> for common_pb::NameOrId { + fn from(str: &str) -> Self { + common_pb::NameOrId { item: Some(common_pb::name_or_id::Item::Name(str.to_string())) } + } +} + +impl From for common_pb::NameOrId { + fn from(str: String) -> Self { + common_pb::NameOrId { item: Some(common_pb::name_or_id::Item::Name(str)) } + } +} + +pub const ID_KEY: &'static str = "~id"; +pub const LABEL_KEY: &'static str = "~label"; +pub const LENGTH_KEY: &'static str = "~len"; +pub const ALL_KEY: &'static str = "~all"; + +impl From for common_pb::Property { + fn from(str: String) -> Self { + if str == ID_KEY { + common_pb::Property { item: Some(common_pb::property::Item::Id(common_pb::IdKey {})) } + } else if str == LABEL_KEY { + common_pb::Property { item: Some(common_pb::property::Item::Label(common_pb::LabelKey {})) } + } else if str == LENGTH_KEY { + common_pb::Property { item: Some(common_pb::property::Item::Len(common_pb::LengthKey {})) } + } else if str == ALL_KEY { + common_pb::Property { item: Some(common_pb::property::Item::All(common_pb::AllKey {})) } + } else { + common_pb::Property { item: Some(common_pb::property::Item::Key(str.into())) } + } + } +} + +fn str_as_tag(str: String) -> Option { + if !str.is_empty() { + Some(if let Ok(str_int) = str.parse::() { str_int.into() } else { str.into() }) + } else { + None + } +} + +impl From for common_pb::Variable { + fn from(str: String) -> Self { + assert!(str.starts_with(VAR_PREFIX)); + // skip the var variable + let str: String = str.chars().skip(1).collect(); + if !str.contains(SPLITTER) { + common_pb::Variable { + // If the tag is represented as an integer + tag: str_as_tag(str), + property: None, + } + } else { + let mut splitter = str.split(SPLITTER); + let tag: Option = + if let Some(first) = splitter.next() { str_as_tag(first.to_string()) } else { None }; + let property: Option = + if let Some(second) = splitter.next() { Some(second.to_string().into()) } else { None }; + common_pb::Variable { tag, property } + } + } +} + +impl From for pb::index_predicate::AndPredicate { + fn from(id: i64) -> Self { + pb::index_predicate::AndPredicate { + predicates: vec![pb::index_predicate::Triplet { + key: Some(common_pb::Property { + item: Some(common_pb::property::Item::Id(common_pb::IdKey {})), + }), + value: Some(id.into()), + cmp: None, + }], + } + } +} + +impl From> for pb::IndexPredicate { + fn from(ids: Vec) -> Self { + let or_predicates: Vec = + ids.into_iter().map(|id| id.into()).collect(); + + pb::IndexPredicate { or_predicates } + } +} + +impl From for pb::index_predicate::AndPredicate { + fn from(label: String) -> Self { + pb::index_predicate::AndPredicate { + predicates: vec![pb::index_predicate::Triplet { + key: Some(common_pb::Property { + item: Some(common_pb::property::Item::Label(common_pb::LabelKey {})), + }), + value: Some(label.into()), + cmp: None, + }], + } + } +} + +impl From> for pb::IndexPredicate { + fn from(names: Vec) -> Self { + let or_predicates: Vec = names + .into_iter() + .map(|name| name.into()) + .collect(); + + pb::IndexPredicate { or_predicates } + } +} + +impl TryFrom for Object { + type Error = ParsePbError; + + fn try_from(value: common_pb::Value) -> Result { + use common_pb::value::Item::*; + if let Some(item) = value.item.as_ref() { + return match item { + Boolean(b) => Ok((*b).into()), + I32(i) => Ok((*i).into()), + I64(i) => Ok((*i).into()), + F64(f) => Ok((*f).into()), + Str(s) => Ok(s.clone().into()), + Blob(blob) => Ok(blob.clone().into()), + None(_) => Ok(Object::None), + I32Array(v) => Ok(v.item.clone().into()), + I64Array(v) => Ok(v.item.clone().into()), + F64Array(v) => Ok(v.item.clone().into()), + StrArray(v) => Ok(v.item.clone().into()), + PairArray(pairs) => { + let mut vec = Vec::<(Object, Object)>::with_capacity(pairs.item.len()); + for item in pairs.item.clone().into_iter() { + let (key_obj, val_obj) = + (Object::try_from(item.key.unwrap())?, Object::try_from(item.val.unwrap())?); + vec.push((key_obj, val_obj)); + } + Ok(vec.into()) + } + }; + } + + Err(ParsePbError::from("empty value provided")) + } +} + +impl TryFrom for Vec { + type Error = ParsePbError; + + fn try_from(value: pb::IndexPredicate) -> Result { + let mut global_ids = vec![]; + for and_predicate in value.or_predicates { + let predicate = and_predicate + .predicates + .get(0) + .ok_or(ParsePbError::EmptyFieldError("`AndCondition` is emtpy".to_string()))?; + + let (key, value) = (predicate.key.as_ref(), predicate.value.as_ref()); + let key = key.ok_or("key is empty in kv_pair in indexed_scan")?; + if let Some(common_pb::property::Item::Id(_id_key)) = key.item.as_ref() { + let value = value.ok_or("value is empty in kv_pair in indexed_scan")?; + + match &value.item { + Some(common_pb::value::Item::I64(v)) => { + global_ids.push(*v); + } + Some(common_pb::value::Item::I64Array(arr)) => { + global_ids.extend(arr.item.iter().cloned()) + } + Some(common_pb::value::Item::I32(v)) => { + global_ids.push(*v as i64); + } + Some(common_pb::value::Item::I32Array(arr)) => { + global_ids.extend(arr.item.iter().map(|i| *i as i64)); + } + _ => Err(ParsePbError::Unsupported( + "indexed value other than integer (I32, I64) and integer array".to_string(), + ))?, + } + } else { + Err(ParsePbError::Unsupported("indexed field other than `IdKey`".to_string()))? + } + } + Ok(global_ids) + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Project) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Project(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Select) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Select(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Join) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Join(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Union) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Union(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::GroupBy) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::GroupBy(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::OrderBy) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::OrderBy(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Dedup) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Dedup(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Unfold) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Unfold(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Apply) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Apply(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::SegmentApply) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::SegApply(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Scan) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Scan(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Limit) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Limit(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Auxilia) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Auxilia(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::As) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::As(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::EdgeExpand) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Edge(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::PathExpand) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Path(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::PathStart) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::PathStart(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::PathEnd) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::PathEnd(opr)) } + } +} + +/* +impl From for pb::logical_plan::Operator { + fn from(opr: pb::ShortestPathExpand) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::ShortestPath(opr)) } + } +} + */ + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::GetV) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Vertex(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Pattern) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Pattern(opr)) } + } +} + +impl From for pb::logical_plan::Operator { + fn from(opr: pb::Sink) -> Self { + pb::logical_plan::Operator { opr: Some(pb::logical_plan::operator::Opr::Sink(opr)) } + } +} + +impl From for common_pb::Value { + fn from(value: Object) -> Self { + let item = match value { + Object::Primitive(v) => match v { + // TODO: It seems that Byte is only used for bool for now + Primitives::Byte(v) => common_pb::value::Item::Boolean(!(v == 0)), + Primitives::Integer(v) => common_pb::value::Item::I32(v), + Primitives::Long(v) => common_pb::value::Item::I64(v), + Primitives::ULLong(v) => common_pb::value::Item::Str(v.to_string()), + Primitives::Float(v) => common_pb::value::Item::F64(v), + }, + Object::String(s) => common_pb::value::Item::Str(s), + Object::Blob(b) => common_pb::value::Item::Blob(b.to_vec()), + Object::Vector(v) => common_pb::value::Item::StrArray(common_pb::StringArray { + item: v + .into_iter() + .map(|obj| obj.to_string()) + .collect(), + }), + Object::KV(kv) => { + let mut pairs: Vec = Vec::with_capacity(kv.len()); + for (key, val) in kv { + let key_pb: common_pb::Value = key.into(); + let val_pb: common_pb::Value = val.into(); + pairs.push(common_pb::Pair { key: Some(key_pb), val: Some(val_pb) }) + } + common_pb::value::Item::PairArray(common_pb::PairArray { item: pairs }) + } + Object::None => common_pb::value::Item::None(common_pb::None {}), + _ => unimplemented!(), + }; + + common_pb::Value { item: Some(item) } + } +} + +impl Encode for result_pb::Results { + fn write_to(&self, writer: &mut W) -> io::Result<()> { + let mut bytes = vec![]; + self.encode_raw(&mut bytes); + writer.write_u32(bytes.len() as u32)?; + writer.write_all(bytes.as_slice())?; + Ok(()) + } +} + +impl Decode for result_pb::Results { + fn read_from(reader: &mut R) -> io::Result { + let len = reader.read_u32()? as usize; + let mut buffer = Vec::with_capacity(len); + reader.read_exact(&mut buffer)?; + result_pb::Results::decode(buffer.as_slice()) + .map_err(|_e| std::io::Error::new(std::io::ErrorKind::Other, "decoding result_pb failed!")) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_str_to_variable() { + let case1 = "@1"; + assert_eq!( + common_pb::Variable { tag: Some(common_pb::NameOrId::from(1)), property: None }, + common_pb::Variable::from(case1.to_string()) + ); + + let case2 = "@a"; + assert_eq!( + common_pb::Variable { tag: Some(common_pb::NameOrId::from("a".to_string())), property: None }, + common_pb::Variable::from(case2.to_string()) + ); + + let case3 = "@1.~id"; + assert_eq!( + common_pb::Variable { + tag: Some(common_pb::NameOrId::from(1)), + property: Some(common_pb::Property { + item: Some(common_pb::property::Item::Id(common_pb::IdKey {})) + }) + }, + common_pb::Variable::from(case3.to_string()) + ); + + let case4 = "@1.~label"; + assert_eq!( + common_pb::Variable { + tag: Some(common_pb::NameOrId::from(1)), + property: Some(common_pb::Property { + item: Some(common_pb::property::Item::Label(common_pb::LabelKey {})) + }) + }, + common_pb::Variable::from(case4.to_string()) + ); + + let case5 = "@1.name"; + assert_eq!( + common_pb::Variable { + tag: Some(common_pb::NameOrId::from(1)), + property: Some(common_pb::Property { + item: Some(common_pb::property::Item::Key("name".to_string().into())) + }) + }, + common_pb::Variable::from(case5.to_string()) + ); + + let case6 = "@.name"; + assert_eq!( + common_pb::Variable { + tag: None, + property: Some(common_pb::Property { + item: Some(common_pb::property::Item::Key("name".to_string().into())) + }) + }, + common_pb::Variable::from(case6.to_string()) + ); + + let case7 = "@"; + assert_eq!( + common_pb::Variable { tag: None, property: None }, + common_pb::Variable::from(case7.to_string()) + ); + } +} diff --git a/research/query_service/ir/compiler/Makefile b/research/query_service/ir/compiler/Makefile new file mode 100644 index 000000000000..15b50601d405 --- /dev/null +++ b/research/query_service/ir/compiler/Makefile @@ -0,0 +1,63 @@ +OPT?=poc + +CUR_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) + +ifeq ($(JAVA_HOME),) + java:=java +else + java:=$(JAVA_HOME)/bin/java +endif + +OSFLAG := +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +graph.schema:= + +ifeq ($(UNAME_S),Darwin) + ifeq ($(UNAME_M),arm64) + OSFLAG += -Dos.detected.classifier=osx-x86_64 + endif +endif + +build: + cd $(CUR_DIR)/../../../engine/pegasus/clients/java/client && \ + mvn clean install -DskipTests $(OSFLAG) && \ + cd $(CUR_DIR)/../core && cargo build --release && \ + cd ../integrated && cargo build --release --bin start_rpc_server && \ + cd $(CUR_DIR) && mvn clean package -DskipTests $(OSFLAG) + +clean: + cd $(CUR_DIR)/../../../engine/pegasus/clients/java/client && mvn clean && \ + cd $(CUR_DIR)/../ && cargo clean && \ + cd $(CUR_DIR) && mvn clean + +test: + cd $(CUR_DIR)/../ && cargo test && \ + cd $(CUR_DIR) && mvn test + +# start rpc server +# make run +gremlin_test: + mvn test -Dtest=com.alibaba.graphscope.integration.IrGremlinTest $(OSFLAG) + +submit: + cd $(CUR_DIR) && $(java) \ + -cp ".:./target/libs/*:./target/ir-compiler-1.0-SNAPSHOT.jar" \ + -Djna.library.path=../target/release \ + com.alibaba.graphscope.common.SubmitPlanServiceMain \ + $(OPT) + +run: + cd $(CUR_DIR) && $(java) \ + -cp ".:./target/libs/*:./target/ir-compiler-1.0-SNAPSHOT.jar" \ + -Djna.library.path=../target/release \ + -Dgraph.schema=${graph.schema} \ + com.alibaba.graphscope.gremlin.service.GraphServiceMain + +# start rpc server +# make run graph.schema:=../core/resource/ldbc_schema.json +ldbc_test: + mvn test -Dtest=com.alibaba.graphscope.integration.ldbc.IrLdbcTest $(OSFLAG) + +.PHONY: build run diff --git a/research/query_service/ir/compiler/conf/ir.compiler.properties b/research/query_service/ir/compiler/conf/ir.compiler.properties new file mode 100644 index 000000000000..327ca719efbb --- /dev/null +++ b/research/query_service/ir/compiler/conf/ir.compiler.properties @@ -0,0 +1,10 @@ +# pegasus service config +pegasus.worker.num: 2 +pegasus.timeout: 240000 +pegasus.batch.size: 1024 +pegasus.output.capacity: 16 +pegasus.hosts: localhost:1234 +pegasus.server.num: 1 + +# graph.schema +graph.schema: ../core/resource/modern_schema.json diff --git a/research/query_service/ir/compiler/ir_exprimental_ci.sh b/research/query_service/ir/compiler/ir_exprimental_ci.sh new file mode 100755 index 000000000000..1f866bbbd3df --- /dev/null +++ b/research/query_service/ir/compiler/ir_exprimental_ci.sh @@ -0,0 +1,19 @@ +#!/bin/bash +base_dir=$(cd $(dirname $0); pwd) +# start engine service and load modern graph +cd ${base_dir}/../target/release && RUST_LOG=info ./start_rpc_server & +sleep 5s +# start compiler service +cd ${base_dir} && make run & +sleep 5s +# run gremlin standard tests +cd ${base_dir} && make gremlin_test +exit_code=$? +# clean service +ps -ef | grep "com.alibaba.graphscope.gremlin.service.GraphServiceMain" | grep -v grep | awk '{print $2}' | xargs kill -9 || true +ps -ef | grep "start_rpc_server" | grep -v grep | awk '{print $2}' | xargs kill -9 +# report test result +if [ $exit_code -ne 0 ]; then + echo "ir integration test on experimental store fail" + exit 1 +fi diff --git a/research/query_service/ir/compiler/pom.xml b/research/query_service/ir/compiler/pom.xml new file mode 100644 index 000000000000..0cc8d81cceae --- /dev/null +++ b/research/query_service/ir/compiler/pom.xml @@ -0,0 +1,249 @@ + + + + 4.0.0 + + ir-compiler + com.alibaba.graphscope + 1.0-SNAPSHOT + + + 2.12.1 + 3.12.0 + 2.6 + 3.18.0 + 1.42.1 + 1.4.1.Final + 3.0.0 + 0.5.0 + 1.4 + 3.0.0 + 3.7.0 + 5.7.0 + 3.0.0 + 4.13.2 + 2.22.2 + 3.5.1 + 2.6 + 4.9.1 + + + + + org.antlr + antlr4 + ${antlr4.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + com.alibaba.pegasus + pegasus-client + 1.0-SNAPSHOT + + + org.apache.commons + commons-lang3 + ${commons.langs.version} + + + net.java.dev.jna + jna + ${jna.version} + + + junit + junit + ${junit.version} + + + org.apache.tinkerpop + gremlin-core + ${tinkerpop.version} + + + org.apache.tinkerpop + tinkergraph-gremlin + ${tinkerpop.version} + + + org.apache.tinkerpop + gremlin-server + ${tinkerpop.version} + + + org.apache.tinkerpop + gremlin-test + ${tinkerpop.version} + + + commons-io + commons-io + ${commons.io.version} + + + com.fasterxml.jackson.module + jackson-module-paranamer + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + + + kr.motd.maven + os-maven-plugin + ${os.maven.version} + + + + + org.apache.maven.plugins + maven-clean-plugin + ${maven.clean.version} + + + + src/main/generated + false + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf.maven.version} + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc.grpc.version}:exe:${os.detected.classifier} + + ${project.basedir}/../proto/ + src/main/generated/ + false + + + + + compile + compile-custom + + + + + + org.codehaus.mojo + build-helper-maven-plugin + ${build.helper.version} + + + generate-sources + + add-source + + + + src/main/generated/ + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + prepare-package + + copy-dependencies + + + + ${project.build.directory}/libs + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.version} + + + + true + libs/ + + com.alibaba.graphscope.common.SubmitPlanServiceMain + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compile.version} + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.version} + + + -Djna.library.path=${project.basedir}/../target/release + + + **/IrGremlinTest.java + **/IrLdbcTest.java + + + + + org.antlr + antlr4-maven-plugin + ${antlr4.version} + + + antlr + + antlr4 + + + + -package + org.apache.tinkerpop.gremlin.language.grammar + -encoding + utf-8 + + true + + ${project.build.directory}/generated-sources/antlr4/org/apache/tinkerpop/gremlin/language/grammar/ + + + + + + + + + \ No newline at end of file diff --git a/research/query_service/ir/compiler/src/main/antlr4/GremlinGS.g4 b/research/query_service/ir/compiler/src/main/antlr4/GremlinGS.g4 new file mode 100644 index 000000000000..9db87200e74e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/antlr4/GremlinGS.g4 @@ -0,0 +1,641 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +grammar GremlinGS; + +// g or g.rootTraversal() +query + : rootTraversal + ; + +// g +traversalSource + : TRAVERSAL_ROOT + ; + +// g.rootTraversal() +rootTraversal + : traversalSource DOT traversalSourceSpawnMethod + | traversalSource DOT traversalSourceSpawnMethod DOT chainedTraversal + ; + +// A recursive definition of chained traversal, where +// it is either a traversal method itself, e.g. V(), has(), out() +// or it is . +chainedTraversal + : traversalMethod + | chainedTraversal DOT traversalMethod + ; + +traversalSourceSpawnMethod + : traversalSourceSpawnMethod_V // V() + | traversalSourceSpawnMethod_E // E() + ; + +// Defining supported traversal methods +traversalMethod + : traversalMethod_as // as() + | traversalMethod_hasLabel // hasLabel() + | traversalMethod_hasId // hasId() + | traversalMethod_has // has() + | traversalMethod_out // out() + | traversalMethod_in // in() + | traversalMethod_both // in() + | traversalMethod_outE // outE()[.inV()] + | traversalMethod_inE // inE()[.outV()] + | traversalMethod_bothE // bothE()[.otherV()] + | traversalMethod_limit // limit() + | traversalMethod_valueMap // valueMap() + | traversalMethod_order // order() + | traversalMethod_select // select() + | traversalMethod_dedup // dedup() + | traversalMethod_group // group() + | traversalMethod_groupCount // groupCount() + | traversalMethod_values // values() + | traversalMethod_count // count() + | traversalMethod_is // is() + | traversalMethod_where // where() + | traversalMethod_inV // inV() + | traversalMethod_outV // outV() + | traversalMethod_endV // endV() + | traversalMethod_otherV // otherV() + | traversalMethod_not // not() + | traversalMethod_union // union() + | traversalMethod_range // range() + | traversalMethod_match // match() + ; + +traversalSourceSpawnMethod_V + : 'V' LPAREN integerLiteralList RPAREN + ; + +traversalSourceSpawnMethod_E + : 'E' LPAREN integerLiteralList RPAREN + ; + +traversalMethod_as + : 'as' LPAREN stringLiteral RPAREN + ; + +// hasLabel('') +traversalMethod_hasLabel + : 'hasLabel' LPAREN stringLiteral (COMMA stringLiteralList)? RPAREN + ; + +// hasId(1, 2, 3) +traversalMethod_hasId + : 'hasId' LPAREN integerLiteral (COMMA integerLiteralList)? RPAREN + ; + +// has("str", y), has("str", eq/neq/gt/gte/lt/lte(y)) +// has("person", "name", "marko") +// has("person", "name", P.eq("marko")) +// has("name") +traversalMethod_has + : 'has' LPAREN stringLiteral COMMA genericLiteral RPAREN // indicate eq + | 'has' LPAREN stringLiteral COMMA traversalPredicate RPAREN + | 'has' LPAREN stringLiteral COMMA stringLiteral COMMA genericLiteral RPAREN + | 'has' LPAREN stringLiteral COMMA stringLiteral COMMA traversalPredicate RPAREN + | 'has' LPAREN stringLiteral RPAREN + ; + +// out('str1', ...) +// out('1..5', 'str1') +traversalMethod_out + : 'out' LPAREN stringLiteralList RPAREN + ; + +// in('str1', ...) +// in('1..5', 'str1') +traversalMethod_in + : 'in' LPAREN stringLiteralList RPAREN + ; + +// both('str1', ...) +// both('1..5', 'str1', ...) +traversalMethod_both + : 'both' LPAREN stringLiteralList RPAREN + ; + +// outE('str1', ...), outE().inV() +traversalMethod_outE + : 'outE' LPAREN stringLiteralList RPAREN (DOT traversalMethod_inV)? + ; + +// inE('str1', ...), inE().outV() +traversalMethod_inE + : 'inE' LPAREN stringLiteralList RPAREN (DOT traversalMethod_outV)? + ; + +// bothE('str1', ...), bothE().otherV() +traversalMethod_bothE + : 'bothE' LPAREN stringLiteralList RPAREN (DOT traversalMethod_otherV)? + ; + +// outV() +traversalMethod_outV + : 'outV' LPAREN RPAREN + ; + +// inV() +traversalMethod_inV + : 'inV' LPAREN RPAREN + ; + +// otherV() +traversalMethod_otherV + : 'otherV' LPAREN RPAREN + ; + +// endV() +traversalMethod_endV + : 'endV' LPAREN RPAREN + ; + +// limit(n) +traversalMethod_limit + : 'limit' LPAREN integerLiteral RPAREN + ; + +// valueMap('s1', ...) +// valueMap() is unsupported +traversalMethod_valueMap + : 'valueMap' LPAREN stringLiteralExpr RPAREN + ; + +// order() +// order().by +traversalMethod_order + : 'order' LPAREN RPAREN (DOT traversalMethod_orderby_list)? + ; + +// by() +// by('asc' | 'desc') +// by('a', 'asc' | 'desc') +// by(values(..), 'asc' | 'desc') +// by(select("a"), 'asc' | 'desc') +// by(select("a").by("name"), 'asc' | 'desc') +// by(select("a").by(valueMap("name")), 'asc' | 'desc') +// by(out().count()), by(out().count(), desc) +traversalMethod_orderby + : 'by' LPAREN RPAREN + | 'by' LPAREN traversalOrder RPAREN + | 'by' LPAREN stringLiteral (COMMA traversalOrder)? RPAREN + | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_values (COMMA traversalOrder)? RPAREN + | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_select (COMMA traversalOrder)? RPAREN + | 'by' LPAREN nestedTraversal (COMMA traversalOrder)? RPAREN + ; + +traversalMethod_orderby_list + : traversalMethod_orderby (DOT traversalMethod_orderby)* + ; + +// select('s', ...) +// select('s', ...).by(...).by(...) +// select(expr('@.age')) +traversalMethod_select + : 'select' LPAREN stringLiteral (COMMA stringLiteralList)? RPAREN (DOT traversalMethod_selectby_list)? + | 'select' LPAREN traversalColumn RPAREN + | 'select' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_expr RPAREN + ; + +// by() +// by("name") +// by(valueMap()) +// by(out().count()) +traversalMethod_selectby + : 'by' LPAREN RPAREN + | 'by' LPAREN stringLiteral RPAREN + | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_valueMap RPAREN + | 'by' LPAREN nestedTraversal RPAREN + ; + +traversalMethod_selectby_list + : traversalMethod_selectby (DOT traversalMethod_selectby)* + ; + +// dedup in global scope +traversalMethod_dedup + : 'dedup' LPAREN RPAREN + ; + +traversalMethod_group + : 'group' LPAREN RPAREN (DOT traversalMethod_group_keyby)? + | 'group' LPAREN RPAREN DOT traversalMethod_group_keyby DOT traversalMethod_group_valueby + ; + +traversalMethod_groupCount + : 'groupCount' LPAREN RPAREN (DOT traversalMethod_group_keyby)? + ; + +// group().by() +// group().by('name') +// group().by(values('name')) +// group().by(values('name').as('key')) +// group().by(out().count()) +traversalMethod_group_keyby + : 'by' LPAREN RPAREN + | 'by' LPAREN stringLiteral RPAREN + | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_values (DOT traversalMethod_as)? RPAREN + | 'by' LPAREN nestedTraversal RPAREN + ; + +// group().by(...).by() +// group().by(...).by(fold().as("value")) +// group().by(...).by(count()) +// group().by(...).by(count().as("value")) +traversalMethod_group_valueby + : 'by' LPAREN RPAREN + | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_aggregate_func (DOT traversalMethod_as)? RPAREN + ; + +traversalMethod_aggregate_func + : traversalMethod_count + | traversalMethod_fold + ; + +// count in global scope +traversalMethod_count + : 'count' LPAREN RPAREN + ; + +// only one argument is permitted +// values("name") +traversalMethod_values + : 'values' LPAREN stringLiteral RPAREN + ; + +// fold() +traversalMethod_fold + : 'fold' LPAREN RPAREN + ; + +// is(27) +// is(P.eq(27)) +traversalMethod_is + : 'is' LPAREN genericLiteral RPAREN + | 'is' LPAREN traversalPredicate RPAREN + ; + +// where(P.eq("a")) +// where("c", P.eq("a")) +// where(P.eq("a")).by("age") +// where("c", P.eq("a")).by("id").by("age") +// where(out().out()...) +// where(__.as("a")...as("b")) +// where(__.not(__.out)) equal to not(__.out) +// where(expr("@.age && @.age > 20")) +traversalMethod_where + : 'where' LPAREN traversalPredicate RPAREN (DOT traversalMethod_whereby_list)? + | 'where' LPAREN stringLiteral COMMA traversalPredicate RPAREN (DOT traversalMethod_whereby_list)? + | 'where' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_not RPAREN // match not(__.out) as traversalMethod_not instead of nestedTraversal + | 'where' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_expr RPAREN + | 'where' LPAREN nestedTraversal RPAREN + ; + +// where().by() +// where().by('name') +// where().by(values('name')) +traversalMethod_whereby + : 'by' LPAREN RPAREN + | 'by' LPAREN stringLiteral RPAREN + | 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_values RPAREN + | 'by' LPAREN nestedTraversal RPAREN + ; + +traversalMethod_whereby_list + : traversalMethod_whereby (DOT traversalMethod_whereby)* + ; + +traversalMethod_not + : 'not' LPAREN nestedTraversal RPAREN + ; + +// union(__.out(), __.out().out()) +traversalMethod_union + : 'union' LPAREN nestedTraversalExpr RPAREN + ; + +nestedTraversalExpr + : nestedTraversal (COMMA nestedTraversal)* + ; + +traversalMethod_range + : 'range' LPAREN integerLiteral COMMA integerLiteral RPAREN + ; + +traversalMethod_match + : 'match' LPAREN nestedTraversalExpr RPAREN + ; + +traversalMethod_expr + : 'expr' LPAREN stringLiteral RPAREN + ; + +// only permit non empty, \'\' or \"\" or \'null\' is meaningless as a parameter +stringLiteral + : NonEmptyStringLiteral + ; + +stringLiteralList + : stringLiteralExpr? + | LBRACK stringLiteralExpr? RBRACK + ; + +stringLiteralExpr + : stringLiteral (COMMA stringLiteral)* + ; + +genericLiteral + : integerLiteral + | floatLiteral + | booleanLiteral + | stringLiteral + ; + +genericLiteralList + : genericLiteralExpr? + | LBRACK genericLiteralExpr? RBRACK + ; + +genericLiteralExpr + : genericLiteral (COMMA genericLiteral)* + ; + +integerLiteral + : IntegerLiteral + ; + +integerLiteralList + : integerLiteralExpr? + | LBRACK integerLiteralExpr? RBRACK + ; + +integerLiteralExpr + : integerLiteral (COMMA integerLiteral)* + ; + +floatLiteral + : FloatingPointLiteral + ; + +booleanLiteral + : BooleanLiteral + ; + +nullLiteral + : NullLiteral + ; + +// Traversal predicate +traversalPredicate + : traversalPredicate_eq + | traversalPredicate_neq + | traversalPredicate_lt + | traversalPredicate_lte + | traversalPredicate_gt + | traversalPredicate_gte + | traversalPredicate_within + | traversalPredicate_without + | traversalPredicate DOT 'and' LPAREN traversalPredicate RPAREN + | traversalPredicate DOT 'or' LPAREN traversalPredicate RPAREN + ; + +nestedTraversal + : chainedTraversal + | ANON_TRAVERSAL_ROOT DOT chainedTraversal + ; + +traversalPredicate_eq + : ('P.eq' | 'eq') LPAREN genericLiteral RPAREN + ; + +traversalPredicate_neq + : ('P.neq' | 'neq') LPAREN genericLiteral RPAREN + ; + +traversalPredicate_lt + : ('P.lt' | 'lt') LPAREN genericLiteral RPAREN + ; + +traversalPredicate_lte + : ('P.lte' | 'lte') LPAREN genericLiteral RPAREN + ; + +traversalPredicate_gt + : ('P.gt' | 'gt') LPAREN genericLiteral RPAREN + ; + +traversalPredicate_gte + : ('P.gte' | 'gte') LPAREN genericLiteral RPAREN + ; + +traversalPredicate_within + : ('P.within' | 'within') LPAREN genericLiteralList RPAREN + ; + +traversalPredicate_without + : ('P.without' | 'without') LPAREN genericLiteralList RPAREN + ; + +// incr and decr is unsupported in 3.5.1 +traversalOrder + : 'asc' | 'Order.asc' + | 'desc' | 'Order.desc' + | 'shuffle' | 'Order.shuffle' + ; + +traversalColumn + : 'keys' | 'Column.keys' + | 'values' | 'Column.values' + ; + +// Integer Literals + +IntegerLiteral + : Sign? DecimalIntegerLiteral + ; + +fragment +DecimalIntegerLiteral + : DecimalNumeral IntegerTypeSuffix? + ; + +fragment +IntegerTypeSuffix + : [lL] + ; + +fragment +DecimalNumeral + : '0' + | NonZeroDigit (Digits? | Underscores Digits) + ; + +fragment +Digits + : Digit (DigitsAndUnderscores? Digit)? + ; + +fragment +Digit + : '0' + | NonZeroDigit + ; + +fragment +NonZeroDigit + : [1-9] + ; + +fragment +DigitsAndUnderscores + : DigitOrUnderscore+ + ; + +fragment +DigitOrUnderscore + : Digit + | '_' + ; + +fragment +Underscores + : '_'+ + ; + +// Floating-Point Literals + +FloatingPointLiteral + : Sign? DecimalFloatingPointLiteral + ; + +fragment +DecimalFloatingPointLiteral + : Digits ('.' Digits ExponentPart? | ExponentPart) FloatTypeSuffix? + | Digits FloatTypeSuffix + ; + +fragment +ExponentPart + : ExponentIndicator SignedInteger + ; + +fragment +ExponentIndicator + : [eE] + ; + +fragment +SignedInteger + : Sign? Digits + ; + +fragment +Sign + : [+-] + ; + +fragment +FloatTypeSuffix + : [fFdD] + ; + +// Boolean Literals + +BooleanLiteral + : 'true' + | 'false' + ; + +// Null Literal + +NullLiteral + : 'null' + ; + + +fragment +DoubleQuotedStringCharacters + : DoubleQuotedStringCharacter+ + ; + +EmptyStringLiteral + : '""' + | '\'\'' + ; + +NonEmptyStringLiteral + : '"' DoubleQuotedStringCharacters '"' + | '\'' SingleQuotedStringCharacters '\'' + ; + +fragment +DoubleQuotedStringCharacter + : ~('"' | '\\') + | JoinLineEscape + | EscapeSequence + ; + +fragment +SingleQuotedStringCharacters + : SingleQuotedStringCharacter+ + ; + +fragment +SingleQuotedStringCharacter + : ~('\'' | '\\') + | JoinLineEscape + | EscapeSequence + ; + +// Escape Sequences for Character and String Literals +fragment JoinLineEscape + : '\\' '\r'? '\n' + ; + +fragment +EscapeSequence + : '\\' [btnfr"'\\] + ; + +// Separators + +LPAREN : '('; +RPAREN : ')'; +LBRACE : '{'; +RBRACE : '}'; +LBRACK : '['; +RBRACK : ']'; +SEMI : ';'; +COMMA : ','; +DOT : '.'; +COLON : ':'; + +TRAVERSAL_ROOT: 'g'; +ANON_TRAVERSAL_ROOT: '__'; + +// Trim whitespace and comments if present + +WS : [ \t\r\n\u000C]+ -> skip + ; + +LINE_COMMENT + : '//' ~[\r\n]* -> skip + ; diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java new file mode 100644 index 000000000000..5a6c111d9aea --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java @@ -0,0 +1,699 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.graphscope.common.exception.*; +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.MatchSentence; +import com.alibaba.graphscope.common.intermediate.operator.*; +import com.alibaba.graphscope.common.intermediate.process.SinkArg; +import com.alibaba.graphscope.common.jna.IrCoreLibrary; +import com.alibaba.graphscope.common.jna.type.*; +import com.alibaba.graphscope.common.utils.ClassUtils; +import com.alibaba.graphscope.gremlin.Utils; +import com.sun.jna.Pointer; +import com.sun.jna.ptr.IntByReference; + +import org.apache.commons.io.FileUtils; +import org.javatuples.Pair; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.Function; + +// represent ir plan as a chain of operators +public class IrPlan implements Closeable { + private static IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE; + private static String PLAN_JSON_FILE = "plan.json"; + private Pointer ptrPlan; + + // call libc to transform from InterOpBase to c structure + private enum TransformFactory implements Function { + SCAN_FUSION_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + ScanFusionOp op = (ScanFusionOp) baseOp; + Optional scanOpt = op.getScanOpt(); + if (!scanOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "scanOpt", "not present"); + } + FfiScanOpt ffiScanOpt = (FfiScanOpt) scanOpt.get().applyArg(); + Pointer scan = irCoreLib.initScanOperator(ffiScanOpt); + + // set params + FfiError e1 = irCoreLib.setScanParams(scan, getQueryParams(op)); + if (e1.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "params", "setScanParams returns " + e1.msg); + } + + // set index predicate + Optional ids = op.getIds(); + if (ids.isPresent()) { + Pointer idsPredicate = irCoreLib.initIndexPredicate(); + List ffiIds = (List) ids.get().applyArg(); + for (int i = 0; i < ffiIds.size(); ++i) { + FfiError e2 = + irCoreLib.orEquivPredicate( + idsPredicate, irCoreLib.asIdKey(), ffiIds.get(i)); + if (e2.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "ids", "orEquivPredicate returns " + e2.msg); + } + } + if (!ffiIds.isEmpty()) { + irCoreLib.addScanIndexPredicate(scan, idsPredicate); + } + } + // todo: add other predicates + // todo: add limit + + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent()) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) aliasOpt.get().applyArg(); + irCoreLib.setScanAlias(scan, alias); + } + return scan; + } + + private Pointer getQueryParams(ScanFusionOp op) { + Optional labels = op.getLabels(); + Pointer ptrParams = irCoreLib.initQueryParams(); + if (labels.isPresent()) { + List ffiLabels = + (List) labels.get().applyArg(); + for (FfiNameOrId.ByValue label : ffiLabels) { + FfiError error = irCoreLib.addParamsTable(ptrParams, label); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + op.getClass(), "tables", "addParamsTable returns " + error.msg); + } + } + } + Optional predicate = op.getPredicate(); + if (predicate.isPresent()) { + String expr = (String) predicate.get().applyArg(); + FfiError error = irCoreLib.setParamsPredicate(ptrParams, expr); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + op.getClass(), + "predicate", + "setParamsPredicate returns " + error.msg); + } + } + return ptrParams; + } + }, + SELECT_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + SelectOp op = (SelectOp) baseOp; + Optional predicate = op.getPredicate(); + if (!predicate.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "predicate", "not present"); + } + String expr = (String) predicate.get().applyArg(); + Pointer select = irCoreLib.initSelectOperator(); + FfiError error = irCoreLib.setSelectPredicate(select, expr); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + baseOp.getClass(), + "predicate", + "setSelectPredicate returns " + error.msg); + } + return select; + } + }, + EXPAND_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + ExpandOp op = (ExpandOp) baseOp; + Optional direction = op.getDirection(); + if (!direction.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "direction", "not present"); + } + FfiDirection ffiDirection = (FfiDirection) direction.get().applyArg(); + Optional edgeOpt = op.getIsEdge(); + if (!edgeOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "edgeOpt", "not present"); + } + Boolean isEdge = (Boolean) edgeOpt.get().applyArg(); + Pointer expand = irCoreLib.initEdgexpdOperator(isEdge, ffiDirection); + + FfiError e1 = irCoreLib.setEdgexpdParams(expand, getQueryParams(op)); + if (e1.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + op.getClass(), "params", "setEdgexpdParams returns " + e1.msg); + } + // todo: add properties + // todo: add predicates + // todo: add limit + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent() && ClassUtils.equalClass(baseOp, ExpandOp.class)) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) aliasOpt.get().applyArg(); + irCoreLib.setEdgexpdAlias(expand, alias); + } + return expand; + } + + private Pointer getQueryParams(ExpandOp op) { + Pointer ptrParams = irCoreLib.initQueryParams(); + Optional labels = op.getLabels(); + if (labels.isPresent()) { + List ffiLabels = + (List) labels.get().applyArg(); + for (FfiNameOrId.ByValue label : ffiLabels) { + FfiError error = irCoreLib.addParamsTable(ptrParams, label); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + op.getClass(), "tables", "addParamsTable returns " + error.msg); + } + } + } + return ptrParams; + } + }, + LIMIT_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + LimitOp op = (LimitOp) baseOp; + Optional lower = op.getLower(); + if (!lower.isPresent()) { + throw new InterOpIllegalArgException(baseOp.getClass(), "lower", "not present"); + } + Optional upper = op.getUpper(); + if (!upper.isPresent()) { + throw new InterOpIllegalArgException(baseOp.getClass(), "upper", "not present"); + } + Pointer ptrLimit = irCoreLib.initLimitOperator(); + FfiError error = + irCoreLib.setLimitRange( + ptrLimit, + (Integer) lower.get().applyArg(), + (Integer) upper.get().applyArg()); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "lower+upper", "setLimitRange returns " + error.msg); + } + return ptrLimit; + } + }, + PROJECT_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + ProjectOp op = (ProjectOp) baseOp; + Optional exprOpt = op.getExprWithAlias(); + if (!exprOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "exprWithAlias", "not present"); + } + List exprWithAlias = (List) exprOpt.get().applyArg(); + // append always and sink by parameters + Pointer ptrProject = irCoreLib.initProjectOperator(true); + exprWithAlias.forEach( + p -> { + String expr = (String) p.getValue0(); + FfiAlias.ByValue alias = (FfiAlias.ByValue) p.getValue1(); + FfiError error = irCoreLib.addProjectExprAlias(ptrProject, expr, alias); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + baseOp.getClass(), + "exprWithAlias", + "append returns " + error.msg); + } + }); + return ptrProject; + } + }, + ORDER_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + OrderOp op = (OrderOp) baseOp; + Optional varWithOpt = op.getOrderVarWithOrder(); + if (!varWithOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "varWithOrder", "not present"); + } + List orderList = (List) varWithOpt.get().applyArg(); + if (orderList.isEmpty()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "varWithOrder", "should not be empty"); + } + Pointer ptrOrder = irCoreLib.initOrderbyOperator(); + orderList.forEach( + pair -> { + FfiVariable.ByValue var = (FfiVariable.ByValue) pair.getValue0(); + FfiOrderOpt opt = (FfiOrderOpt) pair.getValue1(); + irCoreLib.addOrderbyPair(ptrOrder, var, opt); + }); + // top k + Optional lower = op.getLower(); + Optional upper = op.getUpper(); + if (lower.isPresent() && upper.isPresent()) { + irCoreLib.setOrderbyLimit( + ptrOrder, + (Integer) lower.get().applyArg(), + (Integer) upper.get().applyArg()); + } + return ptrOrder; + } + }, + GROUP_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + Pointer ptrGroup = irCoreLib.initGroupbyOperator(); + GroupOp op = (GroupOp) baseOp; + Optional groupKeysOpt = op.getGroupByKeys(); + if (!groupKeysOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "groupKeys", "not present"); + } + Optional groupValuesOpt = op.getGroupByValues(); + if (!groupValuesOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "groupValues", "not present"); + } + // groupKeys is empty -> count + List groupKeys = (List) groupKeysOpt.get().applyArg(); + // set group key + groupKeys.forEach( + p -> { + FfiVariable.ByValue key = (FfiVariable.ByValue) p.getValue0(); + FfiAlias.ByValue alias = (FfiAlias.ByValue) p.getValue1(); + irCoreLib.addGroupbyKeyAlias(ptrGroup, key, alias); + }); + List groupValues = (List) groupValuesOpt.get().applyArg(); + if (groupValues.isEmpty()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "groupValues", "not present"); + } + // set group value + groupValues.forEach( + p -> { + irCoreLib.addGroupbyAggFn(ptrGroup, ArgUtils.asFfiAggFn(p)); + }); + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), + "groupKeys+groupValues", + "the query given alias is unsupported"); + } + return ptrGroup; + } + }, + DEDUP_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + Pointer ptrDedup = irCoreLib.initDedupOperator(); + DedupOp op = (DedupOp) baseOp; + Optional dedupKeysOpt = op.getDedupKeys(); + if (!dedupKeysOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "dedupKeys", "not present"); + } + List dedupKeys = + (List) dedupKeysOpt.get().applyArg(); + if (dedupKeys.isEmpty()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "dedupKeys", "should not be empty if present"); + } + dedupKeys.forEach( + k -> { + irCoreLib.addDedupKey(ptrDedup, k); + }); + return ptrDedup; + } + }, + SINK_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + SinkOp sinkOp = (SinkOp) baseOp; + Optional argOpt = sinkOp.getSinkArg(); + if (!argOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "sinkArg", "not present"); + } + SinkArg sinkArg = (SinkArg) argOpt.get().applyArg(); + List columns = sinkArg.getColumnNames(); + if (columns.isEmpty()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "selected columns", "is empty"); + } + Pointer ptrSink = irCoreLib.initSinkOperator(); + columns.forEach( + column -> { + FfiError error = irCoreLib.addSinkColumn(ptrSink, column); + if (error.code != ResultCode.Success) { + throw new InterOpIllegalArgException( + baseOp.getClass(), + "columns", + "addSinkColumn returns " + error.msg); + } + }); + return ptrSink; + } + }, + PATH_EXPAND_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + PathExpandOp pathOp = (PathExpandOp) baseOp; + int lower = (Integer) pathOp.getLower().get().applyArg(); + int upper = (Integer) pathOp.getUpper().get().applyArg(); + + Pointer expand = EXPAND_OP.apply(baseOp); + // todo: make isWholePath configurable + Pointer pathExpand = irCoreLib.initPathxpdOperator(expand, false); + irCoreLib.setPathxpdHops(pathExpand, lower, upper); + + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent()) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) aliasOpt.get().applyArg(); + irCoreLib.setPathxpdAlias(pathExpand, alias); + } + return pathExpand; + } + }, + GETV_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + GetVOp getVOp = (GetVOp) baseOp; + Optional vOpt = getVOp.getGetVOpt(); + if (!vOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "getVOpt", "not present"); + } + FfiVOpt ffiVOpt = (FfiVOpt) vOpt.get().applyArg(); + Pointer ptrGetV = irCoreLib.initGetvOperator(ffiVOpt); + + // set alias + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent()) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) aliasOpt.get().applyArg(); + irCoreLib.setGetvAlias(ptrGetV, alias); + } + return ptrGetV; + } + }, + APPLY_OP { + @Override + public Pointer apply(InterOpBase baseOp) { + ApplyOp applyOp = (ApplyOp) baseOp; + Optional subRootOpt = applyOp.getSubRootId(); + if (!subRootOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "subRootId", "not present"); + } + Optional joinKindOpt = applyOp.getJoinKind(); + if (!joinKindOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "joinKind", "not present"); + } + int subRootId = (Integer) subRootOpt.get().applyArg(); + FfiJoinKind joinKind = (FfiJoinKind) joinKindOpt.get().applyArg(); + Pointer ptrApply = irCoreLib.initApplyOperator(subRootId, joinKind); + + // set alias + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent()) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) aliasOpt.get().applyArg(); + irCoreLib.setApplyAlias(ptrApply, alias); + } + return ptrApply; + } + }, + UNION_OP { + public Pointer apply(InterOpBase baseOp) { + UnionOp unionOp = (UnionOp) baseOp; + Optional parentIdsOpt = unionOp.getParentIdList(); + if (!parentIdsOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "parentIdList", "not present"); + } + List parentIds = (List) parentIdsOpt.get().applyArg(); + if (parentIds.isEmpty()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "parentIdList", "is empty"); + } + Pointer ptrUnion = irCoreLib.initUnionOperator(); + parentIds.forEach( + id -> { + irCoreLib.addUnionParent(ptrUnion, id); + }); + return ptrUnion; + } + }, + MATCH_OP { + public Pointer apply(InterOpBase baseOp) { + MatchOp matchOp = (MatchOp) baseOp; + List sentences = + (List) matchOp.getSentences().get().applyArg(); + if (sentences.isEmpty()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "sentences", "is empty"); + } + Pointer ptrMatch = irCoreLib.initPatternOperator(); + sentences.forEach( + s -> { + InterOpCollection ops = s.getBinders(); + Pointer ptrSentence = irCoreLib.initPatternSentence(s.getJoinKind()); + irCoreLib.setSentenceStart(ptrSentence, s.getStartTag().alias); + irCoreLib.setSentenceEnd(ptrSentence, s.getEndTag().alias); + ops.unmodifiableCollection() + .forEach( + o -> { + Pointer binder; + FfiBinderOpt opt; + if (Utils.equalClass(o, ExpandOp.class)) { + binder = EXPAND_OP.apply(o); + opt = FfiBinderOpt.Edge; + } else if (Utils.equalClass( + o, PathExpandOp.class)) { + binder = PATH_EXPAND_OP.apply(o); + opt = FfiBinderOpt.Path; + } else if (Utils.equalClass(o, GetVOp.class)) { + binder = GETV_OP.apply(o); + opt = FfiBinderOpt.Vertex; + } else { + throw new InterOpIllegalArgException( + baseOp.getClass(), + "sentences", + "binder " + + o.getClass() + + " is unsupported yet"); + } + irCoreLib.addSentenceBinder( + ptrSentence, binder, opt); + }); + irCoreLib.addPatternSentence(ptrMatch, ptrSentence); + }); + Optional aliasOpt = baseOp.getAlias(); + if (aliasOpt.isPresent()) { + throw new InterOpIllegalArgException( + baseOp.getClass(), "match", "the query given alias is unsupported"); + } + return ptrMatch; + } + } + } + + public IrPlan() { + this.ptrPlan = irCoreLib.initLogicalPlan(); + } + + @Override + public void close() { + if (ptrPlan != null) { + irCoreLib.destroyLogicalPlan(ptrPlan); + } + } + + public String getPlanAsJson() throws IOException { + String json = ""; + if (ptrPlan != null) { + File file = new File(PLAN_JSON_FILE); + if (file.exists()) { + file.delete(); + } + irCoreLib.write_plan_to_json(ptrPlan, PLAN_JSON_FILE); + json = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + if (file.exists()) { + file.delete(); + } + } + return json; + } + + // return id of the first operator, id of the last operator + public Pair appendInterOpCollection( + int parentId, InterOpCollection opCollection) { + int subTaskRootId = 0; + int unionParentId = 0; + IntByReference oprId = new IntByReference(parentId); + List opList = opCollection.unmodifiableCollection(); + for (int i = 0; i < opList.size(); ++i) { + InterOpBase op = opList.get(i); + oprId = appendInterOp(oprId.getValue(), op); + if (i == 0) { + subTaskRootId = oprId.getValue(); + } + unionParentId = oprId.getValue(); + } + return Pair.with(subTaskRootId, unionParentId); + } + + // return id of the current operator + public IntByReference appendInterOp(int parentId, InterOpBase base) + throws InterOpIllegalArgException, InterOpUnsupportedException, AppendInterOpException { + FfiError error; + IntByReference oprId = new IntByReference(parentId); + if (ClassUtils.equalClass(base, ScanFusionOp.class)) { + Pointer ptrScan = TransformFactory.SCAN_FUSION_OP.apply(base); + error = irCoreLib.appendScanOperator(ptrPlan, ptrScan, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, SelectOp.class)) { + Pointer ptrSelect = TransformFactory.SELECT_OP.apply(base); + error = irCoreLib.appendSelectOperator(ptrPlan, ptrSelect, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, ExpandOp.class)) { + Pointer ptrExpand = TransformFactory.EXPAND_OP.apply(base); + error = irCoreLib.appendEdgexpdOperator(ptrPlan, ptrExpand, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, LimitOp.class)) { + Pointer ptrLimit = TransformFactory.LIMIT_OP.apply(base); + error = irCoreLib.appendLimitOperator(ptrPlan, ptrLimit, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, ProjectOp.class)) { + Pointer ptrProject = TransformFactory.PROJECT_OP.apply(base); + error = irCoreLib.appendProjectOperator(ptrPlan, ptrProject, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, OrderOp.class)) { + Pointer ptrOrder = TransformFactory.ORDER_OP.apply(base); + error = irCoreLib.appendOrderbyOperator(ptrPlan, ptrOrder, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, GroupOp.class)) { + Pointer ptrGroup = TransformFactory.GROUP_OP.apply(base); + error = irCoreLib.appendGroupbyOperator(ptrPlan, ptrGroup, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, DedupOp.class)) { + Pointer ptrDedup = TransformFactory.DEDUP_OP.apply(base); + error = irCoreLib.appendDedupOperator(ptrPlan, ptrDedup, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, SinkOp.class)) { + Pointer ptrSink = TransformFactory.SINK_OP.apply(base); + error = irCoreLib.appendSinkOperator(ptrPlan, ptrSink, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, PathExpandOp.class)) { + Pointer ptrPathXPd = TransformFactory.PATH_EXPAND_OP.apply(base); + error = irCoreLib.appendPathxpdOperator(ptrPlan, ptrPathXPd, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, GetVOp.class)) { + Pointer ptrGetV = TransformFactory.GETV_OP.apply(base); + error = irCoreLib.appendGetvOperator(ptrPlan, ptrGetV, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, ApplyOp.class)) { + ApplyOp applyOp = (ApplyOp) base; + Optional subOps = applyOp.getSubOpCollection(); + if (!subOps.isPresent()) { + throw new InterOpIllegalArgException( + base.getClass(), "subOpCollection", "is not present in apply"); + } + InterOpCollection opCollection = (InterOpCollection) subOps.get().applyArg(); + Pair oprIdPair = appendInterOpCollection(-1, opCollection); + applyOp.setSubRootId( + new OpArg(Integer.valueOf(oprIdPair.getValue0()), Function.identity())); + + Pointer ptrApply = TransformFactory.APPLY_OP.apply(base); + error = irCoreLib.appendApplyOperator(ptrPlan, ptrApply, oprId.getValue(), oprId); + } else if (ClassUtils.equalClass(base, UnionOp.class)) { + UnionOp unionOp = (UnionOp) base; + Optional subOpsListOpt = unionOp.getSubOpCollectionList(); + if (!subOpsListOpt.isPresent()) { + throw new InterOpIllegalArgException( + base.getClass(), "subOpCollectionList", "is not present in union"); + } + List subOpsList = + (List) subOpsListOpt.get().applyArg(); + if (subOpsList.isEmpty()) { + throw new InterOpIllegalArgException( + base.getClass(), "subOpCollectionList", "is empty in union"); + } + List unionParentIds = new ArrayList<>(); + for (InterOpCollection opCollection : subOpsList) { + Pair oprIdPair = appendInterOpCollection(parentId, opCollection); + unionParentIds.add(oprIdPair.getValue1()); + } + unionOp.setParentIdList(new OpArg(unionParentIds, Function.identity())); + Pointer ptrUnion = TransformFactory.UNION_OP.apply(base); + error = irCoreLib.appendUnionOperator(ptrPlan, ptrUnion, oprId); + } else if (ClassUtils.equalClass(base, MatchOp.class)) { + Pointer ptrMatch = TransformFactory.MATCH_OP.apply(base); + error = irCoreLib.appendPatternOperator(ptrPlan, ptrMatch, oprId.getValue(), oprId); + } else { + throw new InterOpUnsupportedException(base.getClass(), "unimplemented yet"); + } + if (error != null && error.code != ResultCode.Success) { + throw new AppendInterOpException( + base.getClass(), error.code.name() + ", msg is " + error.msg); + } + // add alias after the op if necessary + return setPostAlias(oprId.getValue(), base); + } + + private IntByReference setPostAlias(int parentId, InterOpBase base) { + IntByReference oprId = new IntByReference(parentId); + if (isPostAliasOp(base) && base.getAlias().isPresent()) { + FfiAlias.ByValue ffiAlias = (FfiAlias.ByValue) base.getAlias().get().applyArg(); + Pointer ptrAs = irCoreLib.initAsOperator(); + FfiError error = irCoreLib.setAsAlias(ptrAs, ffiAlias); + if (error != null && error.code != ResultCode.Success) { + throw new AppendInterOpException(base.getClass(), error.msg); + } + FfiError appendOp = irCoreLib.appendAsOperator(ptrPlan, ptrAs, parentId, oprId); + if (appendOp != null && appendOp.code != ResultCode.Success) { + throw new AppendInterOpException(base.getClass(), appendOp.msg); + } + } + return oprId; + } + + private boolean isPostAliasOp(InterOpBase base) { + return base instanceof SelectOp + || base instanceof LimitOp + || base instanceof OrderOp + || base instanceof DedupOp + || base instanceof UnionOp; + } + + public byte[] toPhysicalBytes(Configs configs) throws BuildPhysicalException { + if (ptrPlan == null) { + throw new BuildPhysicalException("ptrPlan is NullPointer"); + } + // hack way to notify shuffle + int servers = PegasusConfig.PEGASUS_HOSTS.get(configs).split(",").length; + int workers = PegasusConfig.PEGASUS_WORKER_NUM.get(configs); + FfiData.ByValue buffer = irCoreLib.buildPhysicalPlan(ptrPlan, workers, servers); + FfiError error = buffer.error; + if (error.code != ResultCode.Success) { + throw new BuildPhysicalException( + "call libc returns " + error.code.name() + ", msg is " + error.msg); + } + byte[] bytes = buffer.getBytes(); + buffer.close(); + return bytes; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/HostsChannelFetcher.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/HostsChannelFetcher.java new file mode 100644 index 000000000000..3e69bfb5033b --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/HostsChannelFetcher.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.client; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.pegasus.RpcChannel; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class HostsChannelFetcher implements RpcChannelFetcher { + private Configs config; + + public HostsChannelFetcher(Configs config) { + this.config = config; + } + + @Override + public List fetch() { + String hosts = PegasusConfig.PEGASUS_HOSTS.get(config); + String[] hostsArr = hosts.split(","); + List hostAddresses = Arrays.asList(hostsArr); + List rpcChannels = new ArrayList<>(); + hostAddresses.forEach( + k -> { + String[] host = k.split(":"); + rpcChannels.add(new RpcChannel(host[0], Integer.valueOf(host[1]))); + }); + return rpcChannels; + } + + @Override + public boolean isDynamic() { + return false; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/ResultParser.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/ResultParser.java new file mode 100644 index 000000000000..3207d6263a25 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/ResultParser.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.client; + +import com.alibaba.pegasus.service.protocol.PegasusClient; + +import java.util.List; + +public interface ResultParser { + List parseFrom(PegasusClient.JobResponse response); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/RpcBroadcastProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/RpcBroadcastProcessor.java new file mode 100644 index 000000000000..961400497c32 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/RpcBroadcastProcessor.java @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.client; + +import com.alibaba.pegasus.RpcClient; +import com.alibaba.pegasus.intf.CloseableIterator; +import com.alibaba.pegasus.intf.ResultProcessor; +import com.alibaba.pegasus.service.protocol.PegasusClient; + +import io.grpc.Status; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class RpcBroadcastProcessor implements AutoCloseable { + private static final Logger logger = LoggerFactory.getLogger(RpcBroadcastProcessor.class); + + protected RpcClient rpcClient; + protected RpcChannelFetcher fetcher; + + public RpcBroadcastProcessor(RpcChannelFetcher fetcher) { + this.fetcher = fetcher; + if (!fetcher.isDynamic()) { + this.rpcClient = new RpcClient(fetcher.fetch()); + } + } + + public void broadcast(PegasusClient.JobRequest request, ResultProcessor processor) { + CloseableIterator iterator = null; + try { + if (fetcher.isDynamic()) { + this.rpcClient = new RpcClient(fetcher.fetch()); + } + iterator = rpcClient.submit(request); + // process response + while (iterator.hasNext()) { + PegasusClient.JobResponse response = iterator.next(); + processor.process(response); + } + processor.finish(); + } catch (Exception e) { + logger.error("broadcast exception {}", e); + processor.error(Status.fromThrowable(e)); + } finally { + if (iterator != null) { + try { + iterator.close(); + } catch (IOException ioe) { + logger.error("iterator close fail {}", ioe); + } + } + } + } + + @Override + public void close() throws Exception { + this.rpcClient.shutdown(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/RpcChannelFetcher.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/RpcChannelFetcher.java new file mode 100644 index 000000000000..daf79cbb7895 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/client/RpcChannelFetcher.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.client; + +import com.alibaba.pegasus.RpcChannel; + +import java.util.List; + +public interface RpcChannelFetcher { + List fetch(); + + // dynamic channel need update with the changes of host access url + boolean isDynamic(); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/Config.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/Config.java new file mode 100644 index 000000000000..82b4bb7e339a --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/Config.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.config; + +import java.util.function.Function; + +public class Config { + private String key; + private String defaultVal; + + private Function parseFunc; + + public Config(String key, String defaultVal, Function parseFunc) { + this.key = key; + this.defaultVal = defaultVal; + this.parseFunc = parseFunc; + } + + public T get(Configs configs) { + String valStr = configs.get(key, defaultVal); + try { + T val = parseFunc.apply(valStr); + return val; + } catch (Exception e) { + throw new IllegalArgumentException( + "key [" + key + "] val [" + valStr + "] parse failed", e); + } + } + + public static Config shortConfig(String key, short defaultVal) { + return new Config<>(key, String.valueOf(defaultVal), (s) -> Short.parseShort(s)); + } + + public static Config intConfig(String key, int defaultVal) { + return new Config<>(key, String.valueOf(defaultVal), (s) -> Integer.parseInt(s)); + } + + public static Config longConfig(String key, long defaultVal) { + return new Config<>(key, String.valueOf(defaultVal), (s) -> Long.parseLong(s)); + } + + public static Config stringConfig(String key, String defaultVal) { + return new Config<>(key, defaultVal, Function.identity()); + } + + public static Config boolConfig(String key, boolean defaultVal) { + return new Config<>(key, String.valueOf(defaultVal), (s) -> Boolean.parseBoolean(s)); + } + + public String getKey() { + return this.key; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/Configs.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/Configs.java new file mode 100644 index 000000000000..f53c3b24fc0d --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/Configs.java @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.config; + +import org.apache.commons.lang3.NotImplementedException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; + +public class Configs { + private Properties properties; + + public Configs(String file, FileLoadType loadType) throws IOException, NotImplementedException { + properties = new Properties(); + switch (loadType) { + case RELATIVE_PATH: + properties.load(new FileInputStream(new File(file))); + break; + default: + throw new NotImplementedException("unimplemented load type " + loadType); + } + // replace with the value from system property + properties + .keySet() + .forEach( + k -> { + String value = System.getProperty((String) k); + String trimValue; + if (value != null && !(trimValue = value.trim()).isEmpty()) { + properties.setProperty((String) k, trimValue); + } + }); + } + + public Configs(Map configs) { + this.properties = new Properties(); + if (configs != null && !configs.isEmpty()) { + configs.forEach( + (k, v) -> { + this.properties.setProperty(k, v); + }); + } + } + + public String get(String name, String defaultValue) { + return this.properties.getProperty(name, defaultValue); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/FileLoadType.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/FileLoadType.java new file mode 100644 index 000000000000..dc4af7901ff0 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/FileLoadType.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.config; + +public enum FileLoadType { + // read file from relative path + RELATIVE_PATH, + // read file from resources + RESOURCES +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java new file mode 100644 index 000000000000..72788631cacb --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java @@ -0,0 +1,5 @@ +package com.alibaba.graphscope.common.config; + +public class GraphConfig { + public static final Config GRAPH_SCHEMA = Config.stringConfig("graph.schema", "."); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/PegasusConfig.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/PegasusConfig.java new file mode 100644 index 000000000000..a0b14a954785 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/config/PegasusConfig.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.config; + +public class PegasusConfig { + public static final Config PEGASUS_WORKER_NUM = + Config.intConfig("pegasus.worker.num", 1); + + public static final Config PEGASUS_TIMEOUT = + Config.intConfig("pegasus.timeout", 240000); + + public static final Config PEGASUS_BATCH_SIZE = + Config.intConfig("pegasus.batch.size", 1024); + + public static final Config PEGASUS_OUTPUT_CAPACITY = + Config.intConfig("pegasus.output.capacity", 16); + + public static final Config PEGASUS_MEMORY_LIMIT = + Config.intConfig("pegasus.memory.limit", Integer.MAX_VALUE); + + public static final Config PEGASUS_HOSTS = + Config.stringConfig("pegasus.hosts", "localhost:8080"); + + public static final Config PEGASUS_SERVER_NUM = + Config.intConfig("pegasus.server.num", 1); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/AppendInterOpException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/AppendInterOpException.java new file mode 100644 index 000000000000..a7c2aee9293f --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/AppendInterOpException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.exception; + +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; + +public class AppendInterOpException extends RuntimeException { + public AppendInterOpException(Class op, String msg) { + super(String.format("op type {%s} returns {%s}", op, msg)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/BuildPhysicalException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/BuildPhysicalException.java new file mode 100644 index 000000000000..caa9530c26d1 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/BuildPhysicalException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.exception; + +public class BuildPhysicalException extends RuntimeException { + public BuildPhysicalException(String error) { + super(error); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/InterOpIllegalArgException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/InterOpIllegalArgException.java new file mode 100644 index 000000000000..c204884c7ce9 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/InterOpIllegalArgException.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.exception; + +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; + +public class InterOpIllegalArgException extends IllegalArgumentException { + public InterOpIllegalArgException( + Class opType, String opName, String error) { + super(String.format("op type {%s} op name {%s} returns error {%s}", opType, opName, error)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/InterOpUnsupportedException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/InterOpUnsupportedException.java new file mode 100644 index 000000000000..5740fba17042 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/InterOpUnsupportedException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.exception; + +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; + +public class InterOpUnsupportedException extends UnsupportedOperationException { + public InterOpUnsupportedException(Class op, String cause) { + super(String.format("op type {%s} is unsupported, cause is {%s}", op, cause)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/OpArgIllegalException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/OpArgIllegalException.java new file mode 100644 index 000000000000..ebaa95424367 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/exception/OpArgIllegalException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.exception; + +public class OpArgIllegalException extends IllegalArgumentException { + public enum Cause { + UNSUPPORTED_TYPE, + INVALID_TYPE + } + + public OpArgIllegalException(Cause cause, String error) { + super(String.format("cause is {%s} error is {%s}", cause, error)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgAggFn.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgAggFn.java new file mode 100644 index 000000000000..8d3d0fd754dc --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgAggFn.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate; + +import com.alibaba.graphscope.common.jna.type.FfiAggOpt; +import com.alibaba.graphscope.common.jna.type.FfiAlias; +import com.alibaba.graphscope.common.jna.type.FfiVariable; +import com.google.common.base.Objects; + +import java.util.ArrayList; +import java.util.List; + +// represent AggFn of Group Value, as a variable of GroupOp +public class ArgAggFn { + private List vars; + private FfiAggOpt aggregate; + private FfiAlias.ByValue alias; + + public ArgAggFn(FfiAggOpt aggregate, FfiAlias.ByValue alias) { + this.aggregate = aggregate; + this.alias = alias; + this.vars = new ArrayList<>(); + } + + public void addVar(FfiVariable.ByValue var) { + this.vars.add(var); + } + + public List getVars() { + return vars; + } + + public FfiAggOpt getAggregate() { + return aggregate; + } + + public FfiAlias.ByValue getAlias() { + return alias; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArgAggFn argAggFn = (ArgAggFn) o; + return Objects.equal(vars, argAggFn.vars) + && aggregate == argAggFn.aggregate + && Objects.equal(alias, argAggFn.alias); + } + + @Override + public int hashCode() { + return Objects.hashCode(vars, aggregate, alias); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgUtils.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgUtils.java new file mode 100644 index 000000000000..55e04c051913 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgUtils.java @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate; + +import com.alibaba.graphscope.common.exception.OpArgIllegalException; +import com.alibaba.graphscope.common.jna.IrCoreLibrary; +import com.alibaba.graphscope.common.jna.type.*; + +public class ArgUtils { + private static IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE; + private static String LABEL = "~label"; + private static String ID = "~id"; + private static String LEN = "~len"; + + public static FfiConst.ByValue asFfiConst(int id) { + return irCoreLib.int32AsConst(id); + } + + public static FfiConst.ByValue asFfiConst(long id) { + return irCoreLib.int64AsConst(id); + } + + // "" indicates NONE + public static FfiProperty.ByValue asFfiProperty(String property) { + if (property.isEmpty()) { + return irCoreLib.asNoneKey(); + } else if (property.equals(LABEL)) { + return irCoreLib.asLabelKey(); + } else if (property.equals(ID)) { + return irCoreLib.asIdKey(); + } else if (property.equals(LEN)) { + return irCoreLib.asLenKey(); + } else { + return irCoreLib.asPropertyKey(irCoreLib.cstrAsNameOrId(property)); + } + } + + // "" indicates NONE or HEAD + public static FfiNameOrId.ByValue asFfiTag(String tag) { + if (tag.isEmpty()) { + return irCoreLib.noneNameOrId(); + } else { + return irCoreLib.cstrAsNameOrId(tag); + } + } + + public static FfiNameOrId.ByValue asFfiNoneTag() { + return irCoreLib.noneNameOrId(); + } + + public static FfiVariable.ByValue asFfiVar(String tag, String property) { + FfiNameOrId.ByValue ffiTag = asFfiTag(tag); + FfiProperty.ByValue ffiProperty = asFfiProperty(property); + return irCoreLib.asVar(ffiTag, ffiProperty); + } + + public static FfiVariable.ByValue asFfiNoneVar() { + return irCoreLib.asNoneVar(); + } + + public static FfiAlias.ByValue asFfiNoneAlias() { + FfiNameOrId.ByValue alias = irCoreLib.noneNameOrId(); + FfiAlias.ByValue ffiAlias = new FfiAlias.ByValue(); + ffiAlias.alias = alias; + ffiAlias.isQueryGiven = false; + return ffiAlias; + } + + public static FfiAlias.ByValue asFfiAlias(String aliasName, boolean isQueryGiven) { + FfiNameOrId.ByValue alias = irCoreLib.cstrAsNameOrId(aliasName); + FfiAlias.ByValue ffiAlias = new FfiAlias.ByValue(); + ffiAlias.alias = alias; + ffiAlias.isQueryGiven = isQueryGiven; + return ffiAlias; + } + + public static FfiAggFn.ByValue asFfiAggFn(ArgAggFn aggFn) { + FfiAggFn.ByValue ffiAggFn = irCoreLib.initAggFn(aggFn.getAggregate(), aggFn.getAlias()); + // todo: add var + return ffiAggFn; + } + + public static String propertyName(FfiProperty.ByValue property) { + switch (property.opt) { + case None: + return ""; + case Id: + return ID; + case Label: + return LABEL; + case Len: + return LEN; + case Key: + return property.key.name; + default: + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, "invalid type"); + } + } + + public static String tagName(FfiNameOrId.ByValue tag) { + switch (tag.opt) { + case None: + return ""; + case Name: + return tag.name; + case Id: + default: + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, "invalid type"); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/InterOpCollection.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/InterOpCollection.java new file mode 100644 index 000000000000..677f0833fb3e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/InterOpCollection.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.exception.InterOpIllegalArgException; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.OpArg; +import com.alibaba.graphscope.common.intermediate.process.InterOpProcessor; +import com.alibaba.graphscope.common.intermediate.process.SinkOutputProcessor; +import com.alibaba.graphscope.common.intermediate.strategy.InterOpStrategy; +import com.alibaba.graphscope.common.intermediate.strategy.TopKStrategy; + +import org.apache.commons.collections.list.UnmodifiableList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +// collection of intermediate operators +public class InterOpCollection { + private List opCollection; + private static List strategies = Arrays.asList(TopKStrategy.INSTANCE); + private static List processors = Arrays.asList(SinkOutputProcessor.INSTANCE); + + public InterOpCollection() { + opCollection = new ArrayList<>(); + } + + public IrPlan buildIrPlan() { + applyStrategies(this); + process(this); + IrPlan irPlan = new IrPlan(); + irPlan.appendInterOpCollection(-1, this); + return irPlan; + } + + public List unmodifiableCollection() { + return UnmodifiableList.decorate(this.opCollection); + } + + public void appendInterOp(InterOpBase op) { + this.opCollection.add(op); + } + + public void removeInterOp(int i) { + opCollection.remove(i); + } + + private void applyStrategies(InterOpCollection opCollection) { + opCollection + .unmodifiableCollection() + .forEach( + op -> { + if (op instanceof ApplyOp) { + ApplyOp applyOp = (ApplyOp) op; + Optional subOps = applyOp.getSubOpCollection(); + if (!subOps.isPresent()) { + throw new InterOpIllegalArgException( + op.getClass(), + "subOpCollection", + "is not present in apply"); + } + InterOpCollection subCollection = + (InterOpCollection) subOps.get().applyArg(); + applyStrategies(subCollection); + } + }); + strategies.forEach(k -> k.apply(opCollection)); + } + + private void process(InterOpCollection opCollection) { + // only traverse root opCollection for SinkOutputProcessor + processors.forEach(k -> k.process(opCollection)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/MatchSentence.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/MatchSentence.java new file mode 100644 index 000000000000..54dc9d72f9e0 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/MatchSentence.java @@ -0,0 +1,35 @@ +package com.alibaba.graphscope.common.intermediate; + +import com.alibaba.graphscope.common.jna.type.FfiAlias; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; + +public class MatchSentence { + private FfiAlias.ByValue startTag; + private FfiAlias.ByValue endTag; + private InterOpCollection binders; + private FfiJoinKind joinKind; + + public MatchSentence( + String startTag, String endTag, InterOpCollection binders, FfiJoinKind joinKind) { + this.startTag = ArgUtils.asFfiAlias(startTag, true); + this.endTag = ArgUtils.asFfiAlias(endTag, true); + this.joinKind = joinKind; + this.binders = binders; + } + + public FfiAlias.ByValue getStartTag() { + return startTag; + } + + public FfiAlias.ByValue getEndTag() { + return endTag; + } + + public InterOpCollection getBinders() { + return binders; + } + + public FfiJoinKind getJoinKind() { + return joinKind; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ApplyOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ApplyOp.java new file mode 100644 index 000000000000..fd41fc57e334 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ApplyOp.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class ApplyOp extends InterOpBase { + // InterOpCollection + private Optional subOpCollection; + + // Integer + private Optional subRootId; + + // FfiJoinKind + private Optional joinKind; + + public ApplyOp() { + super(); + subOpCollection = Optional.empty(); + subRootId = Optional.empty(); + joinKind = Optional.empty(); + } + + public Optional getSubOpCollection() { + return subOpCollection; + } + + public Optional getSubRootId() { + return subRootId; + } + + public Optional getJoinKind() { + return joinKind; + } + + public void setSubOpCollection(OpArg subOpCollection) { + this.subOpCollection = Optional.of(subOpCollection); + } + + public void setSubRootId(OpArg subRootId) { + this.subRootId = Optional.of(subRootId); + } + + public void setJoinKind(OpArg joinKind) { + this.joinKind = Optional.of(joinKind); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/DedupOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/DedupOp.java new file mode 100644 index 000000000000..77245e500cd9 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/DedupOp.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class DedupOp extends InterOpBase { + // list + private Optional dedupKeys; + + public DedupOp() { + super(); + dedupKeys = Optional.empty(); + } + + public Optional getDedupKeys() { + return dedupKeys; + } + + public void setDedupKeys(OpArg dedupKeys) { + this.dedupKeys = Optional.of(dedupKeys); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOp.java new file mode 100644 index 000000000000..8ffc45b08a83 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOp.java @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class ExpandOp extends InterOpBase { + public ExpandOp() { + super(); + this.isEdge = Optional.empty(); + this.direction = Optional.empty(); + this.labels = Optional.empty(); + this.properties = Optional.empty(); + this.predicate = Optional.empty(); + this.limit = Optional.empty(); + } + + // out or outE + private Optional isEdge; + + // in/out/both + private Optional direction; + + // filter edge by labels + private Optional labels; + + // filter edge by properties + private Optional properties; + + // filter edge by predicates + private Optional predicate; + + private Optional limit; + + public Optional getDirection() { + return direction; + } + + public Optional getLabels() { + return labels; + } + + public Optional getProperties() { + return properties; + } + + public Optional getPredicate() { + return predicate; + } + + public Optional getLimit() { + return limit; + } + + public void setDirection(OpArg direction) { + this.direction = Optional.of(direction); + } + + public void setLabels(OpArg labels) { + this.labels = Optional.of(labels); + } + + public void setProperties(OpArg properties) { + this.properties = Optional.of(properties); + } + + public void setPredicate(OpArg predicate) { + this.predicate = Optional.of(predicate); + } + + public void setLimit(OpArg limit) { + this.limit = Optional.of(limit); + } + + public Optional getIsEdge() { + return isEdge; + } + + public void setEdgeOpt(OpArg isEdge) { + this.isEdge = Optional.of(isEdge); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/GetVOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/GetVOp.java new file mode 100644 index 000000000000..5dc49067b626 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/GetVOp.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class GetVOp extends InterOpBase { + // FfiVOpt + private Optional getVOpt; + + public GetVOp() { + super(); + getVOpt = Optional.empty(); + } + + public Optional getGetVOpt() { + return getVOpt; + } + + public void setGetVOpt(OpArg getVOpt) { + this.getVOpt = Optional.of(getVOpt); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/GroupOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/GroupOp.java new file mode 100644 index 000000000000..85db6728c37a --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/GroupOp.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class GroupOp extends InterOpBase { + // List of Pair + private Optional groupByKeys; + + // List + private Optional groupByValues; + + public GroupOp() { + super(); + groupByKeys = Optional.empty(); + groupByValues = Optional.empty(); + } + + public Optional getGroupByKeys() { + return groupByKeys; + } + + public Optional getGroupByValues() { + return groupByValues; + } + + public void setGroupByKeys(OpArg groupByKeys) { + this.groupByKeys = Optional.of(groupByKeys); + } + + public void setGroupByValues(OpArg groupByValues) { + this.groupByValues = Optional.of(groupByValues); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/InterOpBase.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/InterOpBase.java new file mode 100644 index 000000000000..1d7032cd4e58 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/InterOpBase.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public abstract class InterOpBase { + // set tag to store the intermediate result + // FfiAlias + private Optional alias; + + public InterOpBase() { + this.alias = Optional.empty(); + } + + public Optional getAlias() { + return alias; + } + + public void setAlias(OpArg alias) { + this.alias = Optional.of(alias); + } + + public void clearAlias() { + this.alias = Optional.empty(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/LimitOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/LimitOp.java new file mode 100644 index 000000000000..c64f9194578e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/LimitOp.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class LimitOp extends InterOpBase { + private Optional lower; + + private Optional upper; + + public LimitOp() { + super(); + this.lower = Optional.empty(); + this.upper = Optional.empty(); + } + + public Optional getLower() { + return lower; + } + + public void setLower(OpArg lower) { + this.lower = Optional.of(lower); + } + + public Optional getUpper() { + return upper; + } + + public void setUpper(OpArg upper) { + this.upper = Optional.of(upper); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/MatchOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/MatchOp.java new file mode 100644 index 000000000000..218959b2c2f1 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/MatchOp.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class MatchOp extends InterOpBase { + // List + private Optional sentences; + + public MatchOp() { + super(); + this.sentences = Optional.empty(); + } + + public Optional getSentences() { + return sentences; + } + + public void setSentences(OpArg sentences) { + this.sentences = Optional.of(sentences); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/OpArg.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/OpArg.java new file mode 100644 index 000000000000..53d99ba879f3 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/OpArg.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.function.Function; + +public class OpArg { + private T arg; + + private Function transform; + + public OpArg(T arg, Function transform) { + this.arg = arg; + this.transform = transform; + } + + public OpArg(T arg) { + this.arg = arg; + this.transform = (T t) -> (R) t; + } + + public T getArg() { + return arg; + } + + public R applyArg() { + return transform.apply(arg); + } + + public Function getTransform() { + return transform; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/OrderOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/OrderOp.java new file mode 100644 index 000000000000..e315e84dc2f6 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/OrderOp.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class OrderOp extends InterOpBase { + // List of pair + private Optional orderVarWithOrder; + + // top k + private Optional lower; + + private Optional upper; + + public OrderOp() { + super(); + this.orderVarWithOrder = Optional.empty(); + this.lower = Optional.empty(); + this.upper = Optional.empty(); + } + + public Optional getOrderVarWithOrder() { + return orderVarWithOrder; + } + + public void setOrderVarWithOrder(OpArg orderVarWithOrder) { + this.orderVarWithOrder = Optional.of(orderVarWithOrder); + } + + public Optional getLower() { + return lower; + } + + public void setLower(OpArg lower) { + this.lower = Optional.of(lower); + } + + public Optional getUpper() { + return upper; + } + + public void setUpper(OpArg upper) { + this.upper = Optional.of(upper); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOp.java new file mode 100644 index 000000000000..816ab83c8d86 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOp.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class PathExpandOp extends ExpandOp { + private Optional lower; + + private Optional upper; + + public PathExpandOp() { + super(); + lower = Optional.empty(); + upper = Optional.empty(); + } + + public PathExpandOp(ExpandOp other) { + this(); + if (other.getAlias().isPresent()) { + setAlias(other.getAlias().get()); + } + if (other.getIsEdge().isPresent()) { + setEdgeOpt(other.getIsEdge().get()); + } + if (other.getDirection().isPresent()) { + setDirection(other.getDirection().get()); + } + if (other.getLabels().isPresent()) { + setLabels(other.getLabels().get()); + } + if (other.getPredicate().isPresent()) { + setPredicate(other.getPredicate().get()); + } + if (other.getProperties().isPresent()) { + setProperties(other.getProperties().get()); + } + if (other.getLimit().isPresent()) { + setLimit(other.getLimit().get()); + } + } + + public Optional getLower() { + return lower; + } + + public Optional getUpper() { + return upper; + } + + public void setLower(OpArg lower) { + this.lower = Optional.of(lower); + } + + public void setUpper(OpArg upper) { + this.upper = Optional.of(upper); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ProjectOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ProjectOp.java new file mode 100644 index 000000000000..de6fe5c58c22 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ProjectOp.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.exception.InterOpIllegalArgException; + +import org.javatuples.Pair; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +public class ProjectOp extends InterOpBase { + // List of Pair + private Optional exprWithAlias; + + public ProjectOp() { + super(); + exprWithAlias = Optional.empty(); + } + + public Optional getExprWithAlias() { + if (exprWithAlias.isPresent()) { + Object arg = exprWithAlias.get().getArg(); + Function transform = exprWithAlias.get().getTransform(); + Function thenApply = + transform.andThen( + (Object o) -> { + List exprList = (List) o; + Optional aliasOpt = getAlias(); + if (aliasOpt.isPresent()) { + // replace with the query given alias + if (exprList.size() == 1) { + Pair firstEntry = exprList.get(0); + exprList.set( + 0, firstEntry.setAt1(aliasOpt.get().applyArg())); + } + if (exprList.size() > 1) { + throw new InterOpIllegalArgException( + getClass(), + "exprWithAlias", + "multiple columns as a single alias is" + + " unsupported"); + } + } + return exprList; + }); + setExprWithAlias(new OpArg(arg, thenApply)); + } + return exprWithAlias; + } + + public void setExprWithAlias(OpArg projectExpr) { + this.exprWithAlias = Optional.of(projectExpr); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ScanFusionOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ScanFusionOp.java new file mode 100644 index 000000000000..788d4436fa51 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ScanFusionOp.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class ScanFusionOp extends InterOpBase { + public ScanFusionOp() { + super(); + scanOpt = Optional.empty(); + labels = Optional.empty(); + properties = Optional.empty(); + ids = Optional.empty(); + predicate = Optional.empty(); + limit = Optional.empty(); + } + + // vertex or edge + private Optional scanOpt; + + // scan by labels + private Optional labels; + + // scan by properties + private Optional properties; + + // indexing based on id + private Optional ids; + + // filter by predicate + private Optional predicate; + + // filter by limit + private Optional limit; + + public Optional getScanOpt() { + return scanOpt; + } + + public Optional getLabels() { + return labels; + } + + public Optional getProperties() { + return properties; + } + + public Optional getIds() { + return ids; + } + + public Optional getPredicate() { + return predicate; + } + + public Optional getLimit() { + return limit; + } + + public void setScanOpt(OpArg scanOpt) { + this.scanOpt = Optional.of(scanOpt); + } + + public void setLabels(OpArg labels) { + this.labels = Optional.of(labels); + } + + public void setProperties(OpArg properties) { + this.properties = Optional.of(properties); + } + + public void setIds(OpArg ids) { + this.ids = Optional.of(ids); + } + + public void setPredicate(OpArg predicate) { + this.predicate = Optional.of(predicate); + } + + public void setLimit(OpArg limit) { + this.limit = Optional.of(limit); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SelectOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SelectOp.java new file mode 100644 index 000000000000..2bb1ee8c37bf --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SelectOp.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class SelectOp extends InterOpBase { + public SelectOp() { + super(); + this.predicate = Optional.empty(); + } + + private Optional predicate; + + public Optional getPredicate() { + return predicate; + } + + public void setPredicate(OpArg predicate) { + this.predicate = Optional.of(predicate); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SinkOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SinkOp.java new file mode 100644 index 000000000000..5bc73cdb8202 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SinkOp.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class SinkOp extends InterOpBase { + // SinkArg + private Optional sinkArg; + + public SinkOp() { + super(); + sinkArg = Optional.empty(); + } + + public Optional getSinkArg() { + return sinkArg; + } + + public void setSinkArg(OpArg sinkArg) { + this.sinkArg = Optional.of(sinkArg); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/UnionOp.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/UnionOp.java new file mode 100644 index 000000000000..911f61d2d509 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/UnionOp.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import java.util.Optional; + +public class UnionOp extends InterOpBase { + // List + private Optional subOpCollectionList; + + // List + private Optional parentIdList; + + public UnionOp() { + super(); + this.subOpCollectionList = Optional.empty(); + this.parentIdList = Optional.empty(); + } + + public Optional getSubOpCollectionList() { + return subOpCollectionList; + } + + public Optional getParentIdList() { + return parentIdList; + } + + public void setSubOpCollectionList(OpArg subOpCollectionList) { + this.subOpCollectionList = Optional.of(subOpCollectionList); + } + + public void setParentIdList(OpArg parentIdList) { + this.parentIdList = Optional.of(parentIdList); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/InterOpProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/InterOpProcessor.java new file mode 100644 index 000000000000..8e9946913b5e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/InterOpProcessor.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.process; + +import com.alibaba.graphscope.common.intermediate.InterOpCollection; + +public interface InterOpProcessor { + void process(InterOpCollection opCollection); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkArg.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkArg.java new file mode 100644 index 000000000000..1e209ef6a1e2 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkArg.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.process; + +import com.alibaba.graphscope.common.jna.type.FfiNameOrId; + +import java.util.*; + +public class SinkArg { + private List columnNames; + + public SinkArg() { + columnNames = new ArrayList<>(); + } + + public List getColumnNames() { + return Collections.unmodifiableList(columnNames); + } + + public void addColumnName(FfiNameOrId.ByValue columnName) { + this.columnNames.add(columnName); + } + + public void dedup() { + Set uniqueColumnNames = new HashSet<>(columnNames); + columnNames.clear(); + columnNames.addAll(uniqueColumnNames); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkOutputProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkOutputProcessor.java new file mode 100644 index 000000000000..96121cde8a48 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/process/SinkOutputProcessor.java @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.process; + +import com.alibaba.graphscope.common.exception.InterOpIllegalArgException; +import com.alibaba.graphscope.common.exception.InterOpUnsupportedException; +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.MatchSentence; +import com.alibaba.graphscope.common.intermediate.operator.*; +import com.alibaba.graphscope.common.jna.type.FfiAlias; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; + +import org.javatuples.Pair; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +public class SinkOutputProcessor implements InterOpProcessor { + public static SinkOutputProcessor INSTANCE = new SinkOutputProcessor(); + + private SinkOutputProcessor() {} + + @Override + public void process(InterOpCollection opCollection) { + SinkArg sinkArg = getSinkColumns(opCollection); + if (sinkArg != null && !sinkArg.getColumnNames().isEmpty()) { + SinkOp sinkOp = new SinkOp(); + sinkOp.setSinkArg(new OpArg(sinkArg, Function.identity())); + opCollection.appendInterOp(sinkOp); + } + } + + private SinkArg getSinkColumns(InterOpCollection opCollection) { + List collections = opCollection.unmodifiableCollection(); + SinkArg sinkArg = new SinkArg(); + for (int i = collections.size() - 1; i >= 0; --i) { + InterOpBase cur = collections.get(i); + if (cur instanceof DedupOp + || cur instanceof LimitOp + || cur instanceof OrderOp + || cur instanceof SelectOp) { + continue; + } else if (cur instanceof ExpandOp + || cur instanceof ScanFusionOp + || cur instanceof GetVOp) { + sinkArg.addColumnName(ArgUtils.asFfiNoneTag()); + break; + } else if (cur instanceof ProjectOp) { + ProjectOp op = (ProjectOp) cur; + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + for (Pair pair : exprWithAlias) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) pair.getValue1(); + sinkArg.addColumnName(alias.alias); + } + break; + } else if (cur instanceof GroupOp) { + GroupOp op = (GroupOp) cur; + List groupKeys = (List) op.getGroupByKeys().get().applyArg(); + for (Pair pair : groupKeys) { + FfiAlias.ByValue alias = (FfiAlias.ByValue) pair.getValue1(); + sinkArg.addColumnName(alias.alias); + } + List groupValues = + (List) op.getGroupByValues().get().applyArg(); + for (ArgAggFn aggFn : groupValues) { + sinkArg.addColumnName(aggFn.getAlias().alias); + } + break; + } else if (cur instanceof ApplyOp) { + ApplyOp applyOp = (ApplyOp) cur; + FfiJoinKind joinKind = (FfiJoinKind) applyOp.getJoinKind().get().applyArg(); + // where or not + // order().by(), select().by(), group().by() + if (joinKind == FfiJoinKind.Semi + || joinKind == FfiJoinKind.Anti + || joinKind == FfiJoinKind.Inner) { + continue; + } else { + throw new InterOpUnsupportedException( + cur.getClass(), "join kind is unsupported yet"); + } + } else if (cur instanceof UnionOp) { + UnionOp unionOp = (UnionOp) cur; + Optional subOpsListOpt = unionOp.getSubOpCollectionList(); + if (!subOpsListOpt.isPresent()) { + throw new InterOpIllegalArgException( + cur.getClass(), "subOpCollectionList", "is not present in union"); + } + List subOpsList = + (List) subOpsListOpt.get().applyArg(); + subOpsList.forEach( + op -> { + SinkArg subSink = getSinkColumns(op); + subSink.getColumnNames().forEach(c -> sinkArg.addColumnName(c)); + }); + sinkArg.dedup(); + break; + } else if (cur instanceof MatchOp) { + List sentences = + (List) ((MatchOp) cur).getSentences().get().applyArg(); + sentences.forEach( + s -> { + sinkArg.addColumnName(s.getStartTag().alias); + sinkArg.addColumnName(s.getEndTag().alias); + }); + sinkArg.dedup(); + break; + } else { + throw new InterOpUnsupportedException(cur.getClass(), "unimplemented yet"); + } + } + return sinkArg; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/strategy/InterOpStrategy.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/strategy/InterOpStrategy.java new file mode 100644 index 000000000000..2bed9d330055 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/strategy/InterOpStrategy.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.strategy; + +import com.alibaba.graphscope.common.intermediate.InterOpCollection; + +public interface InterOpStrategy { + void apply(InterOpCollection opCollection); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/strategy/TopKStrategy.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/strategy/TopKStrategy.java new file mode 100644 index 000000000000..d4e6c8731bd9 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/strategy/TopKStrategy.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.strategy; + +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.LimitOp; +import com.alibaba.graphscope.common.intermediate.operator.OpArg; +import com.alibaba.graphscope.common.intermediate.operator.OrderOp; + +import java.util.List; +import java.util.Optional; + +// fuse order with the following limit +public class TopKStrategy implements InterOpStrategy { + public static TopKStrategy INSTANCE = new TopKStrategy(); + + private TopKStrategy() {} + + @Override + public void apply(InterOpCollection opCollection) { + List original = opCollection.unmodifiableCollection(); + for (int i = original.size() - 2; i >= 0; --i) { + InterOpBase cur = original.get(i); + LimitOp next = nextLimit(original, i); + if (cur instanceof OrderOp && next != null) { + ((OrderOp) cur).setLower(next.getLower().get()); + ((OrderOp) cur).setUpper(next.getUpper().get()); + Optional nextAlias = next.getAlias(); + if (nextAlias.isPresent()) { + cur.setAlias(nextAlias.get()); + } + opCollection.removeInterOp(i + 1); + } + } + } + + // return LimitOp if next is, otherwise null + private LimitOp nextLimit(List original, int cur) { + int next = cur + 1; + return (next >= 0 && next < original.size() && original.get(next) instanceof LimitOp) + ? (LimitOp) original.get(next) + : null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/EnumConverter.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/EnumConverter.java new file mode 100644 index 000000000000..82cd0e6230c0 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/EnumConverter.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna; + +import com.sun.jna.FromNativeContext; +import com.sun.jna.ToNativeContext; +import com.sun.jna.TypeConverter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EnumConverter implements TypeConverter { + private static final Logger logger = LoggerFactory.getLogger(EnumConverter.class); + + public Object fromNative(Object input, FromNativeContext context) { + Integer i = (Integer) input; + Class targetClass = context.getTargetType(); + if (!IntEnum.class.isAssignableFrom(targetClass)) { + return null; + } + Object[] enums = targetClass.getEnumConstants(); + if (enums.length == 0) { + logger.error( + "Could not convert desired enum type (), no valid values are defined.", + targetClass.getName()); + return null; + } + IntEnum instance = (IntEnum) enums[0]; + return instance.getEnum(i); + } + + public Object toNative(Object input, ToNativeContext context) { + if (input == null) { + return new Integer(0); + } + IntEnum j = (IntEnum) input; + return new Integer(j.getInt()); + } + + public Class nativeType() { + return Integer.class; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IntEnum.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IntEnum.java new file mode 100644 index 000000000000..fe21516c685d --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IntEnum.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna; + +/** + * the interface is used to serialize between java enum type + * and c enum type which can be represented as int + * @param java enum type + * see {@link EnumConverter} + */ +public interface IntEnum { + int getInt(); + + T getEnum(int i); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java new file mode 100644 index 000000000000..9c8881fba9e5 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java @@ -0,0 +1,227 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna; + +import com.alibaba.graphscope.common.jna.type.*; +import com.google.common.collect.ImmutableMap; +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.Pointer; +import com.sun.jna.ptr.IntByReference; + +public interface IrCoreLibrary extends Library { + IrCoreLibrary INSTANCE = + Native.load( + "ir_core", + IrCoreLibrary.class, + ImmutableMap.of( + Library.OPTION_TYPE_MAPPER, IrTypeMapper.INSTANCE, + Library.OPTION_FUNCTION_MAPPER, IrFunctionMapper.INSTANCE)); + + Pointer initLogicalPlan(); + + void write_plan_to_json(Pointer plan, String jsonFile); + + void destroyLogicalPlan(Pointer plan); + + FfiData.ByValue buildPhysicalPlan(Pointer plan, int workers, int servers); + + Pointer initScanOperator(FfiScanOpt opt); + + FfiError.ByValue appendScanOperator( + Pointer plan, Pointer scan, int parent, IntByReference oprIdx); + + // set primary index + Pointer initIndexPredicate(); + + FfiError.ByValue andEquivPredicate( + Pointer predicate, FfiProperty.ByValue key, FfiConst.ByValue value); + + FfiError.ByValue orEquivPredicate( + Pointer predicate, FfiProperty.ByValue key, FfiConst.ByValue value); + + FfiError.ByValue addScanIndexPredicate(Pointer scan, Pointer predicate); + + FfiError.ByValue setScanParams(Pointer scan, Pointer params); + + FfiError.ByValue setScanAlias(Pointer scan, FfiAlias.ByValue alias); + + Pointer initEdgexpdOperator(boolean isEdge, FfiDirection direction); + + FfiError.ByValue appendEdgexpdOperator( + Pointer plan, Pointer edgeXpd, int parent, IntByReference oprIdx); + + FfiError.ByValue setEdgexpdParams(Pointer edgeXpd, Pointer params); + + FfiError.ByValue setEdgexpdAlias(Pointer edgeXpd, FfiAlias.ByValue alias); + + Pointer initLimitOperator(); + + FfiError.ByValue setLimitRange(Pointer limit, int lower, int upper); + + FfiError.ByValue appendLimitOperator( + Pointer plan, Pointer limit, int parent, IntByReference oprIdx); + + Pointer initSelectOperator(); + + FfiError.ByValue setSelectPredicate(Pointer select, String predicate); + + FfiError.ByValue appendSelectOperator( + Pointer plan, Pointer select, int parent, IntByReference oprIdx); + + Pointer initOrderbyOperator(); + + FfiError.ByValue addOrderbyPair( + Pointer orderBy, FfiVariable.ByValue variable, FfiOrderOpt orderOpt); + + FfiError.ByValue setOrderbyLimit(Pointer orderBy, int lower, int upper); + + FfiError.ByValue appendOrderbyOperator( + Pointer plan, Pointer orderBy, int parent, IntByReference oprIdx); + + Pointer initProjectOperator(boolean isAppend); + + FfiError.ByValue addProjectExprAlias(Pointer project, String expr, FfiAlias.ByValue alias); + + FfiError.ByValue appendProjectOperator( + Pointer plan, Pointer project, int parent, IntByReference oprIdx); + + /// To initialize an As operator + Pointer initAsOperator(); + + /// Set the alias of the entity to As + FfiError.ByValue setAsAlias(Pointer as, FfiAlias.ByValue alias); + + /// Append an As operator to the logical plan + FfiError.ByValue appendAsOperator(Pointer plan, Pointer as, int parent, IntByReference oprIdx); + + Pointer initGroupbyOperator(); + + FfiError.ByValue addGroupbyKeyAlias( + Pointer groupBy, FfiVariable.ByValue key, FfiAlias.ByValue alias); + + FfiError.ByValue addGroupbyAggFn(Pointer groupBy, FfiAggFn.ByValue aggFn); + + FfiError.ByValue appendGroupbyOperator( + Pointer plan, Pointer groupBy, int parent, IntByReference oprIdx); + + Pointer initDedupOperator(); + + FfiError.ByValue addDedupKey(Pointer dedup, FfiVariable.ByValue var); + + FfiError.ByValue appendDedupOperator( + Pointer plan, Pointer dedup, int parent, IntByReference oprIdx); + + Pointer initSinkOperator(); + + FfiError.ByValue addSinkColumn(Pointer sink, FfiNameOrId.ByValue column); + + FfiError.ByValue appendSinkOperator( + Pointer plan, Pointer sink, int parent, IntByReference oprIdx); + + Pointer initGetvOperator(FfiVOpt vOpt); + + FfiError.ByValue setGetvAlias(Pointer getV, FfiAlias.ByValue alias); + + FfiError.ByValue appendGetvOperator( + Pointer plan, Pointer getV, int parent, IntByReference oprIdx); + + FfiError.ByValue setGetvParams(Pointer getV, Pointer params); + + Pointer initApplyOperator(int subtaskRoot, FfiJoinKind joinKind); + + FfiError.ByValue setApplyAlias(Pointer apply, FfiAlias.ByValue alias); + + FfiError.ByValue appendApplyOperator( + Pointer plan, Pointer apply, int parent, IntByReference oprIdx); + + Pointer initPathxpdOperator(Pointer expand, boolean isWholePath); + + FfiError.ByValue setPathxpdAlias(Pointer pathXpd, FfiAlias.ByValue alias); + + FfiError.ByValue setPathxpdHops(Pointer pathXpd, int lower, int upper); + + FfiError.ByValue appendPathxpdOperator( + Pointer plan, Pointer pathXpd, int parent, IntByReference oprIdx); + + Pointer initUnionOperator(); + + FfiError.ByValue addUnionParent(Pointer union, int subRootId); + + FfiError.ByValue appendUnionOperator(Pointer plan, Pointer union, IntByReference oprIdx); + + Pointer initPatternOperator(); + + Pointer initPatternSentence(FfiJoinKind joinKind); + + FfiError.ByValue setSentenceStart(Pointer sentence, FfiNameOrId.ByValue tag); + + FfiError.ByValue setSentenceEnd(Pointer sentence, FfiNameOrId.ByValue tag); + + FfiError.ByValue addSentenceBinder(Pointer sentence, Pointer binder, FfiBinderOpt opt); + + FfiError.ByValue addPatternSentence(Pointer pattern, Pointer sentence); + + FfiError.ByValue appendPatternOperator( + Pointer plan, Pointer pattern, int parent, IntByReference oprIdx); + + FfiNameOrId.ByValue noneNameOrId(); + + FfiNameOrId.ByValue cstrAsNameOrId(String name); + + FfiConst.ByValue cstrAsConst(String value); + + FfiConst.ByValue int32AsConst(int value); + + FfiConst.ByValue int64AsConst(long value); + + FfiProperty.ByValue asNoneKey(); + + FfiProperty.ByValue asLabelKey(); + + FfiProperty.ByValue asIdKey(); + + FfiProperty.ByValue asLenKey(); + + FfiProperty.ByValue asPropertyKey(FfiNameOrId.ByValue key); + + FfiVariable.ByValue asVarTagOnly(FfiNameOrId.ByValue tag); + + FfiVariable.ByValue asVarPropertyOnly(FfiProperty.ByValue property); + + FfiVariable.ByValue asVar(FfiNameOrId.ByValue tag, FfiProperty.ByValue property); + + FfiVariable.ByValue asNoneVar(); + + FfiAggFn.ByValue initAggFn(FfiAggOpt aggregate, FfiAlias.ByValue alias); + + void destroyFfiData(FfiData.ByValue value); + + FfiError.ByValue setSchema(String schemaJson); + + Pointer initQueryParams(); + + FfiError.ByValue addParamsTable(Pointer params, FfiNameOrId.ByValue table); + + FfiError.ByValue addParamsColumn(Pointer params, FfiNameOrId.ByValue column); + + FfiError.ByValue setParamsRange(Pointer params, int lower, int upper); + + FfiError.ByValue setParamsPredicate(Pointer params, String predicate); + + FfiError.ByValue setParamsIsAllColumns(Pointer params); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrFunctionMapper.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrFunctionMapper.java new file mode 100644 index 000000000000..6d2a4a866cf2 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrFunctionMapper.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna; + +import com.sun.jna.FunctionMapper; +import com.sun.jna.NativeLibrary; + +import java.lang.reflect.Method; + +public class IrFunctionMapper implements FunctionMapper { + public static IrFunctionMapper INSTANCE = new IrFunctionMapper(); + + private IrFunctionMapper() { + super(); + } + + @Override + public String getFunctionName(NativeLibrary nativeLibrary, Method method) { + String target = method.getName(); + String[] splits = target.split("(?=\\p{Lu})"); + StringBuilder cName = new StringBuilder(); + for (int i = 0; i < splits.length; ++i) { + if (i != 0) { + cName.append("_"); + } + cName.append(splits[i].toLowerCase()); + } + return cName.toString(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrTypeMapper.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrTypeMapper.java new file mode 100644 index 000000000000..5858ed26c07d --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrTypeMapper.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna; + +import com.sun.jna.DefaultTypeMapper; + +public class IrTypeMapper extends DefaultTypeMapper { + public static IrTypeMapper INSTANCE = new IrTypeMapper(); + + private IrTypeMapper() { + super(); + addTypeConverter(IntEnum.class, new EnumConverter()); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAggFn.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAggFn.java new file mode 100644 index 000000000000..b7514d239c4e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAggFn.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IrTypeMapper; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"vars", "aggregate", "alias"}) +public class FfiAggFn extends Structure { + public FfiAggFn() { + super(IrTypeMapper.INSTANCE); + } + + public static class ByValue extends FfiAggFn implements Structure.ByValue {} + + public Pointer vars; + public FfiAggOpt aggregate; + public FfiAlias.ByValue alias; +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAggOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAggOpt.java new file mode 100644 index 000000000000..e6eefd6d9be8 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAggOpt.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiAggOpt implements IntEnum { + Sum, + Min, + Max, + Count, + CountDistinct, + ToList, + ToSet, + Avg; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiAggOpt getEnum(int i) { + FfiAggOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAlias.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAlias.java new file mode 100644 index 000000000000..0fef5d88cedc --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiAlias.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.google.common.base.Objects; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"alias", "isQueryGiven"}) +public class FfiAlias extends Structure { + public static class ByValue extends FfiAlias implements Structure.ByValue {} + + public FfiNameOrId.ByValue alias; + public boolean isQueryGiven; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FfiAlias ffiAlias = (FfiAlias) o; + return isQueryGiven == ffiAlias.isQueryGiven && Objects.equal(alias, ffiAlias.alias); + } + + @Override + public int hashCode() { + return Objects.hashCode(alias, isQueryGiven); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiBinderOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiBinderOpt.java new file mode 100644 index 000000000000..e550cce842c7 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiBinderOpt.java @@ -0,0 +1,23 @@ +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiBinderOpt implements IntEnum { + Edge, + Path, + Vertex; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiBinderOpt getEnum(int i) { + FfiBinderOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiConst.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiConst.java new file mode 100644 index 000000000000..4fa6f39d9de4 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiConst.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IrTypeMapper; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"dataType", "bool", "int32", "int64", "float64", "cstr", "raw"}) +public class FfiConst extends Structure { + public FfiConst() { + super(IrTypeMapper.INSTANCE); + } + + public static class ByValue extends FfiConst implements Structure.ByValue {} + + public FfiDataType dataType; + public boolean bool; + public int int32; + public long int64; + public double float64; + public String cstr; + public Pointer raw; +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiData.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiData.java new file mode 100644 index 000000000000..671f8a61572f --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiData.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IrCoreLibrary; +import com.alibaba.graphscope.common.jna.IrTypeMapper; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; + +import java.io.Closeable; + +@Structure.FieldOrder({"buffer", "len", "error"}) +public class FfiData extends Structure { + public FfiData() { + super(IrTypeMapper.INSTANCE); + } + + public static class ByValue extends FfiData implements Structure.ByValue, Closeable { + public byte[] getBytes() { + if (buffer != null && len > 0) { + byte[] bytes = new byte[len]; + for (int i = 0; i < len; ++i) { + bytes[i] = buffer.getByte(i); + } + return bytes; + } + return null; + } + + @Override + public void close() { + setAutoSynch(false); + IrCoreLibrary.INSTANCE.destroyFfiData(this); + } + } + + public Pointer buffer; + public int len; + public FfiError.ByValue error; +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiDataType.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiDataType.java new file mode 100644 index 000000000000..c067eb3dac5f --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiDataType.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiDataType implements IntEnum { + Unknown, + Boolean, + I32, + I64, + F64, + Str; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiDataType getEnum(int i) { + FfiDataType opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiDirection.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiDirection.java new file mode 100644 index 000000000000..5abff34689c3 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiDirection.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiDirection implements IntEnum { + Out, + In, + Both; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiDirection getEnum(int i) { + FfiDirection opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiError.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiError.java new file mode 100644 index 000000000000..3597ff1e4d80 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiError.java @@ -0,0 +1,16 @@ +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IrTypeMapper; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"code", "msg"}) +public class FfiError extends Structure { + public FfiError() { + super(IrTypeMapper.INSTANCE); + } + + public static class ByValue extends FfiError implements Structure.ByValue {} + + public ResultCode code; + public String msg; +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiJoinKind.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiJoinKind.java new file mode 100644 index 000000000000..a0a4a9fabe4f --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiJoinKind.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiJoinKind implements IntEnum { + /// Inner join + Inner, + /// Left outer join + LeftOuter, + /// Right outer join + RightOuter, + /// Full outer join + FullOuter, + /// Left semi-join, right alternative can be naturally adapted + Semi, + /// Left anti-join, right alternative can be naturally adapted + Anti, + /// aka. Cartesian product + Times; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiJoinKind getEnum(int i) { + FfiJoinKind opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameIdOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameIdOpt.java new file mode 100644 index 000000000000..4091108dafcb --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameIdOpt.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiNameIdOpt implements IntEnum { + None, + Name, + Id; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiNameIdOpt getEnum(int i) { + FfiNameIdOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameOrId.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameOrId.java new file mode 100644 index 000000000000..84adb8afcb2d --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiNameOrId.java @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IrTypeMapper; +import com.google.common.base.Objects; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"opt", "name", "nameId"}) +public class FfiNameOrId extends Structure { + private FfiNameOrId() { + super(IrTypeMapper.INSTANCE); + } + + public static class ByValue extends FfiNameOrId implements Structure.ByValue {} + + public FfiNameIdOpt opt; + public String name; + public int nameId; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FfiNameOrId that = (FfiNameOrId) o; + return nameId == that.nameId && opt == that.opt && Objects.equal(name, that.name); + } + + // as map key + @Override + public int hashCode() { + return Objects.hashCode(opt, name, nameId); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiOrderOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiOrderOpt.java new file mode 100644 index 000000000000..c1e9dee7ad69 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiOrderOpt.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiOrderOpt implements IntEnum { + Shuffle, + Asc, + Desc; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiOrderOpt getEnum(int i) { + FfiOrderOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiProperty.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiProperty.java new file mode 100644 index 000000000000..741c6664df9a --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiProperty.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IrTypeMapper; +import com.google.common.base.Objects; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"opt", "key"}) +public class FfiProperty extends Structure { + public FfiProperty() { + super(IrTypeMapper.INSTANCE); + } + + public static class ByValue extends FfiProperty implements Structure.ByValue {} + + public FfiPropertyOpt opt; + public FfiNameOrId.ByValue key; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FfiProperty that = (FfiProperty) o; + return opt == that.opt && Objects.equal(key, that.key); + } + + @Override + public int hashCode() { + return Objects.hashCode(opt, key); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiPropertyOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiPropertyOpt.java new file mode 100644 index 000000000000..896f8139d05b --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiPropertyOpt.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiPropertyOpt implements IntEnum { + None, + Id, + Label, + Len, + Key; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiPropertyOpt getEnum(int i) { + FfiPropertyOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiScanOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiScanOpt.java new file mode 100644 index 000000000000..3251cfcbad45 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiScanOpt.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiScanOpt implements IntEnum { + Entity, + Relation; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiScanOpt getEnum(int i) { + FfiScanOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVOpt.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVOpt.java new file mode 100644 index 000000000000..8e30845185f6 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVOpt.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum FfiVOpt implements IntEnum { + Start, + End, + Other; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public FfiVOpt getEnum(int i) { + FfiVOpt opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVariable.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVariable.java new file mode 100644 index 000000000000..3c9b26e36ea7 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiVariable.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.google.common.base.Objects; +import com.sun.jna.Structure; + +@Structure.FieldOrder({"tag", "property"}) +public class FfiVariable extends Structure { + public static class ByValue extends FfiVariable implements Structure.ByValue {} + + public FfiNameOrId.ByValue tag; + public FfiProperty.ByValue property; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FfiVariable that = (FfiVariable) o; + return Objects.equal(tag, that.tag) && Objects.equal(property, that.property); + } + + @Override + public int hashCode() { + return Objects.hashCode(tag, property); + } + + @Override + public String toString() { + String tagName = ArgUtils.tagName(tag); + String propertyName = ArgUtils.propertyName(property); + if (propertyName.isEmpty()) { + return "@" + tagName; + } else { + return "@" + tagName + "." + propertyName; + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/ResultCode.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/ResultCode.java new file mode 100644 index 000000000000..f9e084468556 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/ResultCode.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.jna.type; + +import com.alibaba.graphscope.common.jna.IntEnum; + +public enum ResultCode implements IntEnum { + Success, + ParseExprError, + MissingDataError, + CStringError, + UnknownTypeError, + InvalidRangeError, + NegativeIndexError, + BuildJobError, + ParsePbError, + ParentNotFoundError, + ColumnNotExistError, + TableNotExistError, + UnSupported, + Unknown; + + @Override + public int getInt() { + return this.ordinal(); + } + + @Override + public ResultCode getEnum(int i) { + ResultCode opts[] = values(); + if (i < opts.length && i >= 0) { + return opts[i]; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java new file mode 100644 index 000000000000..8252883d48e5 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.store; + +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.gremlin.Utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Optional; + +public class ExperimentalMetaFetcher extends IrMetaFetcher { + private static final Logger logger = LoggerFactory.getLogger(ExperimentalMetaFetcher.class); + private Configs configs; + + public ExperimentalMetaFetcher(Configs configs) { + this.configs = configs; + super.fetch(); + } + + @Override + protected Optional getIrMeta() { + String schemaFilePath = GraphConfig.GRAPH_SCHEMA.get(configs); + try { + String schema = Utils.readStringFromFile(schemaFilePath); + return Optional.of(schema); + } catch (IOException e) { + logger.info("open schema file {} fail", schemaFilePath); + throw new RuntimeException(e); + } + } + + @Override + public void fetch() { + // the meta from the static file is created in the constructor, here just do nothing + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java new file mode 100644 index 000000000000..e8a3df0bbc2b --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMetaFetcher.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.store; + +import com.alibaba.graphscope.common.jna.IrCoreLibrary; + +import java.util.Optional; + +public abstract class IrMetaFetcher { + private static IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE; + + protected abstract Optional getIrMeta(); + + public void fetch() { + Optional irMetaOpt = getIrMeta(); + if (irMetaOpt.isPresent()) { + irCoreLib.setSchema(irMetaOpt.get()); + } else { + throw new RuntimeException("ir meta is not ready, retry please"); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/ClassUtils.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/ClassUtils.java new file mode 100644 index 000000000000..8763bb3aee9f --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/ClassUtils.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.utils; + +public class ClassUtils { + public static boolean equalClass(T t1, Class target) { + return t1.getClass().equals(target); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/FileUtils.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/FileUtils.java new file mode 100644 index 000000000000..fe505712f7bc --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/FileUtils.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.utils; + +import com.google.common.io.Resources; + +import java.net.URL; +import java.nio.charset.StandardCharsets; + +public class FileUtils { + public static String readJsonFromResource(String file) { + try { + URL url = Thread.currentThread().getContextClassLoader().getResource(file); + return Resources.toString(url, StandardCharsets.UTF_8).trim(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/JsonUtils.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/JsonUtils.java new file mode 100644 index 000000000000..8bbb35e48f31 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/common/utils/JsonUtils.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.utils; + +import static org.apache.commons.lang3.StringUtils.isEmpty; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.module.paranamer.ParanamerModule; + +import java.io.IOException; +import java.util.List; + +public class JsonUtils { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + static { + SimpleModule m = new SimpleModule(); + OBJECT_MAPPER.registerModule(m); + OBJECT_MAPPER.registerModule(new ParanamerModule()); + OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public static String toJson(Object object) { + try { + return OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(object); + } catch (JsonProcessingException e) { + throw new RuntimeException("JsonProcessingException : ", e); + } + } + + public static T fromJson(String json, TypeReference typeRef) throws RuntimeException { + try { + return isEmpty(json) ? null : OBJECT_MAPPER.readValue(json, typeRef); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static T fromJson(String json, Class valueType) { + try { + return OBJECT_MAPPER.readValue(json, valueType); + } catch (IOException e) { + throw new RuntimeException("IOException : ", e); + } + } + + public static JsonNode fromJson(String json) { + try { + return OBJECT_MAPPER.readTree(json); + } catch (IOException e) { + throw new RuntimeException("IOException : ", e); + } + } + + public static JsonNode parseJsonTree(String json) throws Exception { + return isEmpty(json) ? null : OBJECT_MAPPER.readTree(json); + } + + public static List parseAsList(String json, Class valueType) throws IOException { + JavaType t = OBJECT_MAPPER.getTypeFactory().constructParametricType(List.class, valueType); + return OBJECT_MAPPER.readValue(json, t); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/InterOpCollectionBuilder.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/InterOpCollectionBuilder.java new file mode 100644 index 000000000000..b407ffa367ce --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/InterOpCollectionBuilder.java @@ -0,0 +1,150 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.exception.OpArgIllegalException; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.*; +import com.alibaba.graphscope.gremlin.exception.UnsupportedStepException; +import com.alibaba.graphscope.gremlin.plugin.step.ExprStep; +import com.alibaba.graphscope.gremlin.plugin.step.PathExpandStep; +import com.alibaba.graphscope.gremlin.plugin.step.ScanFusionStep; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +// build IrPlan from gremlin traversal +public class InterOpCollectionBuilder { + private static final Logger logger = LoggerFactory.getLogger(InterOpCollectionBuilder.class); + private Traversal traversal; + + public InterOpCollectionBuilder(Traversal traversal) { + this.traversal = traversal; + } + + public InterOpCollection build() throws OpArgIllegalException, UnsupportedStepException { + InterOpCollection opCollection = new InterOpCollection(); + List steps = traversal.asAdmin().getSteps(); + for (Step step : steps) { + List opList = new ArrayList<>(); + // judge by class type instead of instance + if (Utils.equalClass(step, GraphStep.class)) { + opList.add(StepTransformFactory.GRAPH_STEP.apply(step)); + } else if (Utils.equalClass(step, ScanFusionStep.class)) { + opList.add(StepTransformFactory.SCAN_FUSION_STEP.apply(step)); + } else if (Utils.equalClass(step, VertexStep.class)) { + opList.add(StepTransformFactory.VERTEX_STEP.apply(step)); + } else if (Utils.equalClass(step, HasStep.class)) { + opList.add(StepTransformFactory.HAS_STEP.apply(step)); + } else if (Utils.equalClass(step, RangeGlobalStep.class)) { + opList.add(StepTransformFactory.LIMIT_STEP.apply(step)); + } else if (Utils.equalClass(step, PropertyMapStep.class)) { + opList.add(StepTransformFactory.VALUE_MAP_STEP.apply(step)); + } else if (Utils.equalClass(step, DedupGlobalStep.class)) { + opList.add(StepTransformFactory.DEDUP_STEP.apply(step)); + } else if (Utils.equalClass(step, CountGlobalStep.class)) { + opList.add(StepTransformFactory.COUNT_STEP.apply(step)); + } else if (Utils.equalClass(step, PropertiesStep.class)) { + opList.add(StepTransformFactory.VALUES_STEP.apply(step)); + } else if (Utils.equalClass(step, IsStep.class)) { + opList.add(StepTransformFactory.IS_STEP.apply(step)); + } else if (Utils.equalClass(step, EdgeVertexStep.class)) { + opList.add(StepTransformFactory.EDGE_VERTEX_STEP.apply(step)); + } else if (Utils.equalClass(step, EdgeOtherVertexStep.class)) { + opList.add(StepTransformFactory.EDGE_OTHER_STEP.apply(step)); + } else if (Utils.equalClass(step, PathExpandStep.class)) { + opList.add(StepTransformFactory.PATH_EXPAND_STEP.apply(step)); + } else if (Utils.equalClass(step, WhereTraversalStep.WhereStartStep.class)) { + opList.add(StepTransformFactory.WHERE_START_STEP.apply(step)); + } else if (Utils.equalClass(step, WhereTraversalStep.WhereEndStep.class)) { + opList.add(StepTransformFactory.WHERE_END_STEP.apply(step)); + } else if (Utils.equalClass(step, UnionStep.class)) { + opList.add(StepTransformFactory.UNION_STEP.apply(step)); + } else if (Utils.equalClass(step, TraversalMapStep.class)) { + opList.add(StepTransformFactory.TRAVERSAL_MAP_STEP.apply(step)); + } else if (Utils.equalClass(step, SelectOneStep.class)) { + opList.addAll( + TraversalParentTransformFactory.PROJECT_BY_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, SelectStep.class)) { + opList.addAll( + TraversalParentTransformFactory.PROJECT_BY_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, OrderGlobalStep.class)) { + opList.addAll( + TraversalParentTransformFactory.ORDER_BY_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, GroupStep.class)) { + opList.addAll( + TraversalParentTransformFactory.GROUP_BY_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, GroupCountStep.class)) { + opList.addAll( + TraversalParentTransformFactory.GROUP_BY_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, WherePredicateStep.class)) { + opList.addAll( + TraversalParentTransformFactory.WHERE_BY_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, TraversalFilterStep.class) + || Utils.equalClass(step, WhereTraversalStep.class)) { + opList.addAll( + TraversalParentTransformFactory.WHERE_TRAVERSAL_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, NotStep.class)) { + opList.addAll( + TraversalParentTransformFactory.NOT_TRAVERSAL_STEP.apply( + (TraversalParent) step)); + } else if (Utils.equalClass(step, MatchStep.class)) { + opList.add(StepTransformFactory.MATCH_STEP.apply(step)); + } else if (Utils.equalClass(step, ExprStep.class)) { + opList.add(StepTransformFactory.EXPR_STEP.apply(step)); + } else { + throw new UnsupportedStepException(step.getClass(), "unimplemented yet"); + } + for (int i = 0; i < opList.size(); ++i) { + InterOpBase op = opList.get(i); + // last op + if (i == opList.size() - 1) { + // set alias + if (step.getLabels().size() > 1) { + logger.error( + "multiple aliases of one object is unsupported, take the first and" + + " ignore others"); + } + if (!step.getLabels().isEmpty()) { + String label = (String) step.getLabels().iterator().next(); + op.setAlias(new OpArg(ArgUtils.asFfiAlias(label, true))); + } + } + opCollection.appendInterOp(op); + } + } + return opCollection; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/Utils.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/Utils.java new file mode 100644 index 000000000000..85005896fab3 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/Utils.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.utils.ClassUtils; +import com.google.common.base.Preconditions; + +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; + +public class Utils extends ClassUtils { + /** + * Set private field with given value + * + * @param obj The given instance + * @param fieldName The field name + * @param value The field value + * @param The field value type + */ + public static void setFieldValue(Class clazz, Object obj, String fieldName, V value) { + Preconditions.checkNotNull(obj); + Preconditions.checkNotNull(fieldName); + Preconditions.checkNotNull(value); + + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(obj, value); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static V getFieldValue(Class clazz, Object obj, String fieldName) { + Preconditions.checkNotNull(obj); + Preconditions.checkNotNull(fieldName); + + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return (V) field.get(obj); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static String[] removeStringEle(int i, String[] data) { + if (data.length == 0 || i < 0 || i >= data.length) return data; + String[] copy = new String[data.length - 1]; + for (int j = 0; j < data.length - 1; ++j) { + if (j < i) { + copy[j] = data[j]; + } else { + copy[j] = data[j + 1]; + } + } + return copy; + } + + public static String readStringFromFile(String filePath) throws IOException { + return FileUtils.readFileToString(new File(filePath), StandardCharsets.UTF_8); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/AnyValue.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/AnyValue.java new file mode 100644 index 000000000000..affb4b73b1f7 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/AnyValue.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +public class AnyValue { + public static AnyValue INSTANCE = new AnyValue(); + + private AnyValue() {} +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/ExprP.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/ExprP.java new file mode 100644 index 000000000000..b49b6fc0df83 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/ExprP.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import org.apache.tinkerpop.gremlin.process.traversal.P; + +public class ExprP extends P { + public ExprP(String expr) { + super(null, expr); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GenericLiteralVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GenericLiteralVisitor.java new file mode 100644 index 000000000000..c5a6c8ad70e6 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GenericLiteralVisitor.java @@ -0,0 +1,284 @@ +/* + * This file is referred and derived from project apache/tinkerpop + * + * https://github.com/apache/tinkerpop/blob/master/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/language/grammar/GenericLiteralVisitor.java + * + * which has the following license: + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import org.apache.commons.text.StringEscapeUtils; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Visitor class to handle generic literal. All visitor methods return type is Object. It maybe used as a singleton + * in cases where a {@link Traversal} object is not expected, otherwise a new instance must be constructed. + */ +public class GenericLiteralVisitor extends GremlinGSBaseVisitor { + + private static GenericLiteralVisitor instance; + + private GenericLiteralVisitor() {} + + public static GenericLiteralVisitor getInstance() { + if (instance == null) { + instance = new GenericLiteralVisitor(); + } + return instance; + } + + /** + * Parse a string literal context and return the string literal + */ + public static String getStringLiteral( + final GremlinGSParser.StringLiteralContext stringLiteral) { + return (String) (getInstance().visitStringLiteral(stringLiteral)); + } + + /** + * Parse a boolean literal context and return the boolean literal + */ + public static boolean getBooleanLiteral( + final GremlinGSParser.BooleanLiteralContext booleanLiteral) { + return (boolean) (getInstance().visitBooleanLiteral(booleanLiteral)); + } + + /** + * Parse a String literal list context and return a string array + */ + public static String[] getStringLiteralList( + final GremlinGSParser.StringLiteralListContext stringLiteralList) { + if (stringLiteralList == null || stringLiteralList.stringLiteralExpr() == null) { + return new String[0]; + } + return stringLiteralList.stringLiteralExpr().stringLiteral().stream() + .filter(Objects::nonNull) + .map(stringLiteral -> getInstance().visitStringLiteral(stringLiteral)) + .toArray(String[]::new); + } + + /** + * Parse a String literal expr context and return a string array + */ + public static String[] getStringLiteralExpr( + final GremlinGSParser.StringLiteralExprContext stringLiteralExpr) { + return stringLiteralExpr.stringLiteral().stream() + .filter(Objects::nonNull) + .map(stringLiteral -> getInstance().visitStringLiteral(stringLiteral)) + .toArray(String[]::new); + } + + /** + * Parse a generic literal list, and return an object array + */ + public static Object[] getGenericLiteralList( + final GremlinGSParser.GenericLiteralListContext objectLiteralList) { + if (objectLiteralList == null || objectLiteralList.genericLiteralExpr() == null) { + return new Object[0]; + } + return objectLiteralList.genericLiteralExpr().genericLiteral().stream() + .filter(Objects::nonNull) + .map(genericLiteral -> getInstance().visitGenericLiteral(genericLiteral)) + .toArray(Object[]::new); + } + + /** + * Parse a Integer literal list context and return a Integer array + */ + public static Object[] getIntegerLiteralList( + final GremlinGSParser.IntegerLiteralListContext integerLiteralList) { + if (integerLiteralList == null || integerLiteralList.integerLiteralExpr() == null) { + return new Object[0]; + } + return integerLiteralList.integerLiteralExpr().integerLiteral().stream() + .filter(Objects::nonNull) + .map(integerLiteral -> getInstance().visitIntegerLiteral(integerLiteral)) + .toArray(Object[]::new); + } + + /** + * Remove single/double quotes around String literal + * + * @param quotedString : quoted string + * @return quotes stripped string + */ + private static String stripQuotes(final String quotedString) { + return quotedString.substring(1, quotedString.length() - 1); + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitGenericLiteralList(final GremlinGSParser.GenericLiteralListContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitGenericLiteralExpr(final GremlinGSParser.GenericLiteralExprContext ctx) { + final int childCount = ctx.getChildCount(); + switch (childCount) { + case 0: + // handle empty expression + return new Object[0]; + case 1: + // handle single generic literal + return visitGenericLiteral((GremlinGSParser.GenericLiteralContext) ctx.getChild(0)); + default: + // handle multiple generic literal separated by comma + final List genericLiterals = new ArrayList<>(); + int childIndex = 0; + while (childIndex < ctx.getChildCount()) { + genericLiterals.add( + visitGenericLiteral( + (GremlinGSParser.GenericLiteralContext) + ctx.getChild(childIndex))); + // skip comma + childIndex += 2; + } + return genericLiterals.toArray(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitGenericLiteral(final GremlinGSParser.GenericLiteralContext ctx) { + return visitChildren(ctx); + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitIntegerLiteral(final GremlinGSParser.IntegerLiteralContext ctx) { + String integerLiteral = ctx.getText().toLowerCase().replace("_", ""); + // handle suffix: L/l + final int lastCharIndex = integerLiteral.length() - 1; + if (integerLiteral.charAt(lastCharIndex) == 'l') { + integerLiteral = integerLiteral.substring(0, lastCharIndex); + + return Long.decode(integerLiteral); + } + + try { + // try to parse it as integer first + return Integer.decode(integerLiteral); + } catch (NumberFormatException ignoredExpection1) { + try { + // If range exceeds integer limit, try to parse it as long + return Long.decode(integerLiteral); + } catch (NumberFormatException ignoredExpection2) { + // If range exceeds Long limit, parse it as BigInteger + // as the literal range is longer than long, the number of character should be much + // more than 3, + // so we skip boundary check below. + + // parse sign character + int startIndex = 0; + final char firstChar = integerLiteral.charAt(0); + final boolean negative = (firstChar == '-'); + if ((firstChar == '-') || (firstChar == '+')) { + startIndex++; + } + + // parse radix based on format + int radix = 10; + if (integerLiteral.charAt(startIndex + 1) == 'x') { + radix = 16; + startIndex += 2; + integerLiteral = integerLiteral.substring(startIndex); + if (negative) { + integerLiteral = '-' + integerLiteral; + } + } else if (integerLiteral.charAt(startIndex) == '0') { + radix = 8; + } + + // create big integer + return new BigInteger(integerLiteral, radix); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitFloatLiteral(final GremlinGSParser.FloatLiteralContext ctx) { + final String floatLiteral = ctx.getText().toLowerCase(); + + // check suffix + final char lastCharacter = floatLiteral.charAt(floatLiteral.length() - 1); + if (Character.isDigit(lastCharacter)) { + // if there is no suffix, parse it as BigDecimal + return new BigDecimal(floatLiteral); + } + + if (lastCharacter == 'f') { + // parse F/f suffix as Float + return new Float(ctx.getText()); + } else { + // parse D/d suffix as Double + return new Double(floatLiteral); + } + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitBooleanLiteral(final GremlinGSParser.BooleanLiteralContext ctx) { + return Boolean.valueOf(ctx.getText()); + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitStringLiteral(final GremlinGSParser.StringLiteralContext ctx) { + // Using Java string unescaping because it coincides with the Groovy rules: + // https://docs.oracle.com/javase/tutorial/java/data/characters.html + // http://groovy-lang.org/syntax.html#_escaping_special_characters + + return StringEscapeUtils.unescapeJava(stripQuotes(ctx.getText())); + } + + /** + * {@inheritDoc} + */ + @Override + public Object visitNullLiteral(final GremlinGSParser.NullLiteralContext ctx) { + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GraphTraversalSourceVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GraphTraversalSourceVisitor.java new file mode 100644 index 000000000000..d4db5956bf19 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GraphTraversalSourceVisitor.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; + +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; + +public class GraphTraversalSourceVisitor extends GremlinGSBaseVisitor { + private GraphTraversalSource g; + + public GraphTraversalSourceVisitor(GraphTraversalSource g) { + this.g = g; + } + + @Override + public GraphTraversalSource visitTraversalSource(GremlinGSParser.TraversalSourceContext ctx) { + if (ctx.getChildCount() != 1) { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern of source is [g]"); + } + return g; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GremlinAntlrToJava.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GremlinAntlrToJava.java new file mode 100644 index 000000000000..4258be55d021 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/GremlinAntlrToJava.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; + +import java.util.function.Supplier; + +public class GremlinAntlrToJava extends GremlinGSBaseVisitor { + final GraphTraversalSource g; + + final GremlinGSBaseVisitor gvisitor; + final GremlinGSBaseVisitor tvisitor; + + private static GremlinAntlrToJava instance; + private static Supplier> traversalSupplier = __::start; + + public static GremlinAntlrToJava getInstance(GraphTraversalSource g) { + if (instance == null) { + instance = new GremlinAntlrToJava(g); + } + return instance; + } + + public static Supplier> getTraversalSupplier() { + return traversalSupplier; + } + + private GremlinAntlrToJava(GraphTraversalSource g) { + this.g = g; + this.gvisitor = new GraphTraversalSourceVisitor(this.g); + this.tvisitor = new TraversalRootVisitor(this.gvisitor); + } + + @Override + public Object visitQuery(GremlinGSParser.QueryContext ctx) { + final int childCount = ctx.getChildCount(); + String notice = "supported pattern of query is [g.V()...]"; + if (childCount != 1) { + throw new UnsupportedEvalException(ctx.getClass(), notice); + } + final ParseTree firstChild = ctx.getChild(0); + + if (firstChild instanceof GremlinGSParser.RootTraversalContext) { + return this.tvisitor.visitRootTraversal( + (GremlinGSParser.RootTraversalContext) firstChild); + } else { + throw new UnsupportedEvalException(firstChild.getClass(), notice); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/NestedTraversalSourceListVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/NestedTraversalSourceListVisitor.java new file mode 100644 index 000000000000..ec9a81a5e9f2 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/NestedTraversalSourceListVisitor.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; + +/** + * This class implements Gremlin grammar's nested-traversal-list methods that returns a {@link Traversal} {@code []} + * to the callers. + */ +public class NestedTraversalSourceListVisitor extends GremlinGSBaseVisitor { + + protected final GremlinGSBaseVisitor tvisitor; + + public NestedTraversalSourceListVisitor(final GremlinGSBaseVisitor tvisitor) { + this.tvisitor = tvisitor; + } + + @Override + public Traversal[] visitNestedTraversalExpr( + final GremlinGSParser.NestedTraversalExprContext ctx) { + final int childCount = ctx.getChildCount(); + + // handle arbitrary number of traversals that are separated by comma + final Traversal[] results = new Traversal[(childCount + 1) / 2]; + int childIndex = 0; + while (childIndex < ctx.getChildCount()) { + results[childIndex / 2] = + tvisitor.visitNestedTraversal( + (GremlinGSParser.NestedTraversalContext) ctx.getChild(childIndex)); + // skip comma child + childIndex += 2; + } + + return results; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalEnumParser.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalEnumParser.java new file mode 100644 index 000000000000..bcaa2a7de490 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalEnumParser.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import org.antlr.v4.runtime.tree.ParseTree; + +public class TraversalEnumParser { + + /** + * Parse enum text from a parse tree context into a enum object + * + * @param enumType : class of enum + * @param context : parse tree context + * @return enum object + */ + public static , C extends ParseTree> E parseTraversalEnumFromContext( + final Class enumType, final C context) { + final String text = context.getText(); + final String className = enumType.getSimpleName(); + + // Support qualified class names like (ex: T.id or Scope.local) + if (text.startsWith(className)) { + final String strippedText = text.substring(className.length() + 1); + return E.valueOf(enumType, strippedText); + } else { + return E.valueOf(enumType, text); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalMethodVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalMethodVisitor.java new file mode 100644 index 000000000000..5dd2c07d2a02 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalMethodVisitor.java @@ -0,0 +1,643 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.Utils; +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; +import com.alibaba.graphscope.gremlin.plugin.step.ExprStep; +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversal; + +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.ByModulating; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderGlobalStep; +import org.apache.tinkerpop.gremlin.structure.Column; + +public class TraversalMethodVisitor extends TraversalRootVisitor { + final GraphTraversal graphTraversal; + + public TraversalMethodVisitor( + final GremlinGSBaseVisitor gvisitor, + final GraphTraversal graphTraversal) { + super(gvisitor); + this.graphTraversal = graphTraversal; + } + + @Override + public Traversal visitTraversalMethod(GremlinGSParser.TraversalMethodContext ctx) { + return visitChildren(ctx); + } + + @Override + public Traversal visitTraversalMethod_hasLabel( + GremlinGSParser.TraversalMethod_hasLabelContext ctx) { + if (ctx.getChildCount() == 4) { + return graphTraversal.hasLabel( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + } else if (ctx.stringLiteralList() != null) { + return graphTraversal.hasLabel( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()), + GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList())); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), + "supported pattern is [hasLabel('str')] or hasLabel('str1', ...)"); + } + } + + @Override + public Traversal visitTraversalMethod_hasId(GremlinGSParser.TraversalMethod_hasIdContext ctx) { + if (ctx.getChildCount() == 4) { + return graphTraversal.hasId( + GenericLiteralVisitor.getInstance().visitIntegerLiteral(ctx.integerLiteral())); + } else if (ctx.integerLiteral() != null && ctx.integerLiteralList() != null) { + return graphTraversal.hasId( + GenericLiteralVisitor.getInstance().visitIntegerLiteral(ctx.integerLiteral()), + GenericLiteralVisitor.getIntegerLiteralList(ctx.integerLiteralList())); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [hasId(1)] or hasId(1, 2, ...)"); + } + } + + @Override + public Traversal visitTraversalMethod_has(GremlinGSParser.TraversalMethod_hasContext ctx) { + String notice = + "supported pattern is [has('key', 'value')] or [has('key', P)] or [has('label'," + + " 'key', 'value')] or [has('label', 'key', P)]"; + int childCount = ctx.getChildCount(); + if (childCount == 6 && ctx.genericLiteral() != null) { + return graphTraversal.has( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)), + GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral())); + } else if (childCount == 6 && ctx.traversalPredicate() != null) { + return graphTraversal.has( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)), + TraversalPredicateVisitor.getInstance() + .visitTraversalPredicate(ctx.traversalPredicate())); + } else if (childCount == 8 && ctx.genericLiteral() != null) { + return graphTraversal.has( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)), + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(1)), + GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral())); + } else if (childCount == 8 && ctx.traversalPredicate() != null) { + return graphTraversal.has( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)), + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(1)), + TraversalPredicateVisitor.getInstance() + .visitTraversalPredicate(ctx.traversalPredicate())); + } else if (childCount == 4 && ctx.stringLiteral() != null) { + P eqAny = P.eq(AnyValue.INSTANCE); + return graphTraversal.has( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral(0)), eqAny); + } else { + throw new UnsupportedEvalException(ctx.getClass(), notice); + } + } + + @Override + public Traversal visitTraversalMethod_is(GremlinGSParser.TraversalMethod_isContext ctx) { + if (ctx.genericLiteral() != null) { + return graphTraversal.is( + GenericLiteralVisitor.getInstance().visitGenericLiteral(ctx.genericLiteral())); + } else if (ctx.traversalPredicate() != null) { + return graphTraversal.is( + TraversalPredicateVisitor.getInstance() + .visitTraversalPredicate(ctx.traversalPredicate())); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [is(27)] or [is(eq(27))])"); + } + } + + @Override + public Traversal visitTraversalMethod_out(GremlinGSParser.TraversalMethod_outContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (labels != null && labels.length > 0 && isRangeExpression(labels[0])) { + GraphTraversal nestedRange = GremlinAntlrToJava.getTraversalSupplier().get(); + String[] ranges = labels[0].split("\\.\\."); + RangeGlobalStep rangeStep = + new RangeGlobalStep( + nestedRange.asAdmin(), + Integer.valueOf(ranges[0]), + Integer.valueOf(ranges[1])); + nestedRange.asAdmin().addStep(rangeStep); + IrCustomizedTraversal traversal = (IrCustomizedTraversal) graphTraversal; + labels = Utils.removeStringEle(0, labels); + return traversal.out(nestedRange, labels); + } else { + return graphTraversal.out(labels); + } + } + + private boolean isRangeExpression(String label) { + return label.matches("^\\d+\\.\\.\\d+"); + } + + @Override + public Traversal visitTraversalMethod_in(GremlinGSParser.TraversalMethod_inContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (labels != null && labels.length > 0 && isRangeExpression(labels[0])) { + GraphTraversal nestedRange = GremlinAntlrToJava.getTraversalSupplier().get(); + String[] ranges = labels[0].split("\\.\\."); + RangeGlobalStep rangeStep = + new RangeGlobalStep( + nestedRange.asAdmin(), + Integer.valueOf(ranges[0]), + Integer.valueOf(ranges[1])); + nestedRange.asAdmin().addStep(rangeStep); + IrCustomizedTraversal traversal = (IrCustomizedTraversal) graphTraversal; + labels = Utils.removeStringEle(0, labels); + return traversal.in(nestedRange, labels); + } else { + return graphTraversal.in(labels); + } + } + + @Override + public Traversal visitTraversalMethod_both(GremlinGSParser.TraversalMethod_bothContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (labels != null && labels.length > 0 && isRangeExpression(labels[0])) { + GraphTraversal nestedRange = GremlinAntlrToJava.getTraversalSupplier().get(); + String[] ranges = labels[0].split("\\.\\."); + RangeGlobalStep rangeStep = + new RangeGlobalStep( + nestedRange.asAdmin(), + Integer.valueOf(ranges[0]), + Integer.valueOf(ranges[1])); + nestedRange.asAdmin().addStep(rangeStep); + IrCustomizedTraversal traversal = (IrCustomizedTraversal) graphTraversal; + labels = Utils.removeStringEle(0, labels); + return traversal.both(nestedRange, labels); + } else { + return graphTraversal.both(labels); + } + } + + @Override + public Traversal visitTraversalMethod_outE(GremlinGSParser.TraversalMethod_outEContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (ctx.traversalMethod_inV() != null) { + return graphTraversal.out(labels); + } else { + return graphTraversal.outE(labels); + } + } + + @Override + public Traversal visitTraversalMethod_inE(GremlinGSParser.TraversalMethod_inEContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (ctx.traversalMethod_outV() != null) { + return graphTraversal.in(labels); + } else { + return graphTraversal.inE(labels); + } + } + + @Override + public Traversal visitTraversalMethod_bothE(GremlinGSParser.TraversalMethod_bothEContext ctx) { + String[] labels = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (ctx.traversalMethod_otherV() != null) { + return graphTraversal.both(labels); + } else { + return graphTraversal.bothE(labels); + } + } + + @Override + public Traversal visitTraversalMethod_limit(GremlinGSParser.TraversalMethod_limitContext ctx) { + Number integer = + (Number) + GenericLiteralVisitor.getInstance() + .visitIntegerLiteral(ctx.integerLiteral()); + return graphTraversal.limit(integer.longValue()); + } + + @Override + public Traversal visitTraversalMethod_outV(GremlinGSParser.TraversalMethod_outVContext ctx) { + return graphTraversal.outV(); + } + + @Override + public Traversal visitTraversalMethod_inV(GremlinGSParser.TraversalMethod_inVContext ctx) { + return graphTraversal.inV(); + } + + @Override + public Traversal visitTraversalMethod_endV(GremlinGSParser.TraversalMethod_endVContext ctx) { + IrCustomizedTraversal traversal = (IrCustomizedTraversal) graphTraversal; + return traversal.endV(); + } + + @Override + public Traversal visitTraversalMethod_otherV( + GremlinGSParser.TraversalMethod_otherVContext ctx) { + return graphTraversal.otherV(); + } + + @Override + public Traversal visitTraversalMethod_valueMap( + GremlinGSParser.TraversalMethod_valueMapContext ctx) { + return graphTraversal.valueMap( + GenericLiteralVisitor.getStringLiteralExpr(ctx.stringLiteralExpr())); + } + + @Override + public Traversal visitTraversalMethod_select( + GremlinGSParser.TraversalMethod_selectContext ctx) { + if (ctx.stringLiteral() != null) { + String tag = GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()); + // set tags + if (ctx.stringLiteralList() != null) { + String[] tags = GenericLiteralVisitor.getStringLiteralList(ctx.stringLiteralList()); + if (tags.length == 0) { // select one tag + graphTraversal.select(tag); + } else if (tags.length == 1) { + graphTraversal.select(tag, tags[0]); + } else { + String[] otherTags = Utils.removeStringEle(0, tags); + graphTraversal.select(tag, tags[0], otherTags); + } + } else { // select one tag + graphTraversal.select(tag); + } + } else if (ctx.traversalColumn() != null) { + Column column = + TraversalEnumParser.parseTraversalEnumFromContext( + Column.class, ctx.traversalColumn()); + graphTraversal.select(column); + } else if (ctx.traversalMethod_expr() != null) { + visitExpr(ctx.traversalMethod_expr(), ExprStep.Type.PROJECTION); + } + // set by traversal + if (ctx.traversalMethod_selectby_list() != null) { + ByModulating step = (ByModulating) graphTraversal.asAdmin().getEndStep(); + int childCount = ctx.traversalMethod_selectby_list().getChildCount(); + for (int i = 0; i < childCount; ++i) { + GremlinGSParser.TraversalMethod_selectbyContext byCtx = + ctx.traversalMethod_selectby_list().traversalMethod_selectby(i); + if (byCtx == null) continue; + int byChildCount = byCtx.getChildCount(); + // select(..).by() + if (byChildCount == 3) { + step.modulateBy(); + } else if (byChildCount == 4 + && byCtx.stringLiteral() != null) { // select(..).by('name') + step.modulateBy(GenericLiteralVisitor.getStringLiteral(byCtx.stringLiteral())); + } else if (byCtx.traversalMethod_valueMap() + != null) { // select(..).by(valueMap('name')) + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + Traversal nestedTraversal = + nestedVisitor.visitTraversalMethod_valueMap( + byCtx.traversalMethod_valueMap()); + step.modulateBy(nestedTraversal.asAdmin()); + } else if (byChildCount == 4 + && byCtx.nestedTraversal() != null) { // select(..).by(out().count()) + Traversal nestedTraversal = visitNestedTraversal(byCtx.nestedTraversal()); + step.modulateBy(nestedTraversal.asAdmin()); + } + } + } + return graphTraversal; + } + + @Override + public Traversal visitTraversalMethod_order(GremlinGSParser.TraversalMethod_orderContext ctx) { + OrderGlobalStep step = (OrderGlobalStep) graphTraversal.order().asAdmin().getEndStep(); + // set order().by(...) + if (ctx.traversalMethod_orderby_list() != null) { + int childCount = ctx.traversalMethod_orderby_list().getChildCount(); + for (int i = 0; i < childCount; ++i) { + GremlinGSParser.TraversalMethod_orderbyContext byCtx = + ctx.traversalMethod_orderby_list().traversalMethod_orderby(i); + if (byCtx == null) continue; + Order orderOpt = null; + String strAsKey = null; + Traversal nestedTraversalAskey = null; + int byChildCount = byCtx.getChildCount(); + if (byChildCount == 3) { + orderOpt = Order.asc; + } + if (byCtx.traversalOrder() != null) { + orderOpt = + TraversalEnumParser.parseTraversalEnumFromContext( + Order.class, byCtx.traversalOrder()); + } + if (byCtx.stringLiteral() != null) { + strAsKey = GenericLiteralVisitor.getStringLiteral(byCtx.stringLiteral()); + } + if (byCtx.traversalMethod_values() != null) { + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + nestedTraversalAskey = + nestedVisitor.visitTraversalMethod_values( + byCtx.traversalMethod_values()); + } + if (byCtx.traversalMethod_select() != null) { + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + nestedTraversalAskey = + nestedVisitor.visitTraversalMethod_select( + byCtx.traversalMethod_select()); + } + if (byCtx.nestedTraversal() != null) { + nestedTraversalAskey = visitNestedTraversal(byCtx.nestedTraversal()); + } + if (strAsKey != null && orderOpt == null) { + step.modulateBy(strAsKey); + } else if (strAsKey != null && orderOpt != null) { + step.modulateBy(strAsKey, orderOpt); + } else if (nestedTraversalAskey != null && orderOpt == null) { + step.modulateBy(nestedTraversalAskey.asAdmin()); + } else if (nestedTraversalAskey != null && orderOpt != null) { + step.modulateBy(nestedTraversalAskey.asAdmin(), orderOpt); + } else if (orderOpt != null) { + step.modulateBy(orderOpt); + } else { + throw new UnsupportedEvalException( + byCtx.getClass(), "pattern of order by is unsupported"); + } + } + } + return graphTraversal; + } + + @Override + public Traversal visitTraversalMethod_as(GremlinGSParser.TraversalMethod_asContext ctx) { + if (ctx.getChildCount() == 4 && ctx.stringLiteral() != null) { + return graphTraversal.as(GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + } + throw new UnsupportedEvalException(ctx.getClass(), "supported pattern is [as('..')]"); + } + + @Override + public Traversal visitTraversalMethod_group(GremlinGSParser.TraversalMethod_groupContext ctx) { + Traversal traversal = graphTraversal.group(); + if (ctx.traversalMethod_group_keyby() != null) { + traversal = visitTraversalMethod_group_keyby(ctx.traversalMethod_group_keyby()); + } + if (ctx.traversalMethod_group_valueby() != null) { + traversal = visitTraversalMethod_group_valueby(ctx.traversalMethod_group_valueby()); + } + return traversal; + } + + @Override + public Traversal visitTraversalMethod_groupCount( + GremlinGSParser.TraversalMethod_groupCountContext ctx) { + Traversal traversal = graphTraversal.groupCount(); + if (ctx.traversalMethod_group_keyby() != null) { + traversal = visitTraversalMethod_group_keyby(ctx.traversalMethod_group_keyby()); + } + return traversal; + } + + @Override + public Traversal visitTraversalMethod_group_keyby( + GremlinGSParser.TraversalMethod_group_keybyContext ctx) { + int childCount = ctx.getChildCount(); + if (childCount == 3) { + return graphTraversal.by(); + } else if (childCount == 4 && ctx.stringLiteral() != null) { + return graphTraversal.by(GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + } else if (childCount == 4 && ctx.traversalMethod_values() != null) { + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + Traversal nestedTraversal = + nestedVisitor.visitTraversalMethod_values(ctx.traversalMethod_values()); + return graphTraversal.by(nestedTraversal); + } else if (childCount >= 6 && ctx.traversalMethod_values() != null) { + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + Traversal nestedTraversal = + nestedVisitor.visitTraversalMethod_values(ctx.traversalMethod_values()); + if (ctx.traversalMethod_as() != null) { + nestedTraversal = nestedVisitor.visitTraversalMethod_as(ctx.traversalMethod_as()); + } + return graphTraversal.by(nestedTraversal); + } else if (childCount == 4 && ctx.nestedTraversal() != null) { + Traversal nestedTraversal = visitNestedTraversal(ctx.nestedTraversal()); + return graphTraversal.by(nestedTraversal); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), + "supported pattern is [group().by('..')] or [group().by(values('..'))] or" + + " [group().by(values('..')).as('..')]"); + } + } + + @Override + public Traversal visitTraversalMethod_group_valueby( + GremlinGSParser.TraversalMethod_group_valuebyContext ctx) { + int childCount = ctx.getChildCount(); + if (childCount == 3) { + return graphTraversal.by(); + } else if (childCount == 4 && ctx.traversalMethod_aggregate_func() != null) { + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + Traversal nestedTraversal = + nestedVisitor.visitTraversalMethod_aggregate_func( + ctx.traversalMethod_aggregate_func()); + return graphTraversal.by(nestedTraversal); + } else if (childCount >= 6 && ctx.traversalMethod_aggregate_func() != null) { + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + Traversal nestedTraversal = + nestedVisitor.visitTraversalMethod_aggregate_func( + ctx.traversalMethod_aggregate_func()); + if (ctx.traversalMethod_as() != null) { + nestedTraversal = nestedVisitor.visitTraversalMethod_as(ctx.traversalMethod_as()); + } + return graphTraversal.by(nestedTraversal); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), + "supported pattern is [group().by(..).by(count())] or" + + " [group().by(..).by(fold())] or [group().by(..).by(count().as('..'))] or" + + " [group().by(..).by(fold().as('..'))]"); + } + } + + @Override + public Traversal visitTraversalMethod_aggregate_func( + GremlinGSParser.TraversalMethod_aggregate_funcContext ctx) { + if (ctx.traversalMethod_count() != null) { + return visitTraversalMethod_count(ctx.traversalMethod_count()); + } else if (ctx.traversalMethod_fold() != null) { + return visitTraversalMethod_fold(ctx.traversalMethod_fold()); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [count()] or [fold()]"); + } + } + + @Override + public Traversal visitTraversalMethod_count(GremlinGSParser.TraversalMethod_countContext ctx) { + return graphTraversal.count(); + } + + @Override + public Traversal visitTraversalMethod_values( + GremlinGSParser.TraversalMethod_valuesContext ctx) { + if (ctx.getChildCount() == 4 && ctx.stringLiteral() != null) { + return graphTraversal.values( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral())); + } + throw new UnsupportedEvalException(ctx.getClass(), "supported pattern is [values('..')]"); + } + + @Override + public Traversal visitTraversalMethod_fold(GremlinGSParser.TraversalMethod_foldContext ctx) { + return graphTraversal.fold(); + } + + @Override + public Traversal visitTraversalMethod_dedup(GremlinGSParser.TraversalMethod_dedupContext ctx) { + return graphTraversal.dedup(); + } + + @Override + public Traversal visitTraversalMethod_where(GremlinGSParser.TraversalMethod_whereContext ctx) { + if (ctx.stringLiteral() != null && ctx.traversalPredicate() != null) { + graphTraversal.where( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()), + TraversalPredicateVisitor.getInstance() + .visitTraversalPredicate(ctx.traversalPredicate())); + } else if (ctx.traversalPredicate() != null) { + graphTraversal.where( + TraversalPredicateVisitor.getInstance() + .visitTraversalPredicate(ctx.traversalPredicate())); + } else if (ctx.traversalMethod_not() != null) { + visitTraversalMethod_not(ctx.traversalMethod_not()); + } else if (ctx.traversalMethod_expr() != null) { // where(expr(...)) + visitExpr(ctx.traversalMethod_expr(), ExprStep.Type.FILTER); + } else if (ctx.nestedTraversal() != null) { + Traversal whereTraversal = visitNestedTraversal(ctx.nestedTraversal()); + graphTraversal.where(whereTraversal); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), + "supported pattern is [where(P.eq(...))] or [where(a, P.eq(...))] or" + + " [where(sub_traversal)]"); + } + if (ctx.traversalMethod_whereby_list() != null) { + ByModulating byModulating = (ByModulating) graphTraversal.asAdmin().getEndStep(); + int childCount = ctx.traversalMethod_whereby_list().getChildCount(); + for (int i = 0; i < childCount; ++i) { + GremlinGSParser.TraversalMethod_wherebyContext byCtx = + ctx.traversalMethod_whereby_list().traversalMethod_whereby(i); + if (byCtx == null) continue; + int byChildCount = byCtx.getChildCount(); + if (byChildCount == 3) { // by() + byModulating.modulateBy(); + } else if (byChildCount == 4 && byCtx.stringLiteral() != null) { + byModulating.modulateBy( + GenericLiteralVisitor.getStringLiteral(byCtx.stringLiteral())); + } else if (byCtx.traversalMethod_values() != null) { + // select(..).by(valueMap()) + TraversalMethodVisitor nestedVisitor = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + Traversal nestedTraversal = + nestedVisitor.visitTraversalMethod_values( + byCtx.traversalMethod_values()); + byModulating.modulateBy(nestedTraversal.asAdmin()); + } else if (byChildCount == 4 && byCtx.nestedTraversal() != null) { + Traversal byTraversal = visitNestedTraversal(byCtx.nestedTraversal()); + byModulating.modulateBy(byTraversal.asAdmin()); + } + } + } + return graphTraversal; + } + + @Override + public Traversal visitTraversalMethod_not(GremlinGSParser.TraversalMethod_notContext ctx) { + if (ctx.nestedTraversal() != null) { + Traversal whereTraversal = visitNestedTraversal(ctx.nestedTraversal()); + return graphTraversal.not(whereTraversal); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [not(..out()..)]"); + } + } + + @Override + public Traversal visitTraversalMethod_union(GremlinGSParser.TraversalMethod_unionContext ctx) { + if (ctx.nestedTraversalExpr() != null) { + Traversal[] unionTraversals = + (new NestedTraversalSourceListVisitor((GremlinGSBaseVisitor) this)) + .visitNestedTraversalExpr(ctx.nestedTraversalExpr()); + return graphTraversal.union(unionTraversals); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [union(__.out(), ...)]"); + } + } + + @Override + public Traversal visitTraversalMethod_range(GremlinGSParser.TraversalMethod_rangeContext ctx) { + if (ctx.integerLiteral() != null && ctx.integerLiteral().size() == 2) { + Object lower = + GenericLiteralVisitor.getInstance().visitIntegerLiteral(ctx.integerLiteral(0)); + Object upper = + GenericLiteralVisitor.getInstance().visitIntegerLiteral(ctx.integerLiteral(1)); + return graphTraversal.range(((Number) lower).longValue(), ((Number) upper).longValue()); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [range(1, 2)]"); + } + } + + @Override + public Traversal visitTraversalMethod_match(GremlinGSParser.TraversalMethod_matchContext ctx) { + if (ctx.nestedTraversalExpr() != null) { + Traversal[] matchTraversals = + (new NestedTraversalSourceListVisitor((GremlinGSBaseVisitor) this)) + .visitNestedTraversalExpr(ctx.nestedTraversalExpr()); + return graphTraversal.match(matchTraversals); + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [match(as('a').out()..., ...)]"); + } + } + + public Traversal visitExpr( + GremlinGSParser.TraversalMethod_exprContext ctx, ExprStep.Type type) { + if (ctx.stringLiteral() != null) { + IrCustomizedTraversal traversal = (IrCustomizedTraversal) graphTraversal; + return traversal.expr( + GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()), type); + } else { + throw new UnsupportedEvalException(ctx.getClass(), "supported pattern is [expr(...)]"); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalPredicateVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalPredicateVisitor.java new file mode 100644 index 000000000000..8f58bb6400a0 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalPredicateVisitor.java @@ -0,0 +1,184 @@ +/* + * This file is referred and derived from project apache/tinkerpop + * + * https://github.com/apache/tinkerpop/blob/master/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/language/grammar/TraversalPredicateVisitor.java + * + * which has the following license: + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.P; + +import java.util.Collection; + +public class TraversalPredicateVisitor extends GremlinGSBaseVisitor

{ + private static TraversalPredicateVisitor instance; + + public static TraversalPredicateVisitor getInstance() { + if (instance == null) { + instance = new TraversalPredicateVisitor(); + } + return instance; + } + + private TraversalPredicateVisitor() {} + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate(final GremlinGSParser.TraversalPredicateContext ctx) { + switch (ctx.getChildCount()) { + case 1: + // handle simple predicate + return visitChildren(ctx); + case 6: + final int childIndexOfParameterOperator = 2; + final int childIndexOfCaller = 0; + final int childIndexOfArgument = 4; + + if (ctx.getChild(childIndexOfParameterOperator).getText().equals("or")) { + // handle or + return visit(ctx.getChild(childIndexOfCaller)) + .or(visit(ctx.getChild(childIndexOfArgument))); + } else { + // handle and + return visit(ctx.getChild(childIndexOfCaller)) + .and(visit(ctx.getChild(childIndexOfArgument))); + } + default: + throw new UnsupportedEvalException( + ctx.getClass(), + "unexpected number of children in TraversalPredicateContext " + + ctx.getChildCount()); + } + } + + /** + * get 1 generic literal argument from the antlr parse tree context, + * where the arguments has the child index of 2 + */ + private Object getSingleGenericLiteralArgument(final ParseTree ctx) { + final int childIndexOfParameterValue = 2; + return GenericLiteralVisitor.getInstance() + .visitGenericLiteral( + (GremlinGSParser.GenericLiteralContext) + ctx.getChild(childIndexOfParameterValue)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_eq(final GremlinGSParser.TraversalPredicate_eqContext ctx) { + return P.eq(getSingleGenericLiteralArgument(ctx)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_neq(final GremlinGSParser.TraversalPredicate_neqContext ctx) { + return P.neq(getSingleGenericLiteralArgument(ctx)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_lt(final GremlinGSParser.TraversalPredicate_ltContext ctx) { + return P.lt(getSingleGenericLiteralArgument(ctx)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_lte(final GremlinGSParser.TraversalPredicate_lteContext ctx) { + return P.lte(getSingleGenericLiteralArgument(ctx)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_gt(final GremlinGSParser.TraversalPredicate_gtContext ctx) { + return P.gt(getSingleGenericLiteralArgument(ctx)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_gte(final GremlinGSParser.TraversalPredicate_gteContext ctx) { + return P.gte(getSingleGenericLiteralArgument(ctx)); + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_within(GremlinGSParser.TraversalPredicate_withinContext ctx) { + if (ctx.genericLiteralList() != null) { + Object args = GenericLiteralVisitor.getGenericLiteralList(ctx.genericLiteralList()); + P within; + if (args instanceof Object[]) { + within = P.within((Object[]) args); + } else if (args instanceof Collection) { + within = P.within((Collection) args); + } else { + within = P.within(args); + } + return within; + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [within('a')] or [within('a', 'b')]"); + } + } + + /** + * {@inheritDoc} + */ + @Override + public P visitTraversalPredicate_without( + GremlinGSParser.TraversalPredicate_withoutContext ctx) { + if (ctx.genericLiteralList() != null) { + Object args = GenericLiteralVisitor.getGenericLiteralList(ctx.genericLiteralList()); + P without; + if (args instanceof Object[]) { + without = P.without((Object[]) args); + } else if (args instanceof Collection) { + without = P.without((Collection) args); + } else { + without = P.without(args); + } + return without; + } else { + throw new UnsupportedEvalException( + ctx.getClass(), "supported pattern is [without('a')] or [without('a', 'b')]"); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalRootVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalRootVisitor.java new file mode 100644 index 000000000000..ac44d9f63bfb --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalRootVisitor.java @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.exception.UnsupportedEvalException; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; + +public class TraversalRootVisitor extends GremlinGSBaseVisitor { + final GremlinGSBaseVisitor gvisitor; + + public TraversalRootVisitor(final GremlinGSBaseVisitor gvisitor) { + this.gvisitor = gvisitor; + } + + @Override + public Traversal visitRootTraversal(final GremlinGSParser.RootTraversalContext ctx) { + int childCount = ctx.getChildCount(); + String notice = "supported pattern of root is [g.V()] or [g.V().{...}]"; + if (childCount != 3 && childCount != 5) { + throw new UnsupportedEvalException(ctx.getClass(), notice); + } + ParseTree first = ctx.getChild(0); + GraphTraversalSource g = + this.gvisitor.visitTraversalSource((GremlinGSParser.TraversalSourceContext) first); + ParseTree third = ctx.getChild(2); + GraphTraversal graphTraversal = + (new TraversalSourceSpawnMethodVisitor(g)) + .visitTraversalSourceSpawnMethod( + (GremlinGSParser.TraversalSourceSpawnMethodContext) third); + if (childCount == 5) { + ParseTree forth = ctx.getChild(4); + TraversalMethodVisitor methodVisitor = + new TraversalMethodVisitor(this.gvisitor, graphTraversal); + return methodVisitor.visitChainedTraversal( + (GremlinGSParser.ChainedTraversalContext) forth); + } else { + return graphTraversal; + } + } + + @Override + public Traversal visitChainedTraversal(GremlinGSParser.ChainedTraversalContext ctx) { + int childCount = ctx.getChildCount(); + String notice = "supported pattern of chained is [..out()] or [..{...}.out()]"; + if (childCount != 1 && childCount != 3) { + throw new UnsupportedEvalException(ctx.getClass(), notice); + } + if (childCount == 1) { + return visitChildren(ctx); + } else { + visit(ctx.getChild(0)); + return visit(ctx.getChild(2)); + } + } + + @Override + public Traversal visitNestedTraversal(final GremlinGSParser.NestedTraversalContext ctx) { + TraversalMethodVisitor nestedTraversal = + new TraversalMethodVisitor( + gvisitor, GremlinAntlrToJava.getTraversalSupplier().get()); + return nestedTraversal.visitChainedTraversal(ctx.chainedTraversal()); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalSourceSpawnMethodVisitor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalSourceSpawnMethodVisitor.java new file mode 100644 index 000000000000..3ba3a2a1962a --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalSourceSpawnMethodVisitor.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSBaseVisitor; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; + +public class TraversalSourceSpawnMethodVisitor extends GremlinGSBaseVisitor { + final GraphTraversalSource g; + + public TraversalSourceSpawnMethodVisitor(final GraphTraversalSource g) { + this.g = g; + } + + @Override + public GraphTraversal visitTraversalSourceSpawnMethod( + GremlinGSParser.TraversalSourceSpawnMethodContext ctx) { + return visitChildren(ctx); + } + + @Override + public GraphTraversal visitTraversalSourceSpawnMethod_V( + GremlinGSParser.TraversalSourceSpawnMethod_VContext ctx) { + if (ctx.integerLiteralList().getChildCount() > 0) { + return g.V(GenericLiteralVisitor.getIntegerLiteralList(ctx.integerLiteralList())); + } else { + return g.V(); + } + } + + @Override + public GraphTraversal visitTraversalSourceSpawnMethod_E( + GremlinGSParser.TraversalSourceSpawnMethod_EContext ctx) { + if (ctx.integerLiteralList().getChildCount() > 0) { + return g.E(GenericLiteralVisitor.getIntegerLiteralList(ctx.integerLiteralList())); + } else { + return g.E(); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/ExtendGremlinStepException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/ExtendGremlinStepException.java new file mode 100644 index 000000000000..5c78087dff87 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/ExtendGremlinStepException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.exception; + +public class ExtendGremlinStepException extends IllegalArgumentException { + public ExtendGremlinStepException(String error) { + super(error); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/GremlinResultParserException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/GremlinResultParserException.java new file mode 100644 index 000000000000..f4c8535c77b5 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/GremlinResultParserException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.exception; + +public class GremlinResultParserException extends IllegalArgumentException { + public GremlinResultParserException(String error) { + super(error); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/InvalidGremlinScriptException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/InvalidGremlinScriptException.java new file mode 100644 index 000000000000..e864b1c6a8e9 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/InvalidGremlinScriptException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.exception; + +public class InvalidGremlinScriptException extends IllegalArgumentException { + public InvalidGremlinScriptException(String error) { + super(error); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/UnsupportedEvalException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/UnsupportedEvalException.java new file mode 100644 index 000000000000..921e2f78f557 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/UnsupportedEvalException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.exception; + +import org.antlr.v4.runtime.tree.ParseTree; + +public class UnsupportedEvalException extends UnsupportedOperationException { + public UnsupportedEvalException(Class antlrCtx, String error) { + super( + String.format( + "antlr context {%s} parsing to traversal is unsupported, error is {%s}", + antlrCtx, error)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/UnsupportedStepException.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/UnsupportedStepException.java new file mode 100644 index 000000000000..ab87aa42c7ba --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/exception/UnsupportedStepException.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.exception; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; + +public class UnsupportedStepException extends UnsupportedOperationException { + public UnsupportedStepException(Class type, String error) { + super(String.format("step type {%s} is unsupported, cause is {%s}", type, error)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/DummyGraph.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/DummyGraph.java new file mode 100644 index 000000000000..c47fc1767edf --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/DummyGraph.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gremlin.integration.graph; + +import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Transaction; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import java.util.Iterator; + +public abstract class DummyGraph implements Graph { + @Override + public Vertex addVertex(Object... keyValues) { + return null; + } + + @Override + public C compute(Class graphComputerClass) + throws IllegalArgumentException { + throw new UnsupportedOperationException(); + } + + @Override + public GraphComputer compute() throws IllegalArgumentException { + throw new UnsupportedOperationException(); + } + + @Override + public Iterator vertices(Object... vertexIds) { + throw new UnsupportedOperationException(); + } + + @Override + public Iterator edges(Object... edgeIds) { + throw new UnsupportedOperationException(); + } + + @Override + public Transaction tx() { + throw new UnsupportedOperationException(); + } + + @Override + public Variables variables() { + throw new UnsupportedOperationException(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteGremlinConnection.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteGremlinConnection.java new file mode 100644 index 000000000000..7e4a22405d31 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteGremlinConnection.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.graphscope.gremlin.integration.graph; + +import org.apache.tinkerpop.gremlin.driver.Cluster; +import org.apache.tinkerpop.gremlin.driver.MessageSerializer; +import org.apache.tinkerpop.gremlin.driver.remote.DriverRemoteConnection; +import org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0; +import org.apache.tinkerpop.gremlin.process.remote.RemoteConnection; +import org.apache.tinkerpop.gremlin.process.remote.RemoteConnectionException; +import org.apache.tinkerpop.gremlin.process.remote.traversal.RemoteTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; + +import java.util.concurrent.CompletableFuture; + +public class RemoteGremlinConnection implements RemoteConnection { + private RemoteConnection remoteConnection; + private Cluster cluster; + + public RemoteGremlinConnection(String endpoint) throws Exception { + this.cluster = createCluster(endpoint); + this.remoteConnection = DriverRemoteConnection.using(this.cluster); + } + + public static Cluster createCluster(String endpoint) throws Exception { + String[] split = endpoint.split(":"); + MessageSerializer serializer = new GryoMessageSerializerV1d0(); + Cluster cluster = + Cluster.build() + .addContactPoint(split[0]) + .port(Integer.valueOf(split[1])) + .credentials("admin", "admin") + .serializer(serializer) + .create(); + return cluster; + } + + @Override + public CompletableFuture> submitAsync(Bytecode bytecode) + throws RemoteConnectionException { + return remoteConnection.submitAsync(bytecode); + } + + @Override + public void close() throws Exception { + this.remoteConnection.close(); + this.cluster.close(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraph.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraph.java new file mode 100644 index 000000000000..6bdddb47878d --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraph.java @@ -0,0 +1,1504 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.integration.graph; + +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversalSource; + +import org.apache.commons.configuration2.Configuration; +import org.apache.tinkerpop.gremlin.process.traversal.AnonymousTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.finalization.ProfileStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.FilterRankingStrategy; +import org.apache.tinkerpop.gremlin.structure.Graph; + +@Graph.OptIn("com.alibaba.graphscope.integration.IrGremlinTestSuite") +@Graph.OptIn("com.alibaba.graphscope.integration.ldbc.IrLdbcTestSuite") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_groupCount_selectXvaluesX_unfold_dedup", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_repeatXdedupX_timesX2X_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_out_in_valuesXnameX_fold_dedupXlocalX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_group_byXlabelX_byXbothE_weight_dedup_foldX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasIdXwithoutXemptyXX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasNotXageX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_containingXarkXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_endingWithXasXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_startingWithXmarXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXlocationX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXlabel_isXsoftwareXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1AsStringX_out_hasXid_2AsStringX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_outXcreatedX_hasXname__mapXlengthX_isXgtX3XXX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXperson_name_containingXoX_andXltXmXXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_not_startingWithXmarXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_not_containingXarkXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_bothE_properties_dedup_hasKeyXweightX_hasValueXltX0d3XX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1X_out_hasXid_2AsString_3AsStringX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasLabelXperson_software_blahX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_not_endingWithXasXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXname_gtXmX_andXcontainingXoXXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_both_dedup_properties_hasKeyXageX_hasValueXgtX30XX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_EX11X_outV_outE_hasXid_10X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_notXhasIdXwithinXemptyXXX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VX1X_out_hasXid_lt_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = + "g_V_hasLabelXpersonX_hasXage_notXlteX10X_andXnotXbetweenX11_20XXXX_andXltX29X_orXeqX35XXXX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_both_dedup_properties_hasKeyXageX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_bothE_properties_dedup_hasKeyXweightX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasIdXemptyX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasIdXwithinXemptyXX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_E_hasLabelXuses_traversesX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_EX11X_outV_outE_hasXid_10AsStringX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", + method = "g_V_whereXinXcreatedX_count_isX1XX_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", + method = "g_V_whereXinXcreatedX_count_isXgte_2XX_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_outE_valuesXweightX_fold_orderXlocalX_skipXlocal_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_in_asXaX_in_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_limitXlocal_1X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_in_asXaX_in_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_limitXlocal_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_hasLabelXpersonX_order_byXageX_valuesXnameX_skipX1X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_out_asXaX_out_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_rangeXlocal_1_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_out_asXaX_out_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_rangeXlocal_1_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = + "g_V_asXaX_out_asXaX_out_asXaX_selectXmixed_aX_byXunfold_valuesXnameX_foldX_rangeXlocal_4_5X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_rangeX1_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_repeatXbothX_timesX3X_rangeX5_11X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_in_asXbX_in_asXcX_selectXa_b_cX_byXnameX_limitXlocal_1X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_in_asXbX_in_asXcX_selectXa_b_cX_byXnameX_limitXlocal_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_localXoutE_limitX1X_inVX_limitX3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_hasLabelXpersonX_order_byXageX_skipX1X_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_VX1X_outXcreatedX_inEXcreatedX_rangeX1_3X_outV", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_selectXa_b_cX_byXnameX_rangeXlocal_1_2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_selectXa_b_cX_byXnameX_rangeXlocal_1_3X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_simplePath_byXlabelX_fromXbX_toXcX_path_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_out_asXbX_whereXandXasXaX_outXknowsX_asXbX__orXasXbX_outXcreatedX_hasXname_rippleX__asXbX_inXknowsX_count_isXnotXeqX0XXXXX_selectXa_bX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXa_eqXbXX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_withSideEffectXa_josh_peterX_VX1X_outXcreatedX_inXcreatedX_name_whereXwithinXaXX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXa_outXknowsX_bX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_V_whereXoutXcreatedX_and_outXknowsX_or_inXknowsXX_valuesXnameX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_whereXnotXoutXcreatedXXX_name", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_out_asXbX_whereXin_count_isXeqX3XX_or_whereXoutXcreatedX_and_hasXlabel_personXXX_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_withSideEffectXa_g_VX2XX_VX1X_out_whereXneqXaXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_outEXcreatedX_asXbX_inV_asXcX_inXcreatedX_asXdX_whereXa_ltXbX_orXgtXcXX_andXneqXdXXX_byXageX_byXweightX_byXinXcreatedX_valuesXageX_minX_selectXa_c_dX", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXb_hasXname_markoXX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_VX1X_out_aggregateXxX_out_whereXnotXwithinXaXXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = + "g_V_asXaX_outXcreatedX_asXbX_whereXandXasXbX_in__notXasXaX_outXcreatedX_hasXname_rippleXXX_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + method = "g_VX1X_repeatXbothEXcreatedX_whereXwithoutXeXX_aggregateXeX_otherVX_emit_path", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", +// method = "g_V_hasXageX_asXaX_out_in_hasXageX_asXbX_selectXa_bX_whereXa_neqXbXX", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_fold_countXlocalX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_both_both_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_whereXinXkknowsX_outXcreatedX_count_is_0XX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = "g_V_hasLabelXpersonX_asXpX_VXsoftwareX_addInEXuses_pX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = "g_V_outXknowsX_V_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = + "g_V_hasXname_GarciaX_inXsungByX_asXsongX_V_hasXname_Willie_DixonX_inXwrittenByX_whereXeqXsongXX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_group_byXlabelX_byXname_order_byXdescX_foldX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXsongX_order_byXperformances_descX_byXnameX_rangeX110_120X_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_V_asXvX_mapXbothE_weight_foldX_sumXlocalX_asXsX_selectXv_sX_order_byXselectXsX_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_mapXbothE_weight_foldX_order_byXsumXlocalX_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXpersonX_fold_orderXlocalX_byXageX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_V_hasXsong_name_OHBOYX_outXfollowedByX_outXfollowedByX_order_byXperformancesX_byXsongType_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXpersonX_group_byXnameX_byXoutE_weight_sumX_orderXlocalX_byXvaluesX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_VX1X_hasXlabel_personX_mapXmapXint_ageXX_orderXlocalX_byXvalues_descX_byXkeys_ascX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_name_order_byXa1_b1X_byXb2_a2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_properties_order_byXkey_descX_key", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_hasLabelXpersonX_order_byXvalueXageX_descX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_VX1X_elementMap_orderXlocalX_byXkeys_descXunfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = + "g_V_hasLabelXpersonX_group_byXnameX_byXoutE_weight_sumX_unfold_order_byXvalues_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_both_hasLabelXpersonX_order_byXage_descX_name", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", +// method = "g_V_order_byXname_incrX_name", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_order_byXoutE_count_descX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", + method = "g_V_order_byXname_a1_b1X_byXname_b2_a2X_name", + reason = "unsupported") +// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", +// method = "g_V_outE_order_byXweight_decrX_weight", +// reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_propertiesXage_nameX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_properties_hasXid_nameIdX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_injectXg_VX1X_propertiesXnameX_nextX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_properties_hasXid_nameIdAsStringX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_propertiesXname_ageX_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest", + method = "g_V_hasXageX_propertiesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasXperson_name_markoX_barrier_asXaX_outXknows_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_hasXperson_name_markoX_elementMapXnameX_asXaX_unionXidentity_identityX_selectXaX_selectXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXall_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXlast_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_untilXout_outX_repeatXin_asXaX_in_asXbXX_selectXa_bX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXfirst_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_out_aggregateXxX_asXbX_selectXa_bX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_label_groupCount_asXxX_selectXxX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXfirst_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_outXcreatedX_unionXasXprojectX_inXcreatedX_hasXname_markoX_selectXprojectX__asXprojectX_inXcreatedX_inXknowsX_hasXname_markoX_selectXprojectXX_groupCount_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXlast_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasLabelXpersonX_asXpX_mapXbothE_label_groupCountX_asXrX_selectXp_rX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_chooseXoutE_count_isX0X__asXaX__asXbXX_chooseXselectXaX__selectXaX__selectXbXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_selectXkeysX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_unfold_selectXvaluesX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXfirst_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_unfold_selectXkeysX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXlast_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXall_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasXperson_name_markoX_path_asXaX_unionXidentity_identityX_selectXaX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_groupXmX_by_byXbothE_countX_barrier_selectXmX_selectXselectXaXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_asXaX_groupXmX_by_byXbothE_countX_barrier_selectXmX_selectXselectXaXX_byXmathX_plus_XX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_outXknowsX_asXbX_localXselectXa_bX_byXnameXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_hasXname_gremlinX_inEXusesX_order_byXskill_ascX_asXaX_outV_asXbX_selectXa_bX_byXskillX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXlast_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_untilXout_outX_repeatXin_asXaXX_selectXaX_byXtailXlocalX_nameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXa_bX_out_asXcX_path_selectXkeysX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_VX1X_groupXaX_byXconstantXaXX_byXnameX_selectXaX_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXaX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_asXaX_outXknowsX_asXaX_selectXall_constantXaXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_outE_weight_groupCount_selectXvaluesX_unfold_groupCount_selectXvaluesX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXall_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = + "g_V_hasLabelXsoftwareX_asXnameX_asXlanguageX_asXcreatorsX_selectXname_language_creatorsX_byXnameX_byXlangX_byXinXcreatedX_name_fold_orderXlocalXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXfirst_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_VX1X_asXaX_repeatXout_asXaXX_timesX2X_selectXfirst_aX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_selectXall_a_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_hasXperson_name_markoX_count_asXaX_unionXidentity_identityX_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_outE_weight_groupCount_selectXvaluesX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_V_valueMap_selectXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX4X_bothE_hasXweight_lt_1X_otherV", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_V_hasLabelXloopsX_bothEXselfX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1_2_3_4X_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_V_hasLabelXpersonX_V_hasLabelXsoftwareX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1X_outEXknowsX_bothV_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_V_hasLabelXloopsX_bothXselfX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest", + method = "g_V_valueMap_unfold_mapXkeyX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest", + method = "g_V_localXoutE_foldX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest", + method = "g_VX1X_repeatXboth_simplePathX_untilXhasIdX6XX_path_byXnameX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_withSideEffectXa__marko_666_noone_blahX_V_groupXaX_byXnameX_byXoutE_label_foldX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_repeatXout_groupXaX_byXnameX_byXcountX_timesX2X_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXnameX_by", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXlabelX_byXbothE_groupXaX_byXlabelX_byXweight_sumX_weight_sumX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_V_hasLabelXpersonX_asXpX_outXcreatedX_group_byXnameX_byXselectXpX_valuesXageX_sumX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_hasXlangX_groupXaX_byXlangX_byXnameX_out_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXoutE_countX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXaX_byXlabelX_byXoutE_weight_sumX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = + "g_V_unionXoutXknowsX__outXcreatedX_inXcreatedXX_groupCount_selectXvaluesX_unfold_sum", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = + "g_V_unionXrepeatXoutX_timesX2X_groupCountXmX_byXlangXX__repeatXinX_timesX2X_groupCountXmX_byXnameXX_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_groupCount_byXbothE_countX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_repeatXout_groupCountXaX_byXnameXX_timesX2X_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_groupCount_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_groupCountXxX_capXxX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_both_groupCountXaX_out_capXaX_selectXkeysX_unfold_both_groupCountXaX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_name_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_groupCountXaX_byXnameX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_outXcreatedX_name_groupCountXaX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = + "g_V_both_groupCountXaX_byXlabelX_asXbX_barrier_whereXselectXaX_selectXsoftwareX_isXgtX2XXX_selectXbX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_hasXnoX_groupCountXaX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_V_repeatXoutXknowsXX_untilXrepeatXoutXcreatedXX_emitXhasXname_lopXXX_path_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X_emit_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXout_repeatXoutX_timesX1XX_timesX1X_limitX1X_path_by_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_emit_repeatXoutX_timesX2X_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_repeatXoutX_untilXoutE_count_isX0XX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXbothX_untilXname_eq_marko_or_loops_gt_1X_groupCount_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_repeatXgroupCountXmX_byXloopsX_outX_timesX3X_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_VX1X_repeatXrepeatXunionXout_uses_out_traversesXX_whereXloops_isX0X_timesX1X_timeX2X_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_emitXhasXlabel_personXX_repeatXoutX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_V_hasXname_markoX_repeatXoutE_inV_simplePathX_untilXhasXname_rippleXX_path_byXnameX_byXlabelX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_V_untilXconstantXtrueXX_repeatXrepeatXout_createdXX_untilXhasXname_rippleXXXemit_lang", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_VX6X_repeatXa_bothXcreatedX_simplePathX_emitXrepeatXb_bothXknowsXX_untilXloopsXbX_asXb_whereXloopsXaX_asXbX_hasXname_vadasXX_dedup_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = + "g_VX3X_repeatXbothX_createdXX_untilXloops_is_40XXemit_repeatXin_knowsXX_emit_loopsXisX1Xdedup_values", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_hasXloop_name_loopX_repeatXinX_timesX5X_path_by_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXgroupCountXmX_byXnameX_outX_timesX2X_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXrepeatXout_createdXX_untilXhasXname_rippleXXXemit_lang", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_emit_repeatXa_outXknows_filterXloops_isX0XX_lang", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X_emit", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_emit_timesX2X_repeatXoutX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1_2X_localXunionXoutE_count__inE_count__outE_weight_sumXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_V_chooseXlabel_is_person__unionX__out_lang__out_nameX__in_labelX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1_2X_localXunionXcountXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_V_chooseXlabel_is_person__unionX__out_lang__out_nameX__in_labelX_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = + "g_V_unionXrepeatXunionXoutXcreatedX__inXcreatedXX_timesX2X__repeatXunionXinXcreatedX__outXcreatedXX_timesX2XX_label_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = + "g_V_asXaX_repeatXbothX_timesX3X_emit_name_asXbX_group_byXselectXaXX_byXselectXbX_dedup_order_foldX_selectXvaluesX_unfold_dedup", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_both_dedup_byXlabelX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_hasXlabel_softwareX_dedup_byXlangX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_out_asXxX_in_asXyX_selectXx_yX_byXnameX_fold_dedupXlocal_x_yX_unfold", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_name_order_byXa_bX_dedup_value", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_asXaX_outXcreatedX_asXbX_inXcreatedX_asXcX_dedupXa_bX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_both_dedup_byXoutE_countX_name", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_asXaX_both_asXbX_dedupXa_bX_byXlabelX_selectXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = + "g_V_both_group_by_byXout_dedup_foldX_unfold_selectXvaluesX_unfold_out_order_byXnameX_limitX1X_valuesXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXfalseX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_VX1X_out_filterXage_gt_30X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_E_filterXfalseX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_VX1X_filterXage_gt_30X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXname_startsWith_m_OR_name_startsWith_pX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXtrueX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_E_filterXtrueX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_V_filterXlang_eq_javaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.FilterTest", + method = "g_VX2X_filterXage_gt_30X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXaX_byXnameX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_out_group_byXlabelX_selectXpersonX_unfold_outXcreatedX_name_limitX2X", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = + "g_V_repeatXunionXoutXknowsX_groupXaX_byXageX__outXcreatedX_groupXbX_byXnameX_byXcountXX_groupXaX_byXnameXX_timesX2X_capXa_bX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_hasLabelXsongX_group_byXnameX_byXproperties_groupCount_byXlabelXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXname_substring_1X_byXconstantX1XX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXmX_byXnameX_byXinXknowsX_nameX_capXmX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXaX_byXname_substring_1X_byXconstantX1XX_capXaX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_hasXnoX_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_VX1X_name_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_group_byXbothE_countX_byXgroup_byXlabelXX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_hasXnoX_groupCount", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest", + method = "g_VX1X_asXaX_outXcreatedX_asXbX_inXcreatedX_asXcX_cyclicPath_fromXaX_toXbX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_repeatXoutX_timesX3X_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest", + method = "g_V_repeatXoutX_timesX8X_count", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_asXaX_hasXname_markoX_asXbX_hasXage_29X_asXcX_path", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_out_out_path_byXnameX_byXageX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_repeatXoutX_timesX2X_path_byXitX_byXnameX_byXlangX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_VX1X_out_path_byXageX_byXnameX", + reason = "unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + method = "g_V_asXaX_out_asXbX_out_asXcX_path_fromXbX_toXcX_byXnameX", + reason = "unsupported") + +// todo: return label is integer +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1X_outE", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX4X_bothEXcreatedX", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX4X_bothE", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_E_hasXlabelXknowsX", + reason = "returned label is id") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX2X_inE", + reason = "returned label is id") +@Graph.OptOut( + method = + "g_VX1X_outEXknowsX_asXhereX_hasXweight_1X_asXfakeX_inV_hasXname_joshX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") +@Graph.OptOut( + method = "g_VX1X_outEXknowsX_asXhereX_hasXweight_1X_inV_hasXname_joshX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") +@Graph.OptOut( + method = "g_VX1X_outEXknowsX_hasXweight_1X_asXhereX_inV_hasXname_joshX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") +@Graph.OptOut( + method = "g_VX1X_outE_asXhereX_inV_hasXname_vadasX_selectXhereX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "returned label is id") + +// add more ignored tests which are out of ir range +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_VX1X_timesX2X_repeatXoutX_name", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXbothX_timesX10X_asXaX_out_asXbX_selectXa_bX", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest", + method = "g_V_repeatXoutX_timesX2X_repeatXinX_timesX2X_name", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", + method = "g_VX1X_unionXrepeatXoutX_timesX2X__outX_name", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_cyclicPath_path", + reason = "cyclicPath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.CyclicPathTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_cyclicPath", + reason = "cyclicPath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_properties_dedup_count", + reason = "properties is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_bothE_properties_dedup_count", + reason = "properties is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", + method = "g_V_both_properties_properties_dedup_count", + reason = "properties is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VXv1X_hasXage_gt_30X", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_VXv4X_hasXage_gt_30X", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + method = "g_V_hasXk_withinXcXX_valuesXkX", + reason = "addV is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest", + method = "g_VX1X_outXcreatedX_inXcreatedX_simplePath", + reason = "simplePath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.SimplePathTest", + method = "g_V_repeatXboth_simplePathX_timesX3X_path", + reason = "simplePath is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest", + method = "g_VX1X_V_valuesXnameX", + reason = "g.V().V() is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + method = "g_VX1X_asXaX_repeatXout_asXaXX_timesX2X_selectXlast_aX", + reason = "repeat is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VXv1X_out", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VX1X_to_XOUT_knowsX", + reason = "to(Direction.OUT,\"knows\") is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + method = "g_VXlistXv1_v2_v3XX_name", + reason = "object id is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_VX1X_valueMapXname_locationX_byXunfoldX_by", + reason = "valueMap().by(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMap_withXtokensX", + reason = "valueMap(true) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX_withXtokensX", + reason = "valueMap(true) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX_withXtokens_idsX_byXunfoldX", + reason = "valueMap().with(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_hasLabelXpersonX_filterXoutEXcreatedXX_valueMap_withXtokensX", + reason = "valueMap().with(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + method = "g_V_valueMapXname_ageX_withXtokens_labelsX_byXunfoldX", + reason = "valueMap().with(...) is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + method = "g_V_groupXmX_byXlabelX_byXlabel_countX_capXmX", + reason = "group side effect is unsupported") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest", + method = "g_V_hasXperson_name_markoX_bothXknowsX_groupCount_byXvaluesXnameX_foldX", + reason = "fold() excluding group().by(...).by(fold()) is unsupported") + +// @Graph.OptOut(method="g_V_unionXout__inX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_outE_asXeX_inV_asXvX_selectXeX_order_byXweight_ascX_selectXvX_valuesXnameX_dedup" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupTest", reason = "will be supported") +// @Graph.OptOut(method="g_V_hasXperson_name_markoX_age" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_V_in_hasIdXneqX1XX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_hasXage_isXgt_30XX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EX7X_hasXlabelXknowsX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", + reason = "will be supported") +// @Graph.OptOut(method="g_VX1X_outE_hasXweight_inside_0_06X_inV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_valuesXageX_isXgte_29X_isXlt_34X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_valuesXageX_isXlte_30X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_valuesXageX_isX32X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_out_limitX2X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_outXknowsX_outEXcreatedX_rangeX0_1X_inV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_outXknowsX_outXcreatedX_rangeX0_1X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_VX1X_asXaX_outXcreatedX_inXcreatedX_whereXeqXaXX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outEXcreatedX_asXbX_inV_asXcX_whereXa_gtXbX_orXeqXbXXX_byXageX_byXweightX_byXweightX_selectXa_cX_byXnameX" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be supported") +// @Graph.OptOut(method="g_VX1X_asXaX_outXcreatedX_inXcreatedX_whereXneqXaXX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +@Graph.OptOut( + method = + "g_VX1X_asXaX_outXcreatedX_inXcreatedX_asXbX_whereXasXbX_outXcreatedX_hasXname_rippleXX_valuesXage_nameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", + reason = "values('age', 'name') is unsupported") +// @Graph.OptOut(method="g_VX1X_asXaX_outXcreatedX_inXcreatedX_asXbX_whereXa_neqXbXX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_whereXasXaX_name_isXjoshXX_inXcreatedX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_asXbX_inXcreatedX_asXcX_bothXknowsX_bothXknowsX_asXdX_whereXc__notXeqXaX_orXeqXdXXXX_selectXa_b_c_dX" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_inXcreatedX_asXbX_whereXa_gtXbXX_byXageX_selectXa_bX_byXnameX" , test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "will be supported") +// @Graph.OptOut(method="g_V_hasLabelXpersonX_order_byXageX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_outXcreatedX_asXbX_order_byXshuffleX_selectXa_bX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_order_byXname_ascX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_order_byXnameX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_outE_order_byXweight_descX_weight" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_both_hasLabelXpersonX_order_byXage_descX_limitX5X_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_VX1X_outEXcreatedX_inV_inE_outV_path", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EX11X_propertiesXweightX_asXaX_selectXaX_byXkeyX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_asXaX_hasXname_markoX_asXbX_asXcX_selectXa_b_cX_by_byXnameX_byXageX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "one object can only have one alias") +// @Graph.OptOut(method="g_VX1X_asXaX_outXknowsX_asXbX_selectXa_bX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_V_hasXname_isXmarkoXX_asXaX_selectXaX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_asXaX_name_order_asXbX_selectXa_bX_byXnameX_by_XitX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_asXaX_whereXoutXknowsXX_selectXaX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_EX11X_propertiesXweightX_asXaX_selectXaX_byXvalueX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest", + reason = "will be supported") +// @Graph.OptOut(method="g_VX1X_outE_otherV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_VX1AsStringX_outXknowsX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_out_outE_inV_inE_inV_both_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_EXe11X", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +// @Graph.OptOut(method="g_EX11X" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +// @Graph.OptOut(method="g_V_outE_hasXweight_1X_outV" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_EXlistXe7_e11XX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EX11AsStringX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_EXe7_e11X", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_valueMapXname_ageX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", reason = "will be +// supported") +@Graph.OptOut( + method = "g_VX1X_outXcreatedX_valueMap", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_valueMap", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest", + reason = "will be supported") +@Graph.OptOut( + method = "g_V_group_byXlabelX_byXlabel_countX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", + reason = "will be supported") +// @Graph.OptOut(method="g_V_group_byXageX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", reason = "will +// be supported") + +// @Graph.OptOut(method="g_V_hasXp_neqXvXX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXage_withoutX27X_count" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_VX1X_hasXcircumferenceX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXage_withoutX27_29X_count" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_VX1X_hasXnameX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXblahX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest", reason = "existence of +// property is unsupported") +// @Graph.OptOut(method="g_V_hasXlangX_group_byXlangX_byXcountX" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", reason = +// "existence of property is unsupported") +// @Graph.OptOut(method="g_VX1X_asXaX_out_hasXageX_whereXgtXaXX_byXageX_name" , +// test="org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTest", reason = "existence +// of property is unsupported") + +// complex steps nested in match is unsupported yet, i.e. where(eq)/count/order/match +@Graph.OptOut( + method = + "g_V_matchXa_whereXa_neqXcXX__a_created_b__orXa_knows_vadas__a_0knows_and_a_hasXlabel_personXX__b_0created_c__b_0created_count_isXgtX1XXX_selectXa_b_cX_byXidX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_created_lop_b__b_0created_29_c__c_whereXrepeatXoutX_timesX2XXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_knows_b__andXa_created_c__b_created_c__andXb_created_count_d__a_knows_count_dXXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_hasXsong_name_sunshineX__a_mapX0followedBy_weight_meanX_b__a_0followedBy_c__c_filterXweight_whereXgteXbXXX_outV_dX_selectXdX_byXnameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_created_b__a_repeatXoutX_timesX2XX_selectXa_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_knows_b__b_created_lop__b_matchXb_created_d__d_0created_cX_selectXcX_cX_selectXa_b_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_asXaX_out_asXbX_matchXa_out_count_c__orXa_knows_b__b_in_count_c__and__c_isXgtX2XXXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_outEXcreatedX_order_byXweight_descX_limitX1X_inV_b__b_hasXlang_javaXX_selectXa_bX_byXnameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_created_lop_b__b_0created_29_cX_whereXc_repeatXoutX_timesX2XX_selectXa_b_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_asXaX_out_asXbX_matchXa_out_count_c__b_in_count_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXwhereXandXa_created_b__b_0created_count_isXeqX3XXXX__a_both_b__whereXb_inXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_knows_count_bX_selectXbX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_valueMap_matchXa_selectXnameX_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// steps nested in match is unsupported yet, will be supported latter, i.e values('name') +@Graph.OptOut( + method = "g_V_notXmatchXa_age_b__a_name_cX_whereXb_eqXcXX_selectXaXX_name", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_hasLabelXsongsX_matchXa_name_b__a_performances_cX_selectXb_cX_count", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_outXknowsX_name_bX_identity", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// will be supported in the first milestone, i.e dedup('a', 'b') +@Graph.OptOut( + method = "g_V_matchXa__a_both_b__b_both_cX_dedupXa_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa__a_both_b__b_both_cX_dedupXa_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_both_b__b_both_cX_dedupXa_bX_byXlabelX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXd_0knows_a__d_hasXname_vadasX__a_knows_b__b_created_cX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_out_bX_selectXb_idX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = "g_V_matchXa_knows_b__b_created_c__a_created_cX_dedupXa_b_cX_selectXaX_byXnameX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") + +// need grateful graph +@Graph.OptOut( + method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__b_followedBy_c__c_writtenBy_d__whereXd_neqXaXXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +@Graph.OptOut( + method = + "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX", + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest", + reason = "unsupported") +public class RemoteTestGraph extends DummyGraph { + public static final String GRAPH_NAME = "test.graph.name"; + private RemoteGremlinConnection remoteGremlinConnection; + + public RemoteTestGraph(final Configuration configuration) { + try { + String gremlinEndpoint = configuration.getString(GRAPH_NAME); + this.remoteGremlinConnection = new RemoteGremlinConnection(gremlinEndpoint); + } catch (Exception e) { + throw new RuntimeException("initiate remote test graph fail " + e); + } + } + + public static RemoteTestGraph open(final Configuration configuration) { + return new RemoteTestGraph(configuration); + } + + @Override + public void close() throws Exception { + this.remoteGremlinConnection.close(); + } + + @Override + public Configuration configuration() { + return null; + } + + @Override + public GraphTraversalSource traversal() { + GraphTraversalSource graphTraversalSource = + AnonymousTraversalSource.traversal(IrCustomizedTraversalSource.class) + .withRemote(this.remoteGremlinConnection); + TraversalStrategies strategies = graphTraversalSource.getStrategies(); + strategies.removeStrategies(ProfileStrategy.class, FilterRankingStrategy.class); + return graphTraversalSource; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraphProvider.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraphProvider.java new file mode 100644 index 000000000000..21418d2735df --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraphProvider.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.integration.graph; + +import org.apache.commons.configuration2.Configuration; +import org.apache.tinkerpop.gremlin.AbstractGraphProvider; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.structure.Graph; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class RemoteTestGraphProvider extends AbstractGraphProvider { + private static String GREMLIN_ENDPOINT = "gremlin.endpoint"; + private static String DEFAULT_VALUE = "localhost:8182"; + private String gremlinEndpoint; + + public RemoteTestGraphProvider() { + String property = System.getProperty(GREMLIN_ENDPOINT); + gremlinEndpoint = (property != null) ? property : DEFAULT_VALUE; + } + + @Override + public Map getBaseConfiguration( + String graphName, + Class test, + String testMethodName, + LoadGraphWith.GraphData loadGraphWith) { + Map config = new HashMap(); + config.put(Graph.GRAPH, RemoteTestGraph.class.getName()); + config.put(RemoteTestGraph.GRAPH_NAME, gremlinEndpoint); + return config; + } + + @Override + public void clear(Graph graph, Configuration configuration) throws Exception { + if (graph != null) graph.close(); + } + + @Override + public Set getImplementations() { + return null; + } + + @Override + public void loadGraphData( + final Graph graph, + final LoadGraphWith loadGraphWith, + final Class testClass, + final String testName) { + // do nothing + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java new file mode 100644 index 000000000000..d8b57c408b53 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java @@ -0,0 +1,172 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.integration.processor; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.client.ResultParser; +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; +import com.alibaba.graphscope.gremlin.integration.result.GraphProperties; +import com.alibaba.graphscope.gremlin.integration.result.GremlinTestResultProcessor; +import com.alibaba.graphscope.gremlin.plugin.processor.IrStandardOpProcessor; +import com.alibaba.graphscope.gremlin.plugin.script.AntlrToJavaScriptEngine; +import com.alibaba.graphscope.gremlin.result.GremlinResultAnalyzer; +import com.alibaba.pegasus.service.protocol.PegasusClient; + +import org.apache.tinkerpop.gremlin.driver.Tokens; +import org.apache.tinkerpop.gremlin.driver.message.RequestMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.translator.GroovyTranslator; +import org.apache.tinkerpop.gremlin.server.Context; +import org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor; +import org.apache.tinkerpop.gremlin.util.function.ThrowingConsumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.SimpleBindings; +import javax.script.SimpleScriptContext; + +public class IrTestOpProcessor extends IrStandardOpProcessor { + private static final Logger logger = LoggerFactory.getLogger(TraversalOpProcessor.class); + private AntlrToJavaScriptEngine scriptEngine; + private ScriptContext context; + private GraphProperties testGraph; + + public IrTestOpProcessor( + Configs configs, + IrMetaFetcher irMetaFetcher, + RpcChannelFetcher fetcher, + GraphProperties testGraph) { + super(configs, irMetaFetcher, fetcher); + this.context = new SimpleScriptContext(); + Bindings globalBindings = new SimpleBindings(); + globalBindings.put("g", g); + this.context.setBindings(globalBindings, ScriptContext.ENGINE_SCOPE); + this.scriptEngine = new AntlrToJavaScriptEngine(); + this.testGraph = testGraph; + } + + @Override + public String getName() { + return "traversal"; + } + + @Override + public ThrowingConsumer select(Context ctx) { + final RequestMessage message = ctx.getRequestMessage(); + final ThrowingConsumer op; + logger.debug("tokens ops is {}", message.getOp()); + switch (message.getOp()) { + case Tokens.OPS_BYTECODE: + op = + (context -> { + Bytecode byteCode = + (Bytecode) message.getArgs().get(Tokens.ARGS_GREMLIN); + String script = getScript(byteCode); + Traversal traversal = + (Traversal) scriptEngine.eval(script, this.context); + + applyStrategies(traversal); + + // update the schema before the query is submitted + irMetaFetcher.fetch(); + + InterOpCollection opCollection = + (new InterOpCollectionBuilder(traversal)).build(); + IrPlan irPlan = opCollection.buildIrPlan(); + + byte[] physicalPlanBytes = irPlan.toPhysicalBytes(configs); + irPlan.close(); + + int serverNum = + PegasusConfig.PEGASUS_HOSTS.get(configs).split(",").length; + List servers = new ArrayList<>(); + for (long i = 0; i < serverNum; ++i) { + servers.add(i); + } + + long jobId = JOB_ID_COUNTER.incrementAndGet(); + String jobName = "ir_plan_" + jobId; + + PegasusClient.JobRequest request = + PegasusClient.JobRequest.parseFrom(physicalPlanBytes); + PegasusClient.JobConfig jobConfig = + PegasusClient.JobConfig.newBuilder() + .setJobId(jobId) + .setJobName(jobName) + .setWorkers( + PegasusConfig.PEGASUS_WORKER_NUM.get(configs)) + .setBatchSize( + PegasusConfig.PEGASUS_BATCH_SIZE.get(configs)) + .setMemoryLimit( + PegasusConfig.PEGASUS_MEMORY_LIMIT.get(configs)) + .setOutputCapacity( + PegasusConfig.PEGASUS_OUTPUT_CAPACITY.get( + configs)) + .setTimeLimit( + PegasusConfig.PEGASUS_TIMEOUT.get(configs)) + .addAllServers(servers) + .build(); + request = request.toBuilder().setConf(jobConfig).build(); + + ResultParser resultParser = GremlinResultAnalyzer.analyze(traversal); + broadcastProcessor.broadcast( + request, + new GremlinTestResultProcessor(ctx, resultParser, testGraph)); + }); + return op; + default: + RequestMessage msg = ctx.getRequestMessage(); + String errorMsg = message.getOp() + " is unsupported"; + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code(ResponseStatusCode.SERVER_ERROR_EVALUATION) + .statusMessage(errorMsg) + .create()); + return null; + } + } + + @Override + public void close() throws Exception { + this.broadcastProcessor.close(); + } + + private String getScript(Bytecode byteCode) { + String script = GroovyTranslator.of("g").translate(byteCode).getScript(); + // remove type cast from original script, g.V().has("age",P.gt((int) 30)) + List typeCastStrs = + Arrays.asList("\\(int\\)", "\\(long\\)", "\\(double\\)", "\\(boolean\\)"); + for (String type : typeCastStrs) { + script = script.replaceAll(type, ""); + } + return script; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/GraphProperties.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/GraphProperties.java new file mode 100644 index 000000000000..4c9ff319d354 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/GraphProperties.java @@ -0,0 +1,7 @@ +package com.alibaba.graphscope.gremlin.integration.result; + +import java.util.Map; + +public interface GraphProperties { + Map getProperties(); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/GremlinTestResultProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/GremlinTestResultProcessor.java new file mode 100644 index 000000000000..fd31cdef7c43 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/GremlinTestResultProcessor.java @@ -0,0 +1,124 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.integration.result; + +import com.alibaba.graphscope.common.client.ResultParser; +import com.alibaba.graphscope.gremlin.result.GremlinResultProcessor; +import com.google.common.collect.ImmutableMap; + +import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.process.remote.traversal.DefaultRemoteTraverser; +import org.apache.tinkerpop.gremlin.server.Context; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class GremlinTestResultProcessor extends GremlinResultProcessor { + private static Logger logger = LoggerFactory.getLogger(GremlinTestResultProcessor.class); + private Map cachedProperties; + private static String VERTEX_PROPERTIES = "vertex_properties"; + private static String EDGE_PROPERTIES = "edge_properties"; + + public GremlinTestResultProcessor( + Context writeResult, ResultParser resultParser, GraphProperties testGraph) { + super(writeResult, resultParser); + this.cachedProperties = testGraph.getProperties(); + } + + @Override + public void finish() { + synchronized (this) { + if (!locked) { + logger.debug("process finish"); + formatResultIfNeed(); + writeResultList(writeResult, resultCollectors, ResponseStatusCode.SUCCESS); + locked = true; + } + } + } + + @Override + protected void formatResultIfNeed() { + super.formatResultIfNeed(); + List testTraversers = + resultCollectors.stream() + .map( + k -> { + if (k instanceof DetachedVertex) { + DetachedVertex vertex = (DetachedVertex) k; + return new DetachedVertex( + vertex.id(), + vertex.label(), + getVertexProperties(vertex)); + } else if (k instanceof DetachedEdge) { + DetachedEdge edge = (DetachedEdge) k; + Vertex outVertex = edge.outVertex(); + Vertex inVertex = edge.inVertex(); + return new DetachedEdge( + edge.id(), + edge.label(), + getEdgeProperties(edge), + outVertex.id(), + outVertex.label(), + inVertex.id(), + inVertex.label()); + } else { + return k; + } + }) + .map(k -> new DefaultRemoteTraverser(k, 1)) + .collect(Collectors.toList()); + resultCollectors.clear(); + resultCollectors.addAll(testTraversers); + } + + private Map getVertexProperties(Vertex vertex) { + Map vertexProperties = + (Map) cachedProperties.get(VERTEX_PROPERTIES); + String idAsStr = String.valueOf(vertex.id()); + Map properties = (Map) vertexProperties.get(idAsStr); + if (properties != null) { + Map formatProperties = new HashMap<>(); + properties.forEach( + (k, v) -> { + formatProperties.put( + k, + Collections.singletonList(ImmutableMap.of("id", 1L, "value", v))); + }); + return formatProperties; + } else { + return Collections.emptyMap(); + } + } + + private Map getEdgeProperties(Edge edge) { + Map edgeProperties = + (Map) cachedProperties.get(EDGE_PROPERTIES); + String idAsStr = String.valueOf(edge.id()); + Map properties = (Map) edgeProperties.get(idAsStr); + return (properties == null) ? Collections.emptyMap() : properties; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java new file mode 100644 index 000000000000..dc0273f53e8c --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/result/TestGraphFactory.java @@ -0,0 +1,125 @@ +package com.alibaba.graphscope.gremlin.integration.result; + +import com.alibaba.graphscope.common.utils.JsonUtils; +import com.fasterxml.jackson.core.type.TypeReference; + +import java.util.Map; + +public enum TestGraphFactory implements GraphProperties { + EXPERIMENTAL { + @Override + public Map getProperties() { + String json = + "{\n" + + " \"vertex_properties\": {\n" + + " \"1\": {\n" + + " \"name\": \"marko\",\n" + + " \"age\": 29\n" + + " },\n" + + " \"2\": {\n" + + " \"name\": \"vadas\",\n" + + " \"age\": 27\n" + + " },\n" + + " \"72057594037927939\": {\n" + + " \"name\": \"lop\",\n" + + " \"lang\": \"java\"\n" + + " },\n" + + " \"4\": {\n" + + " \"name\": \"josh\",\n" + + " \"age\": 32\n" + + " },\n" + + " \"72057594037927941\": {\n" + + " \"name\": \"ripple\",\n" + + " \"lang\": \"java\"\n" + + " },\n" + + " \"6\": {\n" + + " \"name\": \"peter\",\n" + + " \"age\": 35\n" + + " }\n" + + " },\n" + + " \"edge_properties\": {\n" + + " \"0\": {\n" + + " \"weight\": 0.5\n" + + " },\n" + + " \"1\": {\n" + + " \"weight\": 0.4\n" + + " },\n" + + " \"2\": {\n" + + " \"weight\": 1.0\n" + + " },\n" + + " \"3\": {\n" + + " \"weight\": 0.4\n" + + " },\n" + + " \"4\": {\n" + + " \"weight\": 1.0\n" + + " },\n" + + " \"5\": {\n" + + " \"weight\": 0.2\n" + + " }\n" + + " }\n" + + "}"; + return JsonUtils.fromJson(json, new TypeReference>() {}); + } + }, + GROOT { + @Override + public Map getProperties() { + String json = + "{\n" + + " \"vertex_properties\": {\n" + + " \"-7732428334775821489\": {\n" + + " \"name\": \"marko\",\n" + + " \"age\": 29\n" + + " },\n" + + " \"6308168136910223060\": {\n" + + " \"name\": \"vadas\",\n" + + " \"age\": 27\n" + + " },\n" + + " \"-7991964441648465618\": {\n" + + " \"name\": \"lop\",\n" + + " \"lang\": \"java\"\n" + + " },\n" + + " \"-6112228345218519679\": {\n" + + " \"name\": \"josh\",\n" + + " \"age\": 32\n" + + " },\n" + + " \"2233628339503041259\": {\n" + + " \"name\": \"ripple\",\n" + + " \"lang\": \"java\"\n" + + " },\n" + + " \"-2045066182110421307\": {\n" + + " \"name\": \"peter\",\n" + + " \"age\": 35\n" + + " }\n" + + " },\n" + + " \"edge_properties\": {\n" + + " \"1000000\": {\n" + + " \"weight\": 0.5\n" + + " },\n" + + " \"1000001\": {\n" + + " \"weight\": 0.4\n" + + " },\n" + + " \"1000004\": {\n" + + " \"weight\": 1.0\n" + + " },\n" + + " \"1000003\": {\n" + + " \"weight\": 0.4\n" + + " },\n" + + " \"1000002\": {\n" + + " \"weight\": 1.0\n" + + " },\n" + + " \"1000005\": {\n" + + " \"weight\": 0.2\n" + + " }\n" + + " }\n" + + "}"; + return JsonUtils.fromJson(json, new TypeReference>() {}); + } + }, + UNKNOWN { + @Override + public Map getProperties() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrOpLoader.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrOpLoader.java new file mode 100644 index 000000000000..66d6d6c56b0b --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrOpLoader.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.processor; + +import org.apache.tinkerpop.gremlin.server.OpProcessor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public final class IrOpLoader { + private static Map processors = new HashMap<>(); + + public static void addProcessor(String name, OpProcessor processor) { + processors.put(name, processor); + } + + public static Optional getProcessor(String name) { + return Optional.ofNullable(processors.get(name)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java new file mode 100644 index 000000000000..b33c5695397e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java @@ -0,0 +1,332 @@ +/* + * This file is referred and derived from project apache/tinkerpop + * + * https://github.com/apache/tinkerpop/blob/master/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/AbstractEvalOpProcessor.java + * + * which has the following license: + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.processor; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.client.*; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.PegasusConfig; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; +import com.alibaba.graphscope.gremlin.Utils; +import com.alibaba.graphscope.gremlin.plugin.script.AntlrToJavaScriptEngineFactory; +import com.alibaba.graphscope.gremlin.plugin.strategy.RemoveUselessStepStrategy; +import com.alibaba.graphscope.gremlin.plugin.strategy.ScanFusionStepStrategy; +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversalSource; +import com.alibaba.graphscope.gremlin.result.GremlinResultAnalyzer; +import com.alibaba.graphscope.gremlin.result.GremlinResultProcessor; +import com.alibaba.pegasus.service.protocol.PegasusClient; +import com.google.protobuf.InvalidProtocolBufferException; + +import org.apache.tinkerpop.gremlin.driver.message.RequestMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; +import org.apache.tinkerpop.gremlin.groovy.jsr223.TimedInterruptTimeoutException; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies; +import org.apache.tinkerpop.gremlin.server.Context; +import org.apache.tinkerpop.gremlin.server.Settings; +import org.apache.tinkerpop.gremlin.server.op.AbstractEvalOpProcessor; +import org.apache.tinkerpop.gremlin.server.op.OpProcessorException; +import org.apache.tinkerpop.gremlin.server.op.standard.StandardOpProcessor; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.codehaus.groovy.control.MultipleCompilationErrorsException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +import javax.script.Bindings; +import javax.script.SimpleBindings; + +public class IrStandardOpProcessor extends StandardOpProcessor { + private static Logger logger = LoggerFactory.getLogger(IrStandardOpProcessor.class); + protected static final AtomicLong JOB_ID_COUNTER = new AtomicLong(0L); + protected Graph graph; + protected GraphTraversalSource g; + protected Configs configs; + protected RpcBroadcastProcessor broadcastProcessor; + protected IrMetaFetcher irMetaFetcher; + + public IrStandardOpProcessor( + Configs configs, IrMetaFetcher irMetaFetcher, RpcChannelFetcher fetcher) { + this.graph = TinkerFactory.createModern(); + this.g = graph.traversal(IrCustomizedTraversalSource.class); + this.configs = configs; + this.irMetaFetcher = irMetaFetcher; + this.broadcastProcessor = new RpcBroadcastProcessor(fetcher); + } + + @Override + protected void evalOpInternal( + final Context ctx, + final Supplier gremlinExecutorSupplier, + final AbstractEvalOpProcessor.BindingSupplier bindingsSupplier) { + com.codahale.metrics.Timer.Context timerContext = evalOpTimer.time(); + RequestMessage msg = ctx.getRequestMessage(); + GremlinExecutor gremlinExecutor = gremlinExecutorSupplier.get(); + Map args = msg.getArgs(); + String script = (String) args.get("gremlin"); + // replace with antlr parser + String language = AntlrToJavaScriptEngineFactory.ENGINE_NAME; + Bindings bindings = new SimpleBindings(); + + GremlinExecutor.LifeCycle lifeCycle = + createLifeCycle(ctx, gremlinExecutorSupplier, bindingsSupplier); + + try { + CompletableFuture evalFuture = + gremlinExecutor.eval(script, language, bindings, lifeCycle); + evalFuture.handle( + (v, t) -> { + long elapsed = timerContext.stop(); + logger.info( + "query \"{}\" total execution time is {} ms", + script, + elapsed / 1000000.0f); + if (t != null) { + Optional possibleTemporaryException = + determineIfTemporaryException(t); + if (possibleTemporaryException.isPresent()) { + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code(ResponseStatusCode.SERVER_ERROR_TEMPORARY) + .statusMessage( + ((Throwable) + possibleTemporaryException + .get()) + .getMessage()) + .statusAttributeException( + (Throwable) + possibleTemporaryException.get()) + .create()); + } else if (t instanceof OpProcessorException) { + ctx.writeAndFlush(((OpProcessorException) t).getResponseMessage()); + } else { + String errorMessage; + if (t instanceof TimedInterruptTimeoutException) { + errorMessage = + String.format( + "A timeout occurred within the script during" + + " evaluation of [%s] - consider" + + " increasing the limit given to" + + " TimedInterruptCustomizerProvider", + msg); + logger.warn(errorMessage); + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code(ResponseStatusCode.SERVER_ERROR_TIMEOUT) + .statusMessage( + "Timeout during script evaluation" + + " triggered by" + + " TimedInterruptCustomizerProvider") + .statusAttributeException(t) + .create()); + } else if (t instanceof TimeoutException) { + errorMessage = + String.format( + "Script evaluation exceeded the configured" + + " threshold for request [%s]", + msg); + logger.warn(errorMessage, t); + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code(ResponseStatusCode.SERVER_ERROR_TIMEOUT) + .statusMessage(t.getMessage()) + .statusAttributeException(t) + .create()); + } else if (t instanceof MultipleCompilationErrorsException + && t.getMessage().contains("Method too large") + && ((MultipleCompilationErrorsException) t) + .getErrorCollector() + .getErrorCount() + == 1) { + errorMessage = + String.format( + "The Gremlin statement that was submitted" + + " exceeds the maximum compilation size" + + " allowed by the JVM, please split it" + + " into multiple smaller statements - %s", + msg); + logger.warn(errorMessage); + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code( + ResponseStatusCode + .SERVER_ERROR_EVALUATION) + .statusMessage(errorMessage) + .statusAttributeException(t) + .create()); + } else { + errorMessage = + t.getMessage() == null ? t.toString() : t.getMessage(); + logger.warn( + String.format( + "Exception processing a script on request" + + " [%s].", + msg), + t); + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code( + ResponseStatusCode + .SERVER_ERROR_EVALUATION) + .statusMessage(errorMessage) + .statusAttributeException(t) + .create()); + } + } + } + return null; + }); + } catch (RejectedExecutionException var17) { + ctx.writeAndFlush( + ResponseMessage.build(msg) + .code(ResponseStatusCode.TOO_MANY_REQUESTS) + .statusMessage("Rate limiting") + .create()); + } + } + + protected GremlinExecutor.LifeCycle createLifeCycle( + Context ctx, + Supplier gremlinExecutorSupplier, + BindingSupplier bindingsSupplier) { + final RequestMessage msg = ctx.getRequestMessage(); + final Settings settings = ctx.getSettings(); + final Map args = msg.getArgs(); + long seto = + args.containsKey("evaluationTimeout") + ? ((Number) args.get("evaluationTimeout")).longValue() + : settings.getEvaluationTimeout(); + + return GremlinExecutor.LifeCycle.build() + .evaluationTimeoutOverride(seto) + .beforeEval( + b -> { + try { + b.putAll(bindingsSupplier.get()); + b.put("graph", graph); + b.put("g", g); + } catch (OpProcessorException ope) { + throw new RuntimeException(ope); + } + }) + .transformResult( + o -> { + if (o != null && o instanceof Traversal) { + applyStrategies((Traversal) o); + } + return o; + }) + .withResult( + o -> { + try { + if (o != null && o instanceof Traversal) { + // update the schema before the query is submitted + irMetaFetcher.fetch(); + + InterOpCollection opCollection = + (new InterOpCollectionBuilder((Traversal) o)).build(); + IrPlan irPlan = opCollection.buildIrPlan(); + + logger.info("{}", irPlan.getPlanAsJson()); + byte[] physicalPlanBytes = irPlan.toPhysicalBytes(configs); + irPlan.close(); + + int serverNum = PegasusConfig.PEGASUS_SERVER_NUM.get(configs); + List servers = new ArrayList<>(); + for (long i = 0; i < serverNum; ++i) { + servers.add(i); + } + + long jobId = JOB_ID_COUNTER.incrementAndGet(); + String jobName = "ir_plan_" + jobId; + + PegasusClient.JobRequest request = + PegasusClient.JobRequest.parseFrom(physicalPlanBytes); + PegasusClient.JobConfig jobConfig = + PegasusClient.JobConfig.newBuilder() + .setJobId(jobId) + .setJobName(jobName) + .setWorkers( + PegasusConfig.PEGASUS_WORKER_NUM.get( + configs)) + .setBatchSize( + PegasusConfig.PEGASUS_BATCH_SIZE.get( + configs)) + .setMemoryLimit( + PegasusConfig.PEGASUS_MEMORY_LIMIT.get( + configs)) + .setOutputCapacity( + PegasusConfig.PEGASUS_OUTPUT_CAPACITY + .get(configs)) + .setTimeLimit( + PegasusConfig.PEGASUS_TIMEOUT.get( + configs)) + .addAllServers(servers) + .build(); + request = request.toBuilder().setConf(jobConfig).build(); + + ResultParser resultParser = + GremlinResultAnalyzer.analyze((Traversal) o); + broadcastProcessor.broadcast( + request, new GremlinResultProcessor(ctx, resultParser)); + } + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .create(); + } + + public static void applyStrategies(Traversal traversal) { + TraversalStrategies traversalStrategies = traversal.asAdmin().getStrategies(); + Set> strategies = + Utils.getFieldValue( + DefaultTraversalStrategies.class, + traversalStrategies, + "traversalStrategies"); + strategies.clear(); + strategies.add(ScanFusionStepStrategy.instance()); + strategies.add(RemoveUselessStepStrategy.instance()); + traversal.asAdmin().applyStrategies(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrToJavaScriptEngine.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrToJavaScriptEngine.java new file mode 100644 index 000000000000..c5b40eaa6ca2 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrToJavaScriptEngine.java @@ -0,0 +1,131 @@ +/* + * This file is referred and derived from project apache/tinkerpop + * + * https://github.com/apache/tinkerpop/blob/3.5.1/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngine.java + * + * which has the following license: + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.script; + +import com.alibaba.graphscope.gremlin.antlr4.GremlinAntlrToJava; +import com.alibaba.graphscope.gremlin.exception.InvalidGremlinScriptException; + +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSLexer; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinGSParser; +import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Reader; + +import javax.script.*; + +public class AntlrToJavaScriptEngine extends AbstractScriptEngine implements GremlinScriptEngine { + private Logger logger = LoggerFactory.getLogger(AntlrToJavaScriptEngine.class); + private volatile AntlrToJavaScriptEngineFactory factory; + + @Override + public Object eval(String script, ScriptContext ctx) { + logger.debug("antlr start to eval \"{}\"", script); + Bindings globalBindings = ctx.getBindings(ScriptContext.ENGINE_SCOPE); + GraphTraversalSource g = (GraphTraversalSource) globalBindings.get("g"); + GremlinAntlrToJava antlrToJava = GremlinAntlrToJava.getInstance(g); + + try { + GremlinGSLexer lexer = new GremlinGSLexer(CharStreams.fromString(script)); + lexer.removeErrorListeners(); + lexer.addErrorListener( + new BaseErrorListener() { + @Override + public void syntaxError( + final Recognizer recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String msg, + final RecognitionException e) { + throw new ParseCancellationException(); + } + }); + // setup error handler on parser + final GremlinGSParser parser = new GremlinGSParser(new CommonTokenStream(lexer)); + parser.setErrorHandler(new BailErrorStrategy()); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + return antlrToJava.visit(parser.query()); + } catch (ParseCancellationException e) { + Throwable t = ExceptionUtils.getRootCause(e); + // todo: return user-friendly errors from different exceptions + String error = + String.format( + "query [%s] is invalid, check the grammar in GremlinGS.g4, ", script); + if (t instanceof LexerNoViableAltException) { + error += + String.format( + "failed at index: %s.", + ((LexerNoViableAltException) t).getStartIndex()); + } else if (t instanceof NoViableAltException) { + error += + String.format( + "token: %s.", + ((NoViableAltException) t).getStartToken().toString()); + } else { + error += String.format("message: %s.", t.getMessage()); + } + throw new InvalidGremlinScriptException(error); + } + } + + @Override + public Object eval(Reader reader, ScriptContext context) { + throw new NotImplementedException("use eval(String, ScriptContext) instead"); + } + + @Override + public Traversal.Admin eval(Bytecode bytecode, Bindings bindings, String traversalSource) { + throw new NotImplementedException("use eval(String, ScriptContext) instead"); + } + + @Override + public Bindings createBindings() { + return new SimpleBindings(); + } + + @Override + public GremlinScriptEngineFactory getFactory() { + if (this.factory == null) { + synchronized (this) { + if (this.factory == null) { + this.factory = new AntlrToJavaScriptEngineFactory(); + } + } + } + return this.factory; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrToJavaScriptEngineFactory.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrToJavaScriptEngineFactory.java new file mode 100644 index 000000000000..d4830fb7fa92 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrToJavaScriptEngineFactory.java @@ -0,0 +1,59 @@ +/* + * This file is referred and derived from project apache/tinkerpop + * + * https://github.com/apache/tinkerpop/blob/3.5.1/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GremlinGroovyScriptEngineFactory.java + * + * which has the following license: + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.script; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.tinkerpop.gremlin.jsr223.AbstractGremlinScriptEngineFactory; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; + +import java.util.Collections; +import java.util.List; + +public class AntlrToJavaScriptEngineFactory extends AbstractGremlinScriptEngineFactory { + public static final String ENGINE_NAME = "antlr-to-java"; + private static final String LANGUAGE_NAME = "antlr-to-java"; + private static final String PLAIN = "plain"; + private static final List EXTENSIONS = Collections.singletonList("gremlin"); + + public AntlrToJavaScriptEngineFactory() { + super(ENGINE_NAME, LANGUAGE_NAME, EXTENSIONS, Collections.singletonList(PLAIN)); + } + + @Override + public String getMethodCallSyntax(String obj, String m, String... args) { + throw new NotImplementedException(); + } + + @Override + public String getOutputStatement(String toDisplay) { + throw new NotImplementedException(); + } + + @Override + public GremlinScriptEngine getScriptEngine() { + return new AntlrToJavaScriptEngine(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/ExprStep.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/ExprStep.java new file mode 100644 index 000000000000..4f7cf1a0cb01 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/ExprStep.java @@ -0,0 +1,36 @@ +package com.alibaba.graphscope.gremlin.plugin.step; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.MapStep; + +import java.util.NoSuchElementException; + +public class ExprStep extends MapStep { + private String expr; + private Type type; + + public ExprStep(Traversal.Admin traversal, String expr, Type type) { + super(traversal); + this.expr = expr; + this.type = type; + } + + @Override + protected Traverser.Admin processNextStart() throws NoSuchElementException { + return null; + } + + public String getExpr() { + return expr; + } + + public Type getType() { + return type; + } + + public enum Type { + FILTER, + PROJECTION + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/PathExpandStep.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/PathExpandStep.java new file mode 100644 index 000000000000..9c634296189c --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/PathExpandStep.java @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.step; + +import com.alibaba.graphscope.gremlin.exception.ExtendGremlinStepException; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +import java.util.Iterator; + +public class PathExpandStep extends VertexStep { + private String[] edgeLabels; + private Direction direction; + private Traversal rangeTraversal; + + public PathExpandStep( + final Traversal.Admin traversal, + final Direction direction, + final Traversal rangeTraversal, + final String... edgeLabels) { + super(traversal, Vertex.class, direction, edgeLabels); + this.direction = direction; + this.rangeTraversal = rangeTraversal; + this.edgeLabels = edgeLabels; + } + + public Direction getDirection() { + return this.direction; + } + + public String[] getEdgeLabels() { + return this.edgeLabels; + } + + public int getLower() { + Traversal.Admin admin = rangeTraversal.asAdmin(); + if (admin.getSteps().size() == 1 && admin.getStartStep() instanceof RangeGlobalStep) { + RangeGlobalStep range = (RangeGlobalStep) admin.getStartStep(); + return (int) range.getLowRange(); + } else { + throw new ExtendGremlinStepException( + "rangeTraversal should only have one RangeGlobalStep"); + } + } + + public int getUpper() { + Traversal.Admin admin = rangeTraversal.asAdmin(); + if (admin.getSteps().size() == 1 && admin.getStartStep() instanceof RangeGlobalStep) { + RangeGlobalStep range = (RangeGlobalStep) admin.getStartStep(); + return (int) range.getHighRange(); + } else { + throw new ExtendGremlinStepException( + "rangeTraversal should only have one RangeGlobalStep"); + } + } + + @Override + public void close() { + throw new ExtendGremlinStepException("unsupported"); + } + + @Override + protected Iterator flatMap(Traverser.Admin traverser) { + throw new ExtendGremlinStepException("unsupported"); + } + + @Override + public void configure(Object... keyValues) { + throw new ExtendGremlinStepException("unsupported"); + } + + @Override + public Parameters getParameters() { + throw new ExtendGremlinStepException("unsupported"); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/ScanFusionStep.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/ScanFusionStep.java new file mode 100644 index 000000000000..cc05a68389cd --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/step/ScanFusionStep.java @@ -0,0 +1,138 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.step; + +import com.alibaba.graphscope.gremlin.exception.ExtendGremlinStepException; +import com.google.common.base.Objects; + +import org.apache.tinkerpop.gremlin.process.traversal.Compare; +import org.apache.tinkerpop.gremlin.process.traversal.Contains; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; +import org.apache.tinkerpop.gremlin.process.traversal.util.AndP; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.T; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +// fuse global ids and labels with the scan operator +public class ScanFusionStep extends GraphStep + implements HasContainerHolder, AutoCloseable { + private final List hasContainers = new ArrayList<>(); + private final List graphLabels = new ArrayList<>(); + + public ScanFusionStep(final GraphStep originalGraphStep) { + super( + originalGraphStep.getTraversal(), + originalGraphStep.getReturnClass(), + originalGraphStep.isStartStep(), + originalGraphStep.getIds()); + originalGraphStep.getLabels().forEach(this::addLabel); + } + + @Override + public String toString() { + return StringFactory.stepString( + this, + new Object[] { + this.returnClass.getSimpleName().toLowerCase(), + Arrays.toString(this.ids), + this.graphLabels, + this.hasContainers + }); + } + + @Override + public List getHasContainers() { + return Collections.unmodifiableList(this.hasContainers); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + ScanFusionStep that = (ScanFusionStep) o; + return Objects.equal(hasContainers, that.hasContainers) + && Objects.equal(graphLabels, that.graphLabels); + } + + @Override + public int hashCode() { + return Objects.hashCode(super.hashCode(), hasContainers, graphLabels); + } + + @Override + public void addHasContainer(final HasContainer hasContainer) { + if (hasContainer.getPredicate() instanceof AndP) { + for (final P predicate : ((AndP) hasContainer.getPredicate()).getPredicates()) { + this.addHasContainer(new HasContainer(hasContainer.getKey(), predicate)); + } + } else this.hasContainers.add(hasContainer); + } + + public List getGraphLabels() { + return Collections.unmodifiableList(this.graphLabels); + } + + public void addGraphLabels(String label) { + this.graphLabels.add(label); + } + + public static boolean processHasLabels( + final ScanFusionStep graphStep, + final HasContainer hasContainer, + List originalContainers) { + if (!hasContainer.getKey().equals(T.label.getAccessor()) + || graphStep.getIds().length != 0 + || graphStep.getGraphLabels().size() != 0 + || hasContainer.getBiPredicate() != Compare.eq + && hasContainer.getBiPredicate() != Contains.within) { + return false; + } + if (getContainer(originalContainers, T.id.getAccessor()) != null) { + return false; + } else { + P predicate = hasContainer.getPredicate(); + if (predicate.getValue() instanceof List + && ((List) predicate.getValue()).size() > 0 + && ((List) predicate.getValue()).get(0) instanceof String) { + List values = (List) predicate.getValue(); + values.forEach(k -> graphStep.addGraphLabels(k)); + } else if (predicate.getValue() instanceof String) { + graphStep.addGraphLabels((String) predicate.getValue()); + } else { + throw new ExtendGremlinStepException( + "hasLabel value type not support " + predicate.getValue().getClass()); + } + return true; + } + } + + public static HasContainer getContainer(List originalContainers, String key) { + for (HasContainer container : originalContainers) { + if (container.getKey().equals(key)) return container; + } + return null; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/strategy/RemoveUselessStepStrategy.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/strategy/RemoveUselessStepStrategy.java new file mode 100644 index 000000000000..9105cb45e61c --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/strategy/RemoveUselessStepStrategy.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.strategy; + +import com.alibaba.graphscope.gremlin.Utils; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTraversalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.ComputerAwareStep; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; + +import java.util.List; + +public class RemoveUselessStepStrategy + extends AbstractTraversalStrategy + implements TraversalStrategy.ProviderOptimizationStrategy { + private static final RemoveUselessStepStrategy INSTANCE = new RemoveUselessStepStrategy(); + + private RemoveUselessStepStrategy() {} + + public static RemoveUselessStepStrategy instance() { + return INSTANCE; + } + + @Override + public void apply(Traversal.Admin traversal) { + List steps = traversal.getSteps(); + for (int i = 0; i < steps.size(); ++i) { + Step step = steps.get(i); + if (Utils.equalClass(step, ComputerAwareStep.EndStep.class) + || Utils.equalClass(step, NoOpBarrierStep.class)) { + traversal.removeStep(step); + } else if (Utils.equalClass(step, WhereTraversalStep.WhereStartStep.class) + && ((WhereTraversalStep.WhereStartStep) step).getScopeKeys().isEmpty()) { + traversal.removeStep(step); + } + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/strategy/ScanFusionStepStrategy.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/strategy/ScanFusionStepStrategy.java new file mode 100644 index 000000000000..c2555749896e --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/strategy/ScanFusionStepStrategy.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.strategy; + +import com.alibaba.graphscope.gremlin.plugin.step.ScanFusionStep; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; + +import java.util.List; + +public class ScanFusionStepStrategy + extends AbstractTraversalStrategy + implements TraversalStrategy.ProviderOptimizationStrategy { + private static final ScanFusionStepStrategy INSTANCE = new ScanFusionStepStrategy(); + + private ScanFusionStepStrategy() {} + + public static ScanFusionStepStrategy instance() { + return INSTANCE; + } + + @Override + public void apply(Traversal.Admin traversal) { + for (final GraphStep originalGraphStep : + TraversalHelper.getStepsOfClass(GraphStep.class, traversal)) { + final ScanFusionStep scanFusionStep = new ScanFusionStep<>(originalGraphStep); + TraversalHelper.replaceStep(originalGraphStep, scanFusionStep, traversal); + Step currentStep = scanFusionStep.getNextStep(); + while (currentStep instanceof HasStep || currentStep instanceof NoOpBarrierStep) { + if (currentStep instanceof HasStep) { + List originalContainers = + ((HasContainerHolder) currentStep).getHasContainers(); + for (final HasContainer hasContainer : originalContainers) { + if (!GraphStep.processHasContainerIds(scanFusionStep, hasContainer) + && !ScanFusionStep.processHasLabels( + scanFusionStep, hasContainer, originalContainers)) { + scanFusionStep.addHasContainer(hasContainer); + } + } + TraversalHelper.copyLabels(currentStep, currentStep.getPreviousStep(), false); + traversal.removeStep(currentStep); + } + currentStep = currentStep.getNextStep(); + } + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/traversal/IrCustomizedTraversal.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/traversal/IrCustomizedTraversal.java new file mode 100644 index 000000000000..8fe9c5712d85 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/traversal/IrCustomizedTraversal.java @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.traversal; + +import com.alibaba.graphscope.gremlin.plugin.step.ExprStep; +import com.alibaba.graphscope.gremlin.plugin.step.PathExpandStep; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep; +import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; + +public class IrCustomizedTraversal extends DefaultTraversal + implements GraphTraversal.Admin { + public IrCustomizedTraversal() { + super(); + } + + public IrCustomizedTraversal(final GraphTraversalSource graphTraversalSource) { + super(graphTraversalSource); + } + + public IrCustomizedTraversal(final Graph graph) { + super(graph); + } + + @Override + public GraphTraversal.Admin asAdmin() { + return this; + } + + @Override + public GraphTraversal iterate() { + return GraphTraversal.Admin.super.iterate(); + } + + @Override + public IrCustomizedTraversal clone() { + return (IrCustomizedTraversal) super.clone(); + } + + public GraphTraversal out(Traversal rangeTraversal, String... labels) { + this.asAdmin().getBytecode().addStep("flatMap", rangeTraversal, labels); + return this.asAdmin() + .addStep(new PathExpandStep(this.asAdmin(), Direction.OUT, rangeTraversal, labels)); + } + + public GraphTraversal in(Traversal rangeTraversal, String... labels) { + this.asAdmin().getBytecode().addStep("flatMap", rangeTraversal, labels); + return this.asAdmin() + .addStep(new PathExpandStep(this.asAdmin(), Direction.IN, rangeTraversal, labels)); + } + + public GraphTraversal both(Traversal rangeTraversal, String... labels) { + this.asAdmin().getBytecode().addStep("flatMap", rangeTraversal, labels); + return this.asAdmin() + .addStep( + new PathExpandStep(this.asAdmin(), Direction.BOTH, rangeTraversal, labels)); + } + + public GraphTraversal expr(String expr, ExprStep.Type type) { + this.asAdmin().getBytecode().addStep("expr", expr, type); + return this.asAdmin().addStep(new ExprStep(this.asAdmin(), expr, type)); + } + + public GraphTraversal endV() { + this.asAdmin().getBytecode().addStep("endV", new Object[0]); + return this.asAdmin().addStep(new EdgeVertexStep(this.asAdmin(), Direction.IN)); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/traversal/IrCustomizedTraversalSource.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/traversal/IrCustomizedTraversalSource.java new file mode 100644 index 000000000000..2644378057af --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/traversal/IrCustomizedTraversalSource.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.plugin.traversal; + +import org.apache.tinkerpop.gremlin.process.remote.RemoteConnection; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal.Admin; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies.GlobalCache; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +public class IrCustomizedTraversalSource extends GraphTraversalSource { + public IrCustomizedTraversalSource( + final Graph graph, final TraversalStrategies traversalStrategies) { + super(graph, traversalStrategies); + } + + public IrCustomizedTraversalSource(final Graph graph) { + this(graph, GlobalCache.getStrategies(graph.getClass())); + } + + public IrCustomizedTraversalSource(final RemoteConnection connection) { + super(connection); + } + + @Override + public IrCustomizedTraversalSource clone() { + IrCustomizedTraversalSource clone = (IrCustomizedTraversalSource) super.clone(); + clone.strategies = this.strategies.clone(); + clone.bytecode = this.bytecode.clone(); + return clone; + } + + @Override + public IrCustomizedTraversal V(final Object... vertexIds) { + IrCustomizedTraversalSource clone = this.clone(); + clone.bytecode.addStep("V", vertexIds); + Admin traversal = new IrCustomizedTraversal(clone); + return (IrCustomizedTraversal) + traversal.addStep(new GraphStep(traversal, Vertex.class, true, vertexIds)); + } + + @Override + public IrCustomizedTraversal E(final Object... edgesIds) { + IrCustomizedTraversalSource clone = this.clone(); + clone.bytecode.addStep("E", edgesIds); + Admin traversal = new IrCustomizedTraversal(clone); + return (IrCustomizedTraversal) + traversal.addStep(new GraphStep(traversal, Edge.class, true, edgesIds)); + } + + @Override + public void close() throws Exception { + if (this.connection != null) { + this.connection.close(); + } + } + + @Override + public String toString() { + return StringFactory.traversalSourceString(this); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/EmptyValue.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/EmptyValue.java new file mode 100644 index 000000000000..b37dda52611f --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/EmptyValue.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.result; + +public class EmptyValue { + public static EmptyValue INSTANCE = new EmptyValue(); + + private EmptyValue() {} +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultAnalyzer.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultAnalyzer.java new file mode 100644 index 000000000000..a6df8c8c9279 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultAnalyzer.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.result; + +import com.alibaba.graphscope.gremlin.Utils; +import com.alibaba.graphscope.gremlin.exception.UnsupportedStepException; +import com.alibaba.graphscope.gremlin.plugin.step.ExprStep; +import com.alibaba.graphscope.gremlin.plugin.step.PathExpandStep; +import com.alibaba.graphscope.gremlin.plugin.step.ScanFusionStep; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; + +import java.util.List; + +public class GremlinResultAnalyzer { + public static GremlinResultParser analyze(Traversal traversal) { + List steps = traversal.asAdmin().getSteps(); + GremlinResultParser parserType = GremlinResultParserFactory.GRAPH_ELEMENT; + for (Step step : steps) { + if (Utils.equalClass(step, GraphStep.class) + || Utils.equalClass(step, ScanFusionStep.class) + || Utils.equalClass(step, VertexStep.class) + || Utils.equalClass(step, EdgeVertexStep.class) + || Utils.equalClass(step, EdgeOtherVertexStep.class) + || Utils.equalClass(step, PathExpandStep.class)) { + parserType = GremlinResultParserFactory.GRAPH_ELEMENT; + } else if (Utils.equalClass(step, CountGlobalStep.class)) { + parserType = GremlinResultParserFactory.SINGLE_VALUE; + } else if (Utils.equalClass(step, SelectOneStep.class) + || Utils.equalClass(step, SelectStep.class) + || Utils.equalClass(step, PropertiesStep.class) + || Utils.equalClass(step, PropertyMapStep.class) + || Utils.equalClass(step, TraversalMapStep.class) + || Utils.equalClass(step, MatchStep.class) + || Utils.equalClass(step, ExprStep.class)) { + parserType = GremlinResultParserFactory.PROJECT_VALUE; + } else if (Utils.equalClass(step, GroupCountStep.class) + || Utils.equalClass(step, GroupStep.class)) { + parserType = GremlinResultParserFactory.GROUP; + } else if (Utils.equalClass(step, UnionStep.class)) { + parserType = GremlinResultParserFactory.UNION; + } else if (Utils.equalClass(step, HasStep.class) + || Utils.equalClass(step, DedupGlobalStep.class) + || Utils.equalClass(step, RangeGlobalStep.class) + || Utils.equalClass(step, OrderGlobalStep.class) + || Utils.equalClass(step, IsStep.class) + || Utils.equalClass(step, WherePredicateStep.class) + || Utils.equalClass(step, TraversalFilterStep.class) + || Utils.equalClass(step, WhereTraversalStep.class) + || Utils.equalClass(step, NotStep.class)) { + // do nothing; + } else { + throw new UnsupportedStepException(step.getClass(), "unimplemented yet"); + } + } + return parserType; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultParser.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultParser.java new file mode 100644 index 000000000000..f32549f4cf50 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultParser.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.result; + +import com.alibaba.graphscope.common.client.ResultParser; +import com.alibaba.graphscope.gaia.proto.IrResult; +import com.alibaba.graphscope.gremlin.exception.GremlinResultParserException; +import com.alibaba.pegasus.service.protocol.PegasusClient; +import com.google.protobuf.InvalidProtocolBufferException; + +import java.util.Collections; +import java.util.List; + +public interface GremlinResultParser extends ResultParser { + @Override + default List parseFrom(PegasusClient.JobResponse response) { + try { + IrResult.Results results = IrResult.Results.parseFrom(response.getData()); + Object parseResult = parseFrom(results); + if (parseResult instanceof EmptyValue) { + return Collections.emptyList(); + } else { + return Collections.singletonList(parseResult); + } + } catch (InvalidProtocolBufferException e) { + throw new GremlinResultParserException("parse from proto failed " + e); + } + } + + Object parseFrom(IrResult.Results results); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultParserFactory.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultParserFactory.java new file mode 100644 index 000000000000..8b64981be362 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultParserFactory.java @@ -0,0 +1,229 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.result; + +import com.alibaba.graphscope.gaia.proto.Common; +import com.alibaba.graphscope.gaia.proto.IrResult; +import com.alibaba.graphscope.gaia.proto.OuterExpression; +import com.alibaba.graphscope.gremlin.exception.GremlinResultParserException; +import com.alibaba.graphscope.gremlin.transform.alias.AliasManager; + +import org.apache.tinkerpop.gremlin.structure.Element; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public enum GremlinResultParserFactory implements GremlinResultParser { + GRAPH_ELEMENT { + @Override + public Object parseFrom(IrResult.Results results) { + IrResult.Element element = ParserUtils.getHeadEntry(results).getElement(); + Object graphElement = ParserUtils.parseElement(element); + if (!(graphElement instanceof Element || graphElement instanceof List)) { + throw new GremlinResultParserException( + "parse element should return vertex or edge or graph path"); + } + return graphElement; + } + }, + SINGLE_VALUE { + @Override + public Object parseFrom(IrResult.Results results) { + IrResult.Element element = ParserUtils.getHeadEntry(results).getElement(); + return ParserUtils.parseElement(element); + } + }, + PROJECT_VALUE { + // values("name") -> key: head, value: "marko" + // valueMap("name") -> key: head, value: {name, "marko"} + // select("a").by("name") -> key: head, value: "marko" + // select("a", "b").by("name") -> key: a, value: "marko"; key: b, value: "josh" + // select("a", "b").by(valueMap("name")) -> key: a, value: {name, "marko"}; key: b, value: + // {name, "josh"} + @Override + public Object parseFrom(IrResult.Results results) { + IrResult.Record record = results.getRecord(); + logger.info("{}", record); + Map projectResult = new HashMap<>(); + record.getColumnsList() + .forEach( + column -> { + String tag = getColumnKeyAsResultKey(column.getNameOrId()); + Object parseElement = + ParserUtils.parseElement(column.getEntry().getElement()); + if (parseElement instanceof Map) { + Map projectTags = + (Map) parseElement; + projectTags.forEach( + (k, v) -> { + if (!(v instanceof EmptyValue)) { + String property = (String) k.get(1); + if (property.isEmpty()) { + throw new GremlinResultParserException( + "map value should have property" + + " key"); + } + Map tagEntry = + (Map) + projectResult.computeIfAbsent( + tag, + k1 -> new HashMap<>()); + tagEntry.put( + property, Collections.singletonList(v)); + } + }); + } else { + if (!(parseElement instanceof EmptyValue)) { + projectResult.put(tag, parseElement); + } + } + }); + if (projectResult.isEmpty()) { + return EmptyValue.INSTANCE; + } else if (projectResult.size() == 1) { + return projectResult.entrySet().iterator().next().getValue(); + } else { + return projectResult; + } + } + + // a_1 -> a, i.e. g.V().as("a").select("a") + // name_1 -> name, i.e. g.V().values("name") + // a_name_1 -> a, i.e. g.V().as("a").select("a").by("name") + private String getColumnKeyAsResultKey(OuterExpression.NameOrId columnKey) { + if (columnKey.getItemCase() == OuterExpression.NameOrId.ItemCase.ITEM_NOT_SET) { + return ""; + } + switch (columnKey.getItemCase()) { + case ITEM_NOT_SET: + return ""; + case NAME: + String key = columnKey.getName(); + return AliasManager.getPrefix(key); + case ID: + return String.valueOf(columnKey.getId()); + default: + throw new GremlinResultParserException(columnKey.getItemCase() + " is invalid"); + } + } + }, + GROUP { + @Override + public Map parseFrom(IrResult.Results results) { + IrResult.Record record = results.getRecord(); + Object key = null; + Object value = null; + for (IrResult.Column column : record.getColumnsList()) { + OuterExpression.NameOrId columnName = column.getNameOrId(); + if (columnName.getItemCase() != OuterExpression.NameOrId.ItemCase.NAME) { + throw new GremlinResultParserException( + "column key in group should be ItemCase.NAME"); + } + String alias = columnName.getName(); + Object parseEntry = parseGroupEntry(column.getEntry()); + if (parseEntry instanceof EmptyValue) { + continue; + } + if (AliasManager.isGroupKeysPrefix(alias)) { + key = parseEntry; + } else { + value = parseEntry; + } + } + // key or value can be null + Map data = new HashMap(); + data.put(key, value); + return data; + } + + private Object parseGroupEntry(IrResult.Entry entry) { + switch (entry.getInnerCase()) { + case ELEMENT: + return ParserUtils.parseElement(entry.getElement()); + case COLLECTION: + return ParserUtils.parseCollection(entry.getCollection()); + default: + throw new GremlinResultParserException(entry.getInnerCase() + " is invalid"); + } + } + }, + UNION { + @Override + public Object parseFrom(IrResult.Results results) { + GremlinResultParser resultParser = inferFromIrResults(results); + return resultParser.parseFrom(results); + } + + // try to infer from the results + private GremlinResultParser inferFromIrResults(IrResult.Results results) { + int columns = results.getRecord().getColumnsList().size(); + logger.debug("result is {}", results); + if (columns == 1) { + IrResult.Entry entry = ParserUtils.getHeadEntry(results); + switch (entry.getInnerCase()) { + case ELEMENT: + IrResult.Element element = entry.getElement(); + if (element.getInnerCase() == IrResult.Element.InnerCase.VERTEX + || element.getInnerCase() == IrResult.Element.InnerCase.EDGE + || element.getInnerCase() + == IrResult.Element.InnerCase.GRAPH_PATH) { + return GRAPH_ELEMENT; + } else if (element.getInnerCase() == IrResult.Element.InnerCase.OBJECT) { + Common.Value value = element.getObject(); + if (value.getItemCase() + == Common.Value.ItemCase.PAIR_ARRAY) { // project + return PROJECT_VALUE; + } else { // simple type + return SINGLE_VALUE; + } + } else { + throw new GremlinResultParserException( + element.getInnerCase() + " is invalid"); + } + case COLLECTION: // path() + default: + throw new GremlinResultParserException( + entry.getInnerCase() + " is unsupported yet"); + } + } else if (columns > 1) { // project or group + IrResult.Column column = results.getRecord().getColumnsList().get(0); + OuterExpression.NameOrId columnName = column.getNameOrId(); + if (columnName.getItemCase() == OuterExpression.NameOrId.ItemCase.NAME) { + String name = columnName.getName(); + if (AliasManager.isGroupKeysPrefix(name) + || AliasManager.isGroupValuesPrefix(name)) { + return GROUP; + } else { + return PROJECT_VALUE; + } + } else { + throw new GremlinResultParserException( + "column key should be ItemCase.NAME to differentiate between group and" + + " project"); + } + } else { + throw new GremlinResultParserException("columns should not be empty"); + } + } + }; + + private static Logger logger = LoggerFactory.getLogger(GremlinResultParserFactory.class); +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultProcessor.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultProcessor.java new file mode 100644 index 000000000000..fe08cab9d290 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/GremlinResultProcessor.java @@ -0,0 +1,180 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.result; + +import com.alibaba.graphscope.common.client.ResultParser; +import com.alibaba.pegasus.intf.ResultProcessor; +import com.alibaba.pegasus.service.protocol.PegasusClient; + +import io.grpc.Status; +import io.netty.channel.ChannelHandlerContext; + +import org.apache.tinkerpop.gremlin.driver.MessageSerializer; +import org.apache.tinkerpop.gremlin.driver.message.RequestMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.server.Context; +import org.apache.tinkerpop.gremlin.server.handler.Frame; +import org.apache.tinkerpop.gremlin.server.handler.StateKey; +import org.apache.tinkerpop.gremlin.server.op.standard.StandardOpProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +public class GremlinResultProcessor extends StandardOpProcessor implements ResultProcessor { + private static Logger logger = LoggerFactory.getLogger(GremlinResultProcessor.class); + protected Context writeResult; + protected List resultCollectors = new ArrayList<>(); + protected boolean locked = false; + protected ResultParser resultParser; + + public GremlinResultProcessor(Context writeResult, ResultParser resultParser) { + this.writeResult = writeResult; + this.resultParser = resultParser; + } + + @Override + public void process(PegasusClient.JobResponse response) { + synchronized (this) { + try { + if (!locked) { + resultCollectors.addAll(resultParser.parseFrom(response)); + } + } catch (Exception e) { + writeResultList( + writeResult, + Collections.singletonList(e.getMessage()), + ResponseStatusCode.SERVER_ERROR); + // cannot write to this context any more + locked = true; + throw new RuntimeException(e); + } + } + } + + @Override + public void finish() { + synchronized (this) { + if (!locked) { + formatResultIfNeed(); + writeResultList(writeResult, resultCollectors, ResponseStatusCode.SUCCESS); + locked = true; + } + } + } + + // format group result as a single map + protected void formatResultIfNeed() { + if (resultParser == GremlinResultParserFactory.GROUP) { + Map groupResult = new LinkedHashMap(); + resultCollectors.forEach( + k -> { + groupResult.putAll((Map) k); + }); + resultCollectors.clear(); + resultCollectors.add(groupResult); + } + } + + @Override + public void error(Status status) { + synchronized (this) { + if (!locked) { + writeResultList( + writeResult, + Collections.singletonList(status.toString()), + ResponseStatusCode.SERVER_ERROR); + locked = true; + } + } + } + + protected void writeResultList( + final Context context, + final List resultList, + final ResponseStatusCode statusCode) { + final ChannelHandlerContext ctx = context.getChannelHandlerContext(); + final RequestMessage msg = context.getRequestMessage(); + final MessageSerializer serializer = ctx.channel().attr(StateKey.SERIALIZER).get(); + final boolean useBinary = ctx.channel().attr(StateKey.USE_BINARY).get(); + + if (statusCode == ResponseStatusCode.SERVER_ERROR) { + ResponseMessage.Builder builder = + ResponseMessage.build(msg).code(ResponseStatusCode.SERVER_ERROR); + if (resultList.size() > 0) { + builder.statusMessage((String) resultList.get(0)); + } + ctx.writeAndFlush(builder.create()); + return; + } + + boolean retryOnce = false; + while (true) { + if (ctx.channel().isWritable()) { + Frame frame = null; + try { + frame = + makeFrame( + context, + msg, + serializer, + useBinary, + resultList, + statusCode, + Collections.emptyMap(), + Collections.emptyMap()); + ctx.writeAndFlush(frame).get(); + break; + } catch (Exception e) { + if (frame != null) { + frame.tryRelease(); + } + logger.error( + "write " + + resultList.size() + + " result to context " + + context + + " status code=>" + + statusCode + + " fail", + e); + throw new RuntimeException(e); + } + } else { + if (retryOnce) { + String message = + "write result to context fail for context " + msg + " is too busy"; + logger.error(message); + throw new RuntimeException(message); + } else { + logger.warn( + "Pausing response writing as writeBufferHighWaterMark exceeded on " + + msg + + " - writing will continue once client has caught up"); + retryOnce = true; + try { + TimeUnit.MILLISECONDS.sleep(10L); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/ParserUtils.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/ParserUtils.java new file mode 100644 index 000000000000..5da391a357dc --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/ParserUtils.java @@ -0,0 +1,130 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.result; + +import com.alibaba.graphscope.gaia.proto.Common; +import com.alibaba.graphscope.gaia.proto.IrResult; +import com.alibaba.graphscope.gremlin.exception.GremlinResultParserException; + +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ParserUtils { + private static final Logger logger = LoggerFactory.getLogger(ParserUtils.class); + + public static Object parseElement(IrResult.Element element) { + switch (element.getInnerCase()) { + case VERTEX: + return parseVertex(element.getVertex()); + case EDGE: + return parseEdge(element.getEdge()); + case GRAPH_PATH: + IrResult.GraphPath graphPath = element.getGraphPath(); + return graphPath.getPathList().stream() + .map( + k -> { + if (k.getInnerCase() + == IrResult.GraphPath.VertexOrEdge.InnerCase.VERTEX) { + return parseVertex(k.getVertex()); + } else if (k.getInnerCase() + == IrResult.GraphPath.VertexOrEdge.InnerCase.EDGE) { + return parseEdge(k.getEdge()); + } else { + throw new GremlinResultParserException( + k.getInnerCase() + " is invalid"); + } + }) + .collect(Collectors.toList()); + case OBJECT: + return parseCommonValue(element.getObject()); + default: + throw new GremlinResultParserException(element.getInnerCase() + " is invalid"); + } + } + + public static List parseCollection(IrResult.Collection collection) { + return collection.getCollectionList().stream() + .map(k -> parseElement(k)) + .collect(Collectors.toList()); + } + + public static IrResult.Entry getHeadEntry(IrResult.Results results) { + return results.getRecord().getColumns(0).getEntry(); + } + + private static Object parseCommonValue(Common.Value value) { + switch (value.getItemCase()) { + case BOOLEAN: + return value.getBoolean(); + case I32: + return value.getI32(); + case I64: + return value.getI64(); + case F64: + return value.getF64(); + case STR: + return value.getStr(); + case PAIR_ARRAY: + Common.PairArray pairs = value.getPairArray(); + Map pairInMap = new HashMap(); + pairs.getItemList() + .forEach( + pair -> { + pairInMap.put( + parseCommonValue(pair.getKey()), + parseCommonValue(pair.getVal())); + }); + return pairInMap; + case STR_ARRAY: + return value.getStrArray().getItemList(); + case NONE: + return EmptyValue.INSTANCE; + default: + throw new GremlinResultParserException(value.getItemCase() + " is unsupported yet"); + } + } + + private static Vertex parseVertex(IrResult.Vertex vertex) { + Map properties = parseProperties(vertex.getPropertiesList()); + return new DetachedVertex(vertex.getId(), vertex.getLabel().getName(), properties); + } + + private static Edge parseEdge(IrResult.Edge edge) { + Map edgeProperties = parseProperties(edge.getPropertiesList()); + return new DetachedEdge( + edge.getId(), + edge.getLabel().getName(), + edgeProperties, + edge.getSrcId(), + edge.getSrcLabel().getName(), + edge.getDstId(), + edge.getDstLabel().getName()); + } + + private static Map parseProperties(List properties) { + return new HashMap<>(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java new file mode 100644 index 000000000000..907a7f4a696d --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.service; + +import com.alibaba.graphscope.common.client.HostsChannelFetcher; +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.FileLoadType; +import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher; +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.gremlin.integration.result.TestGraphFactory; + +public class GraphServiceMain { + public static void main(String[] args) throws Exception { + Configs configs = new Configs("conf/ir.compiler.properties", FileLoadType.RELATIVE_PATH); + IrMetaFetcher irMetaFetcher = new ExperimentalMetaFetcher(configs); + RpcChannelFetcher fetcher = new HostsChannelFetcher(configs); + + IrGremlinServer server = new IrGremlinServer(); + server.start(configs, irMetaFetcher, fetcher, TestGraphFactory.EXPERIMENTAL); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java new file mode 100644 index 000000000000..13cd20afb393 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java @@ -0,0 +1,55 @@ +package com.alibaba.graphscope.gremlin.service; + +import com.alibaba.graphscope.common.client.RpcChannelFetcher; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.gremlin.integration.processor.IrTestOpProcessor; +import com.alibaba.graphscope.gremlin.integration.result.GraphProperties; +import com.alibaba.graphscope.gremlin.plugin.processor.IrOpLoader; +import com.alibaba.graphscope.gremlin.plugin.processor.IrStandardOpProcessor; + +import org.apache.tinkerpop.gremlin.server.GremlinServer; +import org.apache.tinkerpop.gremlin.server.Settings; +import org.apache.tinkerpop.gremlin.server.op.AbstractOpProcessor; + +import java.io.InputStream; + +public class IrGremlinServer implements AutoCloseable { + private GremlinServer gremlinServer; + private Settings settings; + + public IrGremlinServer() { + InputStream input = + getClass().getClassLoader().getResourceAsStream("conf/gremlin-server.yaml"); + settings = Settings.read(input); + } + + public IrGremlinServer(int gremlinPort) { + this(); + settings.port = (gremlinPort >= 0) ? gremlinPort : settings.port; + } + + public void start( + Configs configs, + IrMetaFetcher irMetaFetcher, + RpcChannelFetcher fetcher, + GraphProperties testGraph) + throws Exception { + AbstractOpProcessor standardProcessor = + new IrStandardOpProcessor(configs, irMetaFetcher, fetcher); + IrOpLoader.addProcessor(standardProcessor.getName(), standardProcessor); + AbstractOpProcessor testProcessor = + new IrTestOpProcessor(configs, irMetaFetcher, fetcher, testGraph); + IrOpLoader.addProcessor(testProcessor.getName(), testProcessor); + + this.gremlinServer = new GremlinServer(settings); + this.gremlinServer.start(); + } + + @Override + public void close() throws Exception { + if (this.gremlinServer != null) { + this.gremlinServer.stop(); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrOpSelectorHandler.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrOpSelectorHandler.java new file mode 100644 index 000000000000..81d4e4a38476 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrOpSelectorHandler.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.service; + +import com.alibaba.graphscope.gremlin.plugin.processor.IrOpLoader; + +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; + +import org.apache.tinkerpop.gremlin.driver.message.RequestMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; +import org.apache.tinkerpop.gremlin.server.*; +import org.apache.tinkerpop.gremlin.server.handler.OpSelectorHandler; +import org.apache.tinkerpop.gremlin.server.op.OpProcessorException; +import org.javatuples.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; + +@ChannelHandler.Sharable +public class IrOpSelectorHandler extends OpSelectorHandler { + private static final Logger logger = LoggerFactory.getLogger(IrOpSelectorHandler.class); + + private final Settings settings; + private final GraphManager graphManager; + private final GremlinExecutor gremlinExecutor; + private final ScheduledExecutorService scheduledExecutorService; + + public IrOpSelectorHandler( + final Settings settings, + final GraphManager graphManager, + final GremlinExecutor gremlinExecutor, + final ScheduledExecutorService scheduledExecutorService, + final Channelizer channelizer) { + super(settings, graphManager, gremlinExecutor, scheduledExecutorService, channelizer); + this.settings = settings; + this.graphManager = graphManager; + this.gremlinExecutor = gremlinExecutor; + this.scheduledExecutorService = scheduledExecutorService; + } + + @Override + protected void decode(ChannelHandlerContext ctx, RequestMessage msg, List objects) { + Context gremlinServerContext = + new Context( + msg, + ctx, + this.settings, + this.graphManager, + this.gremlinExecutor, + this.scheduledExecutorService); + try { + Optional processor = IrOpLoader.getProcessor(msg.getProcessor()); + if (!processor.isPresent()) { + String errorMessage = + String.format("Invalid OpProcessor requested [%s]", msg.getProcessor()); + throw new OpProcessorException( + errorMessage, + ResponseMessage.build(msg) + .code(ResponseStatusCode.REQUEST_ERROR_INVALID_REQUEST_ARGUMENTS) + .statusMessage(errorMessage) + .create()); + } + + objects.add(Pair.with(msg, (processor.get()).select(gremlinServerContext))); + } catch (OpProcessorException var7) { + logger.error(var7.getMessage(), var7); + ctx.writeAndFlush(var7.getResponseMessage()); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrWsAndHttpChannelizer.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrWsAndHttpChannelizer.java new file mode 100644 index 000000000000..479587b5bfbc --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrWsAndHttpChannelizer.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.service; + +import com.alibaba.graphscope.gremlin.Utils; + +import org.apache.tinkerpop.gremlin.server.AbstractChannelizer; +import org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer; +import org.apache.tinkerpop.gremlin.server.handler.HttpGremlinEndpointHandler; +import org.apache.tinkerpop.gremlin.server.handler.OpSelectorHandler; +import org.apache.tinkerpop.gremlin.server.handler.WsAndHttpChannelizerHandler; +import org.apache.tinkerpop.gremlin.server.util.ServerGremlinExecutor; + +// config the channelizer in conf/gremlin-server.yaml to set the IrOpSelectorHandler as the default +public class IrWsAndHttpChannelizer extends WsAndHttpChannelizer { + private WsAndHttpChannelizerHandler handler; + + @Override + public void init(ServerGremlinExecutor serverGremlinExecutor) { + super.init(serverGremlinExecutor); + this.handler = new WsAndHttpChannelizerHandler(); + this.handler.init( + serverGremlinExecutor, + new HttpGremlinEndpointHandler( + this.serializers, this.gremlinExecutor, this.graphManager, this.settings)); + OpSelectorHandler irOpSelectorHandler = + new IrOpSelectorHandler( + this.settings, + this.graphManager, + this.gremlinExecutor, + this.scheduledExecutorService, + this); + Utils.setFieldValue( + AbstractChannelizer.class, this, "opSelectorHandler", irOpSelectorHandler); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/ExprArg.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/ExprArg.java new file mode 100644 index 000000000000..984d79e80d47 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/ExprArg.java @@ -0,0 +1,85 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.lambda.IdentityTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.lambda.ValueTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public class ExprArg { + // store all steps of the traversal given by arg + private List stepsInTraversal; + // judge whether the traversal is instance of IdentityTraversal, i.e. by() + private boolean isIdentityTraversal; + // judge whether the traversal is instance of ValueTraversal, i.e. by('name'), + // a property name is present if it is + private Optional propertyKeyOpt; + + public ExprArg() { + isIdentityTraversal = false; + propertyKeyOpt = Optional.empty(); + stepsInTraversal = new ArrayList<>(); + } + + public ExprArg(Traversal.Admin traversal) { + this(); + if (traversal == null || traversal instanceof IdentityTraversal) { + isIdentityTraversal = true; + } else if (traversal instanceof ValueTraversal) { + propertyKeyOpt = Optional.of(((ValueTraversal) traversal).getPropertyKey()); + } else { + traversal + .asAdmin() + .getSteps() + .forEach( + k -> { + stepsInTraversal.add((Step) k); + }); + } + } + + public boolean isIdentityTraversal() { + return isIdentityTraversal; + } + + public int size() { + return stepsInTraversal.size(); + } + + public Step getStartStep() { + return stepsInTraversal.isEmpty() ? EmptyStep.instance() : stepsInTraversal.get(0); + } + + public Step getEndStep() { + return stepsInTraversal.isEmpty() ? EmptyStep.instance() : stepsInTraversal.get(size() - 1); + } + + public Optional getPropertyKeyOpt() { + return propertyKeyOpt; + } + + public ExprArg addStep(Step step) { + this.stepsInTraversal.add(step); + return this; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/ExprResult.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/ExprResult.java new file mode 100644 index 000000000000..4fe57a9974c8 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/ExprResult.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import java.util.*; + +public class ExprResult { + // if all of the by_traversals can be converted to expressions, return true + // otherwise false + private boolean isExprPattern; + // store the tag and the corresponding by_traversal as expression if it can be converted + // especially, tag is "" if no tag exists, i.e. values(..), valueMap(..) + private Map tagExprMap; + + public ExprResult() { + this.isExprPattern = false; + this.tagExprMap = new LinkedHashMap<>(); + } + + public ExprResult(boolean isExprPattern) { + this.tagExprMap = new LinkedHashMap<>(); + this.isExprPattern = isExprPattern; + } + + public List getExprs() { + return new ArrayList<>(tagExprMap.values()); + } + + public Optional getSingleExpr() { + List exprs = getExprs(); + return exprs.isEmpty() ? Optional.empty() : Optional.of(exprs.get(0)); + } + + public Optional getTagExpr(String tag) { + String expr = tagExprMap.get(tag); + return (expr == null || expr.isEmpty()) ? Optional.empty() : Optional.of(expr); + } + + public boolean isExprPattern() { + return this.isExprPattern; + } + + public ExprResult addTagExpr(String tag, String expr) { + tagExprMap.put(tag, expr); + return this; + } + + public ExprResult setExprPattern(boolean exprPattern) { + isExprPattern = exprPattern; + return this; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransform.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransform.java new file mode 100644 index 000000000000..e05330689bdb --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransform.java @@ -0,0 +1,143 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import com.alibaba.graphscope.common.exception.OpArgIllegalException; +import com.alibaba.graphscope.common.jna.type.FfiVariable; +import com.alibaba.graphscope.gremlin.antlr4.AnyValue; + +import org.apache.tinkerpop.gremlin.process.traversal.Compare; +import org.apache.tinkerpop.gremlin.process.traversal.Contains; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.util.AndP; +import org.apache.tinkerpop.gremlin.process.traversal.util.ConnectiveP; + +import java.util.List; +import java.util.function.BiPredicate; +import java.util.function.Function; + +// transform predicate into expression +public interface PredicateExprTransform extends Function { + default String flatPredicate(String subject, P predicate) { + String expr = ""; + if (predicate instanceof ConnectiveP) { + ConnectiveP connectiveP = (ConnectiveP) predicate; + String connector = (connectiveP instanceof AndP) ? " && " : " || "; + List

predicates = connectiveP.getPredicates(); + for (int i = 0; i < predicates.size(); ++i) { + P cur = predicates.get(i); + if (cur instanceof ConnectiveP) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "composition of and & or is unsupported"); + } + if (i > 0) { + expr += connector; + } + String flatPredicate = flatPredicate(subject, cur); + if (i > 0) { + expr += "(" + flatPredicate + ")"; + } else { + expr += flatPredicate; + } + } + } else { + Object predicateValue = predicate.getValue(); + if (predicateValue instanceof AnyValue) { + expr = getExprIfPropertyExist(subject); + } else { + BiPredicate biPredicate = predicate.getBiPredicate(); + if (biPredicate == Compare.eq) { + expr += getPredicateExpr(subject, "==", predicateValue); + } else if (biPredicate == Compare.neq) { + expr += getPredicateExpr(subject, "!=", predicateValue); + } else if (biPredicate == Compare.lt) { + expr += getPredicateExpr(subject, "<", predicateValue); + } else if (biPredicate == Compare.lte) { + expr += getPredicateExpr(subject, "<=", predicateValue); + } else if (biPredicate == Compare.gt) { + expr += getPredicateExpr(subject, ">", predicateValue); + } else if (biPredicate == Compare.gte) { + expr += getPredicateExpr(subject, ">=", predicateValue); + } else if (biPredicate == Contains.within) { + expr += getPredicateExpr(subject, "within", predicateValue); + } else if (biPredicate == Contains.without) { + expr += getPredicateExpr(subject, "without", predicateValue); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "predicate type is unsupported"); + } + } + } + return expr; + } + + default String getPredicateValue(Object value) { + String predicateValue; + if (value instanceof String) { + predicateValue = String.format("\"%s\"", value); + } else if (value instanceof List) { + String content = ""; + List values = (List) value; + for (int i = 0; i < values.size(); ++i) { + if (i != 0) { + content += ", "; + } + Object v = values.get(i); + if (v instanceof List) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "nested list of predicate value is unsupported"); + } + content += getPredicateValue(v); + } + predicateValue = String.format("[%s]", content); + } else { + predicateValue = value.toString(); + } + return predicateValue; + } + + // @a -> "" + // @a.name -> @a.name + // @.name -> @.name + // @ -> "" + default String getExprIfPropertyExist(String expr) { + String[] splitExprs = expr.split("\\."); + return (splitExprs.length == 2) ? expr : ""; + } + + default String getPredicateExpr(String subject, String predicate, Object value) { + String subjectKeyExist = getExprIfPropertyExist(subject); + String valueKeyExist = ""; + if (value instanceof FfiVariable.ByValue) { + valueKeyExist = getExprIfPropertyExist(value.toString()); + } + String predicateExpr = + String.format("%s %s %s", subject, predicate, getPredicateValue(value)); + StringBuilder builder = new StringBuilder(predicateExpr); + if (!valueKeyExist.isEmpty()) { + builder.insert(0, String.format("%s && ", valueKeyExist)); + } + if (!subjectKeyExist.isEmpty()) { + builder.insert(0, String.format("%s && ", subjectKeyExist)); + } + return builder.toString(); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransformFactory.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransformFactory.java new file mode 100644 index 000000000000..5880e85817da --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/PredicateExprTransformFactory.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; + +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTraversalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; + +import java.util.List; + +public enum PredicateExprTransformFactory implements PredicateExprTransform { + HAS_STEP { + @Override + public String apply(Step arg) { + HasContainerHolder hasStep = (HasContainerHolder) arg; + List containers = hasStep.getHasContainers(); + String expr = ""; + for (int i = 0; i < containers.size(); ++i) { + if (i > 0) { + expr += " && "; + } + HasContainer container = containers.get(i); + String key = "@." + container.getKey(); + String flatPredicate = flatPredicate(key, container.getPredicate()); + if (i > 0) { + expr += "(" + flatPredicate + ")"; + } else { + expr += flatPredicate; + } + } + return expr; + } + }, + IS_STEP { + @Override + public String apply(Step arg) { + IsStep isStep = (IsStep) arg; + // current value + String key = "@"; + return flatPredicate(key, isStep.getPredicate()); + } + }, + WHERE_END_STEP { + @Override + public String apply(Step arg) { + WhereTraversalStep.WhereEndStep endStep = (WhereTraversalStep.WhereEndStep) arg; + String matchTag = endStep.getScopeKeys().iterator().next(); + P predicate = P.eq(ArgUtils.asFfiVar(matchTag, "")); + return flatPredicate("@", predicate); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/StepTransformFactory.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/StepTransformFactory.java new file mode 100644 index 000000000000..f060932919ee --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/StepTransformFactory.java @@ -0,0 +1,608 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import com.alibaba.graphscope.common.exception.OpArgIllegalException; +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.MatchSentence; +import com.alibaba.graphscope.common.intermediate.operator.*; +import com.alibaba.graphscope.common.jna.type.*; +import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; +import com.alibaba.graphscope.gremlin.antlr4.GremlinAntlrToJava; +import com.alibaba.graphscope.gremlin.plugin.step.ExprStep; +import com.alibaba.graphscope.gremlin.plugin.step.PathExpandStep; +import com.alibaba.graphscope.gremlin.plugin.step.ScanFusionStep; +import com.alibaba.graphscope.gremlin.transform.alias.AliasArg; +import com.alibaba.graphscope.gremlin.transform.alias.AliasManager; +import com.alibaba.graphscope.gremlin.transform.alias.AliasPrefixType; + +import org.apache.tinkerpop.gremlin.process.traversal.Pop; +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.lambda.ColumnTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; +import org.apache.tinkerpop.gremlin.structure.Column; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.javatuples.Pair; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +public enum StepTransformFactory implements Function { + GRAPH_STEP { + @Override + public InterOpBase apply(Step step) { + GraphStep graphStep = (GraphStep) step; + ScanFusionOp op = new ScanFusionOp(); + op.setScanOpt(new OpArg<>(graphStep, SCAN_OPT)); + + if (graphStep.getIds().length > 0) { + op.setIds(new OpArg<>(graphStep, CONST_IDS_FROM_STEP)); + } + return op; + } + }, + SCAN_FUSION_STEP { + @Override + public InterOpBase apply(Step step) { + ScanFusionStep scanFusionStep = (ScanFusionStep) step; + ScanFusionOp op = new ScanFusionOp(); + + op.setScanOpt(new OpArg<>(scanFusionStep, SCAN_OPT)); + + // set global ids + if (scanFusionStep.getIds() != null && scanFusionStep.getIds().length > 0) { + op.setIds(new OpArg(scanFusionStep, CONST_IDS_FROM_STEP)); + } + // set labels + if (!scanFusionStep.getGraphLabels().isEmpty()) { + List labels = scanFusionStep.getGraphLabels(); + op.setLabels(new OpArg(scanFusionStep, LABELS_FROM_STEP)); + } + // set other containers as predicates + if (!scanFusionStep.getHasContainers().isEmpty()) { + op.setPredicate(new OpArg(step, PredicateExprTransformFactory.HAS_STEP)); + } + return op; + } + }, + HAS_STEP { + @Override + public InterOpBase apply(Step step) { + SelectOp op = new SelectOp(); + List containers = ((HasStep) step).getHasContainers(); + // add corner judgement + if (!containers.isEmpty()) { + op.setPredicate(new OpArg(step, PredicateExprTransformFactory.HAS_STEP)); + } + return op; + } + }, + IS_STEP { + @Override + public InterOpBase apply(Step step) { + SelectOp op = new SelectOp(); + op.setPredicate(new OpArg(step, PredicateExprTransformFactory.IS_STEP)); + return op; + } + }, + VERTEX_STEP { + @Override + public InterOpBase apply(Step step) { + ExpandOp op = new ExpandOp(); + op.setDirection( + new OpArg<>( + (VertexStep) step, + (VertexStep s1) -> { + Direction direction = s1.getDirection(); + switch (direction) { + case IN: + return FfiDirection.In; + case BOTH: + return FfiDirection.Both; + case OUT: + return FfiDirection.Out; + default: + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "invalid direction type"); + } + })); + op.setEdgeOpt( + new OpArg<>( + (VertexStep) step, + (VertexStep s1) -> { + if (s1.returnsEdge()) { + return Boolean.valueOf(true); + } else { + return Boolean.valueOf(false); + } + })); + // add corner judgement + if (((VertexStep) step).getEdgeLabels().length > 0) { + op.setLabels( + new OpArg<>( + (VertexStep) step, + (VertexStep s1) -> + Arrays.stream(s1.getEdgeLabels()) + .map(k -> ArgUtils.asFfiTag(k)) + .collect(Collectors.toList()))); + } + return op; + } + }, + LIMIT_STEP { + @Override + public InterOpBase apply(Step step) { + RangeGlobalStep limitStep = (RangeGlobalStep) step; + int lower = (int) limitStep.getLowRange() + 1; + int upper = (int) limitStep.getHighRange() + 1; + LimitOp op = new LimitOp(); + op.setLower(new OpArg(Integer.valueOf(lower))); + op.setUpper(new OpArg(Integer.valueOf(upper))); + return op; + } + }, + VALUE_MAP_STEP { + @Override + public InterOpBase apply(Step step) { + PropertyMapStep valueMapStep = (PropertyMapStep) step; + ProjectOp op = new ProjectOp(); + String expr = + TraversalParentTransformFactory.PROJECT_BY_STEP + .getSubTraversalAsExpr((new ExprArg()).addStep(valueMapStep)) + .getSingleExpr() + .get(); + op.setExprWithAlias( + new OpArg<>( + expr, + (String expr1) -> { + FfiAlias.ByValue alias = ArgUtils.asFfiNoneAlias(); + return Arrays.asList(Pair.with(expr1, alias)); + })); + return op; + } + }, + VALUES_STEP { + @Override + public InterOpBase apply(Step step) { + PropertiesStep valuesStep = (PropertiesStep) step; + ProjectOp op = new ProjectOp(); + String expr = + TraversalParentTransformFactory.PROJECT_BY_STEP + .getSubTraversalAsExpr((new ExprArg()).addStep(valuesStep)) + .getSingleExpr() + .get(); + op.setExprWithAlias( + new OpArg<>( + expr, + (String expr1) -> { + FfiAlias.ByValue alias = ArgUtils.asFfiNoneAlias(); + return Arrays.asList(Pair.with(expr1, alias)); + })); + return op; + } + }, + DEDUP_STEP { + @Override + public InterOpBase apply(Step step) { + DedupGlobalStep dedupStep = (DedupGlobalStep) step; + Map tagTraversals = getDedupTagTraversal(dedupStep); + DedupOp op = new DedupOp(); + op.setDedupKeys( + new OpArg<>( + tagTraversals, + (Map map) -> { + if (tagTraversals.isEmpty()) { // only support dedup() + return Collections.singletonList(ArgUtils.asFfiNoneVar()); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "supported pattern is [dedup()]"); + } + })); + return op; + } + + // dedup("a").by("name"): a -> "name" + private Map getDedupTagTraversal(DedupGlobalStep step) { + Set dedupTags = step.getScopeKeys(); + List dedupTraversals = step.getLocalChildren(); + Map tagTraversals = new HashMap<>(); + if (dedupTags.isEmpty() && dedupTraversals.isEmpty()) { + return tagTraversals; + } + if (dedupTags.isEmpty()) { + dedupTags = new HashSet<>(); + // set as head + dedupTags.add(""); + } + dedupTags.forEach( + k -> { + Traversal.Admin dedupTraversal = + dedupTraversals.isEmpty() ? null : dedupTraversals.get(0); + tagTraversals.put(k, dedupTraversal); + }); + tagTraversals.entrySet().removeIf(e -> e.getKey().equals("") && e.getValue() == null); + return tagTraversals; + } + }, + COUNT_STEP { + @Override + public InterOpBase apply(Step step) { + CountGlobalStep countStep = (CountGlobalStep) step; + GroupOp op = new GroupOp(); + op.setGroupByKeys(new OpArg(Collections.emptyList())); + op.setGroupByValues(new OpArg(getCountAgg(countStep))); + return op; + } + + private List getCountAgg(CountGlobalStep step1) { + int stepIdx = TraversalHelper.stepIndex(step1, step1.getTraversal()); + FfiAlias.ByValue valueAlias = + AliasManager.getFfiAlias(new AliasArg(AliasPrefixType.GROUP_VALUES, stepIdx)); + // count().as("a"), "a" is the alias of group value + if (!step1.getLabels().isEmpty()) { + String label = (String) step1.getLabels().iterator().next(); + valueAlias = ArgUtils.asFfiAlias(label, true); + step1.removeLabel(label); + } + ArgAggFn countAgg = new ArgAggFn(FfiAggOpt.Count, valueAlias); + return Collections.singletonList(countAgg); + } + }, + PATH_EXPAND_STEP { + @Override + public InterOpBase apply(Step step) { + PathExpandOp op = new PathExpandOp((ExpandOp) VERTEX_STEP.apply(step)); + PathExpandStep pathStep = (PathExpandStep) step; + op.setLower(new OpArg(Integer.valueOf(pathStep.getLower()))); + op.setUpper(new OpArg(Integer.valueOf(pathStep.getUpper()))); + return op; + } + }, + EDGE_VERTEX_STEP { + @Override + public InterOpBase apply(Step step) { + EdgeVertexStep vertexStep = (EdgeVertexStep) step; + GetVOp op = new GetVOp(); + op.setGetVOpt( + new OpArg<>( + vertexStep, + (EdgeVertexStep edgeVertexStep) -> { + Direction direction = edgeVertexStep.getDirection(); + switch (direction) { + case OUT: + return FfiVOpt.Start; + case IN: + return FfiVOpt.End; + case BOTH: + default: + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + direction + " cannot be converted to FfiVOpt"); + } + })); + return op; + } + }, + EDGE_OTHER_STEP { + @Override + public InterOpBase apply(Step step) { + EdgeOtherVertexStep otherStep = (EdgeOtherVertexStep) step; + GetVOp op = new GetVOp(); + op.setGetVOpt( + new OpArg<>(otherStep, (EdgeOtherVertexStep otherStep1) -> FfiVOpt.Other)); + return op; + } + }, + WHERE_START_STEP { + @Override + public InterOpBase apply(Step step) { + WhereTraversalStep.WhereStartStep startStep = (WhereTraversalStep.WhereStartStep) step; + String selectKey = (String) startStep.getScopeKeys().iterator().next(); + + ProjectOp op = new ProjectOp(); + op.setExprWithAlias( + new OpArg<>( + selectKey, + (String key) -> { + String expr = "@" + selectKey; + FfiAlias.ByValue alias = ArgUtils.asFfiNoneAlias(); + return Collections.singletonList(Pair.with(expr, alias)); + })); + return op; + } + }, + WHERE_END_STEP { + @Override + public InterOpBase apply(Step step) { + WhereTraversalStep.WhereEndStep endStep = (WhereTraversalStep.WhereEndStep) step; + SelectOp selectOp = new SelectOp(); + selectOp.setPredicate(new OpArg(endStep, PredicateExprTransformFactory.WHERE_END_STEP)); + return selectOp; + } + }, + UNION_STEP { + @Override + public InterOpBase apply(Step step) { + UnionOp unionOp = new UnionOp(); + unionOp.setSubOpCollectionList( + new OpArg<>( + (UnionStep) step, + (UnionStep unionStep) -> { + List subTraversals = unionStep.getGlobalChildren(); + return subTraversals.stream() + .filter(k -> k != null) + .map(k -> (new InterOpCollectionBuilder(k)).build()) + .collect(Collectors.toList()); + })); + return unionOp; + } + }, + TRAVERSAL_MAP_STEP { + @Override + public InterOpBase apply(Step step) { + TraversalMapStep mapStep = (TraversalMapStep) step; + List mapTraversals = mapStep.getLocalChildren(); + if (mapTraversals.size() != 1 || !(mapTraversals.get(0) instanceof ColumnTraversal)) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "only support select(keys/values)"); + } + Column column = ((ColumnTraversal) mapTraversals.get(0)).getColumn(); + switch (column) { + case keys: + String key = getMapKey(mapStep); + SelectOneStep keySelect = new SelectOneStep(step.getTraversal(), Pop.last, key); + TraversalHelper.copyLabels(mapStep, keySelect, false); + return TraversalParentTransformFactory.PROJECT_BY_STEP.apply(keySelect).get(0); + case values: + String value = getMapValue(mapStep); + SelectOneStep valueSelect = + new SelectOneStep(step.getTraversal(), Pop.last, value); + TraversalHelper.copyLabels(mapStep, valueSelect, false); + return TraversalParentTransformFactory.PROJECT_BY_STEP + .apply(valueSelect) + .get(0); + default: + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + column.name() + " is invalid"); + } + } + + private String getMapKey(TraversalMapStep step) { + Step groupStep = getPreviousGroupStep(step); + int stepIdx = TraversalHelper.stepIndex(groupStep, groupStep.getTraversal()); + FfiAlias.ByValue keyAlias = + AliasManager.getFfiAlias(new AliasArg(AliasPrefixType.GROUP_KEYS, stepIdx)); + return keyAlias.alias.name; + } + + private String getMapValue(TraversalMapStep step) { + Step groupStep = getPreviousGroupStep(step); + int stepIdx = TraversalHelper.stepIndex(groupStep, groupStep.getTraversal()); + FfiAlias.ByValue valueAlias = + AliasManager.getFfiAlias(new AliasArg(AliasPrefixType.GROUP_VALUES, stepIdx)); + return valueAlias.alias.name; + } + + private Step getPreviousGroupStep(Step step) { + Step previous = step.getPreviousStep(); + while (!(previous instanceof EmptyStep + || previous instanceof GroupStep + || previous instanceof GroupCountStep)) { + previous = previous.getPreviousStep(); + } + if (!(previous instanceof EmptyStep)) { + return previous; + } + TraversalParent parent = step.getTraversal().getParent(); + if (!(parent instanceof EmptyStep)) { + return getPreviousGroupStep(parent.asStep()); + } + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "select keys or values should follow a group"); + } + }, + MATCH_STEP { + @Override + public InterOpBase apply(Step step) { + MatchStep matchStep = (MatchStep) step; + List sentences = getSentences(matchStep); + MatchOp matchOp = new MatchOp(); + matchOp.setSentences(new OpArg(sentences)); + return matchOp; + } + + private List getSentences(MatchStep matchStep) { + List matchTraversals = matchStep.getGlobalChildren(); + List sentences = new ArrayList<>(); + matchTraversals.forEach( + traversal -> { + List binderSteps = new ArrayList<>(); + Optional startTag = Optional.empty(); + Optional endTag = Optional.empty(); + FfiJoinKind joinKind = FfiJoinKind.Inner; + for (Object o : traversal.getSteps()) { + Step s = (Step) o; + if (s instanceof MatchStep.MatchStartStep) { // match(__.as("a")...) + Optional selectKey = + ((MatchStep.MatchStartStep) s).getSelectKey(); + if (!startTag.isPresent() && selectKey.isPresent()) { + startTag = selectKey; + } + } else if (s instanceof MatchStep.MatchEndStep) { // match(...as("b")) + Optional matchKey = + ((MatchStep.MatchEndStep) s).getMatchKey(); + if (!endTag.isPresent() && matchKey.isPresent()) { + endTag = matchKey; + } + } else if (s instanceof WhereTraversalStep + && binderSteps.isEmpty()) { // where(__.as("a")...) or not(...) + List children = + ((WhereTraversalStep) s).getLocalChildren(); + Traversal.Admin whereTraversal = + children.isEmpty() ? null : children.get(0); + // not(as("a").out().as("b")) + if (whereTraversal != null + && whereTraversal.getSteps().size() == 1 + && whereTraversal.getStartStep() instanceof NotStep) { + NotStep notStep = (NotStep) whereTraversal.getStartStep(); + List notChildren = notStep.getLocalChildren(); + whereTraversal = + (notChildren.isEmpty()) ? null : notChildren.get(0); + joinKind = FfiJoinKind.Anti; + } else { // where(as("a").out().as("b")) + joinKind = FfiJoinKind.Semi; + } + if (whereTraversal != null) { + for (Object o1 : whereTraversal.getSteps()) { + Step s1 = (Step) o1; + if (s1 + instanceof + WhereTraversalStep + .WhereStartStep) { // not(__.as("a")...) + Set scopeKeys; + if (!startTag.isPresent() + && !(scopeKeys = + ((WhereTraversalStep + .WhereStartStep) + s1) + .getScopeKeys()) + .isEmpty()) { + startTag = Optional.of(scopeKeys.iterator().next()); + } + } else if (s1 + instanceof + WhereTraversalStep + .WhereEndStep) { // not(....as("b")) + Set scopeKeys; + if (!endTag.isPresent() + && !(scopeKeys = + ((WhereTraversalStep + .WhereEndStep) + s1) + .getScopeKeys()) + .isEmpty()) { + endTag = Optional.of(scopeKeys.iterator().next()); + } + } else if (isValidBinderStep(s1)) { + binderSteps.add(s1); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + s1.getClass() + " is unsupported yet in match"); + } + } + } + } else if (isValidBinderStep(s)) { + binderSteps.add(s); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + s.getClass() + " is unsupported yet in match"); + } + } + if (!startTag.isPresent() || !endTag.isPresent()) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "startTag or endTag not exist in match"); + } + Traversal binderTraversal = GremlinAntlrToJava.getTraversalSupplier().get(); + binderSteps.forEach( + b -> { + binderTraversal.asAdmin().addStep(b); + }); + InterOpCollection ops = + (new InterOpCollectionBuilder(binderTraversal)).build(); + sentences.add( + new MatchSentence(startTag.get(), endTag.get(), ops, joinKind)); + }); + return sentences; + } + + private boolean isValidBinderStep(Step step) { + return step instanceof VertexStep // in()/out()/both()/inE()/outE()/bothE() + || step instanceof PathExpandStep // out/in/both('1..5', 'knows') + || step instanceof EdgeOtherVertexStep // otherV() + || step instanceof EdgeVertexStep; // inV()/outV()/endV()(todo) + } + }, + + EXPR_STEP { + @Override + public InterOpBase apply(Step step) { + ExprStep exprStep = (ExprStep) step; + switch (exprStep.getType()) { + case PROJECTION: + ProjectOp projectOp = new ProjectOp(); + projectOp.setExprWithAlias( + new OpArg<>( + exprStep.getExpr(), + (String expr) -> + Arrays.asList( + Pair.with(expr, ArgUtils.asFfiNoneAlias())))); + return projectOp; + case FILTER: + default: + SelectOp selectOp = new SelectOp(); + selectOp.setPredicate(new OpArg(exprStep.getExpr())); + return selectOp; + } + } + }; + + protected Function SCAN_OPT = + (GraphStep s1) -> { + if (s1.returnsVertex()) return FfiScanOpt.Entity; + else return FfiScanOpt.Relation; + }; + + protected Function> CONST_IDS_FROM_STEP = + (GraphStep s1) -> + Arrays.stream(s1.getIds()) + .map( + (id) -> { + if (id instanceof Integer) { + return ArgUtils.asFfiConst((Integer) id); + } else if (id instanceof Long) { + return ArgUtils.asFfiConst((Long) id); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "unimplemented yet"); + } + }) + .collect(Collectors.toList()); + + protected Function> LABELS_FROM_STEP = + (ScanFusionStep step) -> { + List labels = step.getGraphLabels(); + return labels.stream().map(k -> ArgUtils.asFfiTag(k)).collect(Collectors.toList()); + }; +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransform.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransform.java new file mode 100644 index 000000000000..33739a075e66 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransform.java @@ -0,0 +1,199 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import com.alibaba.graphscope.common.exception.OpArgIllegalException; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.ProjectOp; +import com.alibaba.graphscope.common.jna.type.FfiVariable; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTraversalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; +import org.javatuples.Pair; + +import java.util.*; +import java.util.function.Function; + +/** + * judge whether the sub-traversal is the real apply. Especially, judge select('a', 'b').by(..) as a whole, + * thus group().by(select('a', 'b').by(..)) can get multiple expressions as different group key or consider it as a apply + **/ +public interface TraversalParentTransform extends Function> { + default ExprResult getSubTraversalAsExpr(ExprArg exprArg) { + int size = exprArg.size(); + // the followings are considered as expressions instead of apply + if (size <= 1) { + if (exprArg.isIdentityTraversal()) { // by() + return (new ExprResult(true)).addTagExpr("", "@"); + } else if (exprArg.getPropertyKeyOpt().isPresent()) { // by('name') + String property = exprArg.getPropertyKeyOpt().get(); + return (new ExprResult(true)).addTagExpr("", "@." + property); + } else { + Step step = exprArg.getStartStep(); + if (step instanceof PropertyMapStep) { // valueMap(..) + String[] mapKeys = ((PropertyMapStep) step).getPropertyKeys(); + if (mapKeys.length > 0) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("{"); + for (int i = 0; i < mapKeys.length; ++i) { + if (i > 0) { + stringBuilder.append(", "); + } + stringBuilder.append("@." + mapKeys[i]); + } + stringBuilder.append("}"); + return (new ExprResult(true)).addTagExpr("", stringBuilder.toString()); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "valueMap() is unsupported"); + } + } else if (step instanceof PropertiesStep) { // values(..) + String[] mapKeys = ((PropertiesStep) step).getPropertyKeys(); + if (mapKeys.length == 0) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "values() is unsupported"); + } + if (mapKeys.length > 1) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, + "use valueMap(..) instead if there are multiple project keys"); + } + return (new ExprResult(true)).addTagExpr("", "@." + mapKeys[0]); + } else if (step instanceof SelectOneStep || step instanceof SelectStep) { + // select('a'), select('a').by() + // select('a').by('name'/values/valueMap) + // select('a', 'b'), select('a', 'b').by() + // select('a', 'b').by('name'/values/valueMap).by('name'/values/valueMap) + Map selectBys = + getProjectTraversals((TraversalParent) step); + ExprResult exprRes = new ExprResult(); + boolean isExprPattern = true; + for (Map.Entry entry : selectBys.entrySet()) { + String k = entry.getKey(); + Traversal.Admin v = entry.getValue(); + Optional byExpr = + getSubTraversalAsExpr(new ExprArg(v)).getSingleExpr(); + if (byExpr.isPresent()) { + String expr = byExpr.get().replace("@", "@" + k); + exprRes.addTagExpr(k, expr); + } else { + isExprPattern = false; + } + } + return exprRes.setExprPattern(isExprPattern); + } else if (step instanceof WhereTraversalStep.WhereStartStep) { // where(as('a')) + WhereTraversalStep.WhereStartStep startStep = + (WhereTraversalStep.WhereStartStep) step; + String selectKey = (String) startStep.getScopeKeys().iterator().next(); + return (new ExprResult(true)).addTagExpr(selectKey, "@" + selectKey); + } else if (step instanceof TraversalMapStep) { // select(keys), select(values) + ProjectOp mapOp = + (ProjectOp) StepTransformFactory.TRAVERSAL_MAP_STEP.apply(step); + List pairs = (List) mapOp.getExprWithAlias().get().applyArg(); + String mapExpr = (String) pairs.get(0).getValue0(); + String mapKey = mapExpr.substring(1); + return (new ExprResult(true)).addTagExpr(mapKey, mapExpr); + } else { + return new ExprResult(false); + } + } + } else if (size == 2) { + Step startStep = exprArg.getStartStep(); + Step endStep = exprArg.getEndStep(); + if ((startStep instanceof SelectOneStep || startStep instanceof TraversalMapStep) + && (endStep instanceof PropertiesStep || endStep instanceof PropertyMapStep)) { + Optional propertyExpr = + getSubTraversalAsExpr((new ExprArg()).addStep(endStep)).getSingleExpr(); + if (!propertyExpr.isPresent()) { + return new ExprResult(false); + } + String selectKey = null; + if (startStep + instanceof + SelectOneStep) { // select('a').values(..), select('a').valueMap(..) + selectKey = + (String) ((SelectOneStep) startStep).getScopeKeys().iterator().next(); + } else if (startStep + instanceof + TraversalMapStep) { // select(keys).values(..), select(values).valueMap(..) + ProjectOp mapOp = + (ProjectOp) StepTransformFactory.TRAVERSAL_MAP_STEP.apply(startStep); + List pairs = (List) mapOp.getExprWithAlias().get().applyArg(); + String mapExpr = (String) pairs.get(0).getValue0(); + selectKey = mapExpr.substring(1); + } + String expr = propertyExpr.get().replace("@", "@" + selectKey); + return (new ExprResult(true)).addTagExpr(selectKey, expr); + } else { + return new ExprResult(false); + } + } else { + return new ExprResult(false); + } + } + + // @ -> as_none_var + // @a -> as_var_tag_only("a") + // @.name -> as_var_property_only("name") + // @a.name -> as_var("a", "name") + default FfiVariable.ByValue getExpressionAsVar(String expr) { + // {@a.name} can not be represented as variable + if (expr.startsWith("{") && expr.endsWith("}")) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "can not convert expression of valueMap to variable"); + } + String[] splitExpr = expr.split("\\."); + if (splitExpr.length == 0) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, "expr " + expr + " is invalid"); + } + // remove first "@" + String tag = (splitExpr[0].length() > 0) ? splitExpr[0].substring(1) : splitExpr[0]; + // property "" indicates none + String property = (splitExpr.length > 1) ? splitExpr[1] : ""; + return ArgUtils.asFfiVar(tag, property); + } + + default Map getProjectTraversals(TraversalParent parent) { + if (parent instanceof SelectOneStep) { + SelectOneStep step = (SelectOneStep) parent; + Traversal.Admin selectTraversal = null; + List byTraversals = step.getLocalChildren(); + if (!byTraversals.isEmpty()) { + selectTraversal = byTraversals.get(0); + } + String selectKey = (String) step.getScopeKeys().iterator().next(); + Map selectOneByTraversal = new HashMap<>(); + selectOneByTraversal.put(selectKey, selectTraversal); + return selectOneByTraversal; + } else if (parent instanceof SelectStep) { + SelectStep step = (SelectStep) parent; + return step.getByTraversals(); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "cannot get project traversals from " + parent.getClass()); + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransformFactory.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransformFactory.java new file mode 100644 index 000000000000..e00e411b2fc4 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/TraversalParentTransformFactory.java @@ -0,0 +1,519 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform; + +import com.alibaba.graphscope.common.exception.OpArgIllegalException; +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.*; +import com.alibaba.graphscope.common.jna.type.*; +import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; +import com.alibaba.graphscope.gremlin.Utils; +import com.alibaba.graphscope.gremlin.antlr4.GremlinAntlrToJava; +import com.alibaba.graphscope.gremlin.transform.alias.AliasArg; +import com.alibaba.graphscope.gremlin.transform.alias.AliasManager; +import com.alibaba.graphscope.gremlin.transform.alias.AliasPrefixType; + +import org.apache.tinkerpop.gremlin.process.traversal.*; +import org.apache.tinkerpop.gremlin.process.traversal.lambda.IdentityTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.NotStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.TraversalFilterStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WherePredicateStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WhereTraversalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep; +import org.apache.tinkerpop.gremlin.process.traversal.util.ConnectiveP; +import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalRing; +import org.javatuples.Pair; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; + +public enum TraversalParentTransformFactory implements TraversalParentTransform { + // select("a").by("name") -> [ProjectOp("@a.name")] + // select("a").by(out().count()) -> [ApplyOp(select("a").out().count()).as("a_apply"), + // ProjectOp("@a_apply")] + // select("a", "b").by("name").by(out().count()) -> + // [ApplyOp(select("b").out().count()).as("b_apply"), ProjectOp("@a.name", "@b_apply")] + PROJECT_BY_STEP { + @Override + public List apply(TraversalParent parent) { + List interOpList = new ArrayList<>(); + Map byTraversals = getProjectTraversals(parent); + List> projectExprWithAlias = new ArrayList<>(); + int stepIdx = + TraversalHelper.stepIndex(parent.asStep(), parent.asStep().getTraversal()); + int subId = 0; + ExprResult exprRes = getSubTraversalAsExpr((new ExprArg()).addStep(parent.asStep())); + for (Map.Entry entry : byTraversals.entrySet()) { + String k = entry.getKey(); + Traversal.Admin v = entry.getValue(); + String expr; + Optional exprOpt = exprRes.getTagExpr(k); + if (exprOpt.isPresent()) { + expr = exprOpt.get(); + } else { // use apply + ApplyOp applyOp = new ApplyOp(); + applyOp.setJoinKind(new OpArg(FfiJoinKind.Inner)); + // put select("") in apply + Traversal copy = GremlinAntlrToJava.getTraversalSupplier().get(); + // copy steps in by(..) to apply + copy.asAdmin().addStep(new SelectOneStep(copy.asAdmin(), Pop.last, k)); + v.asAdmin().getSteps().forEach(s -> copy.asAdmin().addStep((Step) s)); + applyOp.setSubOpCollection( + new OpArg<>( + copy, + (Traversal traversal) -> + (new InterOpCollectionBuilder(traversal)).build())); + // column key of apply result + FfiAlias.ByValue applyAlias = + AliasManager.getFfiAlias( + new AliasArg(AliasPrefixType.PROJECT_TAG, k, stepIdx, subId)); + applyOp.setAlias(new OpArg(applyAlias, Function.identity())); + interOpList.add(applyOp); + String aliasName = applyAlias.alias.name; + expr = "@" + aliasName; + } + FfiAlias.ByValue alias = + AliasManager.getFfiAlias( + new AliasArg(AliasPrefixType.PROJECT_TAG, k, stepIdx, subId)); + projectExprWithAlias.add(Pair.with(expr, alias)); + ++subId; + } + // optimize: if there is only one expression, alias with NONE + if (projectExprWithAlias.size() == 1) { + Pair single = projectExprWithAlias.get(0); + projectExprWithAlias.set(0, single.setAt1(ArgUtils.asFfiNoneAlias())); + } + ProjectOp op = new ProjectOp(); + op.setExprWithAlias(new OpArg(projectExprWithAlias)); + interOpList.add(op); + return interOpList; + } + }, + // order().by("name"), order().by(values("name")) -> [OrderOp("@.name")] + // order().by(valueMap("name")) -> can not convert to FfiVariable with valueMap + // order().by(select("a").by("name")), order().by(select("a").by(values("name"))) -> + // OrderOp("@a.name") + // order().by(out().count) -> [ApplyOp(out().count()).as("order_1_apply"), + // OrderOp("@order_1_apply")] + // order().by("name").by(out().count) -> [ApplyOp(out().count()).as("order_2_apply"), + // OrderOp("@.name", "@order_2_apply")] + ORDER_BY_STEP { + @Override + public List apply(TraversalParent parent) { + OrderGlobalStep orderStep = (OrderGlobalStep) parent; + List interOpList = new ArrayList<>(); + List> exprWithOrderList = new ArrayList<>(); + List comparators = orderStep.getComparators(); + int stepIdx = + TraversalHelper.stepIndex(parent.asStep(), parent.asStep().getTraversal()); + for (int i = 0; i < comparators.size(); ++i) { + Pair pair = comparators.get(i); + Traversal.Admin admin = (Traversal.Admin) pair.getValue0(); + FfiOrderOpt orderOpt = getFfiOrderOpt((Order) pair.getValue1()); + ExprResult exprRes = getSubTraversalAsExpr(new ExprArg(admin)); + // i.e. order().by(values("name")) + if (exprRes.isExprPattern()) { + List exprs = exprRes.getExprs(); + exprs.forEach( + k -> { + exprWithOrderList.add(Pair.with(k, orderOpt)); + }); + } else { // use apply, i.e. order().by(out().count()) + ApplyOp applyOp = new ApplyOp(); + applyOp.setJoinKind(new OpArg(FfiJoinKind.Inner)); + applyOp.setSubOpCollection( + new OpArg<>( + admin, + (Traversal traversal) -> + (new InterOpCollectionBuilder(traversal)).build())); + FfiAlias.ByValue applyAlias = + AliasManager.getFfiAlias( + new AliasArg(AliasPrefixType.DEFAULT, stepIdx, i)); + applyOp.setAlias(new OpArg(applyAlias, Function.identity())); + interOpList.add(applyOp); + String aliasName = applyAlias.alias.name; + exprWithOrderList.add(Pair.with("@" + aliasName, orderOpt)); + } + } + OrderOp orderOp = new OrderOp(); + List varWithOrder = + exprWithOrderList.stream() + .map( + k -> { + String expr = (String) ((Pair) k).getValue0(); + FfiVariable.ByValue var = getExpressionAsVar(expr); + return ((Pair) k).setAt0(var); + }) + .collect(Collectors.toList()); + orderOp.setOrderVarWithOrder(new OpArg(varWithOrder)); + interOpList.add(orderOp); + return interOpList; + } + + private FfiOrderOpt getFfiOrderOpt(Order order) { + switch (order) { + case asc: + return FfiOrderOpt.Asc; + case desc: + return FfiOrderOpt.Desc; + case shuffle: + return FfiOrderOpt.Shuffle; + default: + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, "invalid order type"); + } + } + }, + GROUP_BY_STEP { + @Override + public List apply(TraversalParent parent) { + List interOpList = new ArrayList<>(); + // handle group key by + Traversal.Admin groupKeyTraversal = getKeyTraversal(parent); + List groupKeyExprs = new ArrayList<>(); + int stepIdx = + TraversalHelper.stepIndex(parent.asStep(), parent.asStep().getTraversal()); + ExprResult exprRes = getSubTraversalAsExpr(new ExprArg(groupKeyTraversal)); + // i.e. group().by("name") + if (exprRes.isExprPattern()) { + groupKeyExprs.addAll(exprRes.getExprs()); + } else { // use apply, i.e. group().by(out().count()) + ApplyOp applyOp = new ApplyOp(); + applyOp.setJoinKind(new OpArg(FfiJoinKind.Inner)); + applyOp.setSubOpCollection( + new OpArg<>( + groupKeyTraversal, + (Traversal traversal) -> + (new InterOpCollectionBuilder(traversal)).build())); + FfiAlias.ByValue applyAlias = + AliasManager.getFfiAlias(new AliasArg(AliasPrefixType.GROUP_KEYS, stepIdx)); + applyOp.setAlias(new OpArg(applyAlias, Function.identity())); + interOpList.add(applyOp); + String aliasName = applyAlias.alias.name; + groupKeyExprs.add("@" + aliasName); + } + List> groupKeyVarWithAlias = + new ArrayList<>(); + for (int i = 0; i < groupKeyExprs.size(); ++i) { + String expr = groupKeyExprs.get(i); + FfiVariable.ByValue groupKeyVar = getExpressionAsVar(expr); + FfiAlias.ByValue groupKeyAlias = + AliasManager.getFfiAlias( + new AliasArg(AliasPrefixType.GROUP_KEYS, stepIdx, i)); + groupKeyVarWithAlias.add(Pair.with(groupKeyVar, groupKeyAlias)); + } + Step endStep; + // group().by(values("name").as("a")), "a" is the query given alias of group key + if (groupKeyTraversal != null + && !((endStep = groupKeyTraversal.getEndStep()) instanceof EmptyStep) + && !endStep.getLabels().isEmpty()) { + String queryAlias = (String) endStep.getLabels().iterator().next(); + if (groupKeyVarWithAlias.size() == 1) { + Pair newPair = + groupKeyVarWithAlias + .get(0) + .setAt1(ArgUtils.asFfiAlias(queryAlias, true)); + groupKeyVarWithAlias.set(0, newPair); + } else if (groupKeyVarWithAlias.size() > 1) { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "the query given alias cannot apply to multiple columns"); + } + } + GroupOp groupOp = new GroupOp(); + groupOp.setGroupByKeys(new OpArg(groupKeyVarWithAlias)); + // handle group value by + groupOp.setGroupByValues(new OpArg(getGroupValueAsAggFn(parent))); + interOpList.add(groupOp); + return interOpList; + } + + private Traversal.Admin getKeyTraversal(TraversalParent step) { + if (step instanceof GroupStep) { + GroupStep groupStep = (GroupStep) step; + return groupStep.getKeyTraversal(); + } else if (step instanceof GroupCountStep) { + GroupCountStep groupStep = (GroupCountStep) step; + List keyTraversals = groupStep.getLocalChildren(); + return (keyTraversals.isEmpty()) ? null : keyTraversals.get(0); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "cannot get key traversal from " + step.getClass()); + } + } + + private Traversal.Admin getValueTraversal(TraversalParent step) { + if (step instanceof GroupStep) { + GroupStep groupStep = (GroupStep) step; + return groupStep.getValueTraversal(); + } else if (step instanceof GroupCountStep) { + Traversal.Admin countTraversal = new DefaultTraversal(); + countTraversal.addStep(new CountGlobalStep(countTraversal)); + return countTraversal; + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "cannot get value traversal from " + step.getClass()); + } + } + + public List getGroupValueAsAggFn(TraversalParent parent) { + Traversal.Admin admin = getValueTraversal(parent); + FfiAggOpt aggOpt; + int stepIdx = + TraversalHelper.stepIndex(parent.asStep(), parent.asStep().getTraversal()); + FfiAlias.ByValue alias = + AliasManager.getFfiAlias(new AliasArg(AliasPrefixType.GROUP_VALUES, stepIdx)); + String notice = + "supported pattern is [group().by(..).by(count())] or" + + " [group().by(..).by(fold())]"; + if (admin == null + || admin instanceof IdentityTraversal + || admin.getSteps().size() == 2 + && isMapIdentity(admin.getStartStep()) + && admin.getEndStep() + instanceof FoldStep) { // group, // group().by(..).by() + aggOpt = FfiAggOpt.ToList; + } else if (admin.getSteps().size() == 1) { + if (admin.getStartStep() instanceof CountGlobalStep) { // group().by(..).by(count()) + aggOpt = FfiAggOpt.Count; + } else if (admin.getStartStep() instanceof FoldStep) { // group().by(..).by(fold()) + aggOpt = FfiAggOpt.ToList; + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, notice); + } + // group().by(..).by(count().as("a")), "a" is the query given alias of group value + Set labels = admin.getStartStep().getLabels(); + if (labels != null && !labels.isEmpty()) { + String label = labels.iterator().next(); + alias = ArgUtils.asFfiAlias(label, true); + } + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.UNSUPPORTED_TYPE, notice); + } + return Collections.singletonList(new ArgAggFn(aggOpt, alias)); + } + + // TraversalMapStep(identity) + private boolean isMapIdentity(Step step) { + if (!(step instanceof TraversalMapStep)) { + return false; + } + TraversalMapStep mapStep = (TraversalMapStep) step; + Traversal.Admin mapTraversal = + mapStep.getLocalChildren().size() > 0 + ? (Traversal.Admin) mapStep.getLocalChildren().get(0) + : null; + return mapTraversal != null && mapTraversal instanceof IdentityTraversal; + } + }, + WHERE_BY_STEP { + @Override + public List apply(TraversalParent parent) { + List interOpList = new ArrayList<>(); + + WherePredicateStep step = (WherePredicateStep) parent; + Optional startKey = step.getStartKey(); + TraversalRing traversalRing = + Utils.getFieldValue(WherePredicateStep.class, step, "traversalRing"); + + int stepId = TraversalHelper.stepIndex(parent.asStep(), parent.asStep().getTraversal()); + AtomicInteger subId = new AtomicInteger(0); + + String startTag = startKey.isPresent() ? startKey.get() : ""; + String startExpr = + getExprWithApplys(startTag, traversalRing.next(), stepId, subId, interOpList); + + P predicate = (P) step.getPredicate().get(); + List selectKeys = + Utils.getFieldValue(WherePredicateStep.class, step, "selectKeys"); + traverseAndUpdateP( + predicate, selectKeys.iterator(), traversalRing, stepId, subId, interOpList); + + String expr = + PredicateExprTransformFactory.HAS_STEP.flatPredicate(startExpr, predicate); + SelectOp selectOp = new SelectOp(); + selectOp.setPredicate(new OpArg(expr)); + + interOpList.add(selectOp); + return interOpList; + } + + private void traverseAndUpdateP( + P predicate, + Iterator selectKeysIterator, + TraversalRing traversalRing, + int stepId, + AtomicInteger subId, + List applys) { + if (predicate instanceof ConnectiveP) { + ((ConnectiveP) predicate) + .getPredicates() + .forEach( + p1 -> { + traverseAndUpdateP( + (P) p1, + selectKeysIterator, + traversalRing, + stepId, + subId, + applys); + }); + } else { + String expr = + getExprWithApplys( + selectKeysIterator.next(), + traversalRing.next(), + stepId, + subId, + applys); + FfiVariable.ByValue var = getExpressionAsVar(expr); + predicate.setValue(var); + } + } + + private String getExprWithApplys( + String tag, + Traversal.Admin whereby, + int stepId, + AtomicInteger subId, + List applys) { + Traversal.Admin tagBy = asSelectTraversal(tag, whereby); + ExprResult exprRes = getSubTraversalAsExpr(new ExprArg(tagBy)); + if (!exprRes.isExprPattern()) { + ApplyOp applyOp = new ApplyOp(); + applyOp.setJoinKind(new OpArg(FfiJoinKind.Inner)); + // put select("") in apply + Traversal copy = GremlinAntlrToJava.getTraversalSupplier().get(); + copy.asAdmin().addStep(new SelectOneStep(copy.asAdmin(), Pop.last, tag)); + // copy steps in by(..) to apply + whereby.getSteps().forEach(s -> copy.asAdmin().addStep((Step) s)); + applyOp.setSubOpCollection( + new OpArg<>( + copy, + (Traversal traversal) -> + (new InterOpCollectionBuilder(traversal)).build())); + FfiAlias.ByValue applyAlias = + AliasManager.getFfiAlias( + new AliasArg( + AliasPrefixType.DEFAULT, stepId, subId.getAndIncrement())); + applyOp.setAlias(new OpArg(applyAlias)); + applys.add(applyOp); + return "@" + applyAlias.alias.name; + } else { + return exprRes.getSingleExpr().get(); + } + } + + private Traversal.Admin asSelectTraversal(String selectKey, Traversal.Admin byTraversal) { + Traversal.Admin traversal = + (Traversal.Admin) GremlinAntlrToJava.getTraversalSupplier().get(); + SelectOneStep oneStep = new SelectOneStep(traversal, Pop.last, selectKey); + oneStep.modulateBy(byTraversal); + traversal.addStep(oneStep); + return traversal; + } + }, + WHERE_TRAVERSAL_STEP { + @Override + public List apply(TraversalParent parent) { + Traversal.Admin subTraversal = getWhereSubTraversal(parent.asStep()); + ExprResult exprRes = getSubTraversalAsExpr(new ExprArg(subTraversal)); + if (exprRes.isExprPattern()) { + String expr = exprRes.getSingleExpr().get(); + SelectOp selectOp = new SelectOp(); + selectOp.setPredicate(new OpArg(expr)); + return Collections.singletonList(selectOp); + } else { // apply + ApplyOp applyOp = new ApplyOp(); + FfiJoinKind joinKind = FfiJoinKind.Semi; + applyOp.setJoinKind(new OpArg(joinKind, Function.identity())); + applyOp.setSubOpCollection( + new OpArg<>( + subTraversal, + (Traversal traversal) -> + (new InterOpCollectionBuilder(traversal)).build())); + return Collections.singletonList(applyOp); + } + } + + private Traversal.Admin getWhereSubTraversal(Step step) { + if (step instanceof TraversalFilterStep) { + return ((TraversalFilterStep) step).getFilterTraversal(); + } else if (step instanceof WhereTraversalStep) { + WhereTraversalStep whereStep = (WhereTraversalStep) step; + List subTraversals = whereStep.getLocalChildren(); + return subTraversals.isEmpty() ? null : subTraversals.get(0); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "cannot get where traversal from " + step.getClass()); + } + } + }, + NOT_TRAVERSAL_STEP { + @Override + public List apply(TraversalParent parent) { + Traversal.Admin subTraversal = getNotSubTraversal(parent.asStep()); + ExprResult exprRes = getSubTraversalAsExpr(new ExprArg(subTraversal)); + if (exprRes.isExprPattern()) { // not(select("a").by("name")) + String notExpr = getNotExpr(exprRes.getSingleExpr().get()); + SelectOp selectOp = new SelectOp(); + selectOp.setPredicate(new OpArg(notExpr)); + return Collections.singletonList(selectOp); + } else { // apply + ApplyOp applyOp = new ApplyOp(); + FfiJoinKind joinKind = FfiJoinKind.Anti; + applyOp.setJoinKind(new OpArg(joinKind)); + applyOp.setSubOpCollection( + new OpArg<>( + subTraversal, + (Traversal traversal) -> + (new InterOpCollectionBuilder(traversal)).build())); + return Collections.singletonList(applyOp); + } + } + + private Traversal.Admin getNotSubTraversal(Step step) { + if (step instanceof NotStep) { + NotStep notStep = (NotStep) step; + List subTraversals = notStep.getLocalChildren(); + return subTraversals.isEmpty() ? null : subTraversals.get(0); + } else { + throw new OpArgIllegalException( + OpArgIllegalException.Cause.INVALID_TYPE, + "cannot get where traversal from " + step.getClass()); + } + } + + private String getNotExpr(String expr) { + return (expr.contains("&&") || expr.contains("||")) + ? String.format("!(%s)", expr) + : "!" + expr; + } + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasArg.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasArg.java new file mode 100644 index 000000000000..fba32f80f1df --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasArg.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform.alias; + +public class AliasArg { + public static String GROUP_KEYS = "~keys"; + public static String GROUP_VALUES = "~values"; + public static String DEFAULT = "~alias"; + + // the prefix is composed of the type and the tag + // there are 4 kinds of alias prefix, {keys, values, default, tag} + private String tag; + private AliasPrefixType type; + // stepIdx_subTraversalIdx is as suffix + // stepIdx is used to differentiate multiple select(..) in a gremlin query + // subTraversalIdx is used to differentiate multiple apply from a same operator + private int stepIdx; + private int subTraversalIdx; + + private AliasArg() { + this.tag = ""; + this.type = AliasPrefixType.DEFAULT; + this.stepIdx = 0; + this.subTraversalIdx = 0; + } + + public AliasArg(AliasPrefixType type) { + super(); + this.type = type; + } + + public AliasArg(AliasPrefixType type, int stepIdx) { + this(type); + this.stepIdx = stepIdx; + } + + public AliasArg(AliasPrefixType type, int stepIdx, int subTraversalIdx) { + this(type, stepIdx); + this.subTraversalIdx = subTraversalIdx; + } + + public AliasArg(AliasPrefixType type, String tag) { + super(); + if (tag.isEmpty()) { + this.type = AliasPrefixType.DEFAULT; + } else { + this.tag = tag; + this.type = type; + } + } + + public AliasArg(AliasPrefixType type, String tag, int stepIdx) { + this(type, tag); + this.stepIdx = stepIdx; + } + + public AliasArg(AliasPrefixType type, String tag, int stepIdx, int subTraversalIdx) { + this(type, tag, stepIdx); + this.subTraversalIdx = subTraversalIdx; + } + + public String getPrefix() { + switch (type) { + case PROJECT_TAG: + return this.tag; + case GROUP_KEYS: + return GROUP_KEYS; + case GROUP_VALUES: + return GROUP_VALUES; + case DEFAULT: + default: + return DEFAULT; + } + } + + public int getStepIdx() { + return stepIdx; + } + + public int getSubTraversalIdx() { + return subTraversalIdx; + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasManager.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasManager.java new file mode 100644 index 000000000000..32c545a97ee8 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasManager.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform.alias; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.type.FfiAlias; + +public class AliasManager { + public static FfiAlias.ByValue getFfiAlias(AliasArg prefix) { + int stepIdx = prefix.getStepIdx(); + int subTraversalId = prefix.getSubTraversalIdx(); + String alias = prefix.getPrefix() + "_" + stepIdx + "_" + subTraversalId; + return ArgUtils.asFfiAlias(alias, false); + } + + // prefix is as the gremlin result key which is used to display + public static String getPrefix(String aliasName) { + String[] splits = aliasName.split("_"); + return (splits.length == 0) ? "" : splits[0]; + } + + public static boolean isGroupKeysPrefix(String aliasName) { + return aliasName.startsWith(AliasArg.GROUP_KEYS); + } + + public static boolean isGroupValuesPrefix(String aliasName) { + return aliasName.startsWith(AliasArg.GROUP_VALUES); + } +} diff --git a/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasPrefixType.java b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasPrefixType.java new file mode 100644 index 000000000000..50a1c782fdf2 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/transform/alias/AliasPrefixType.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.transform.alias; + +public enum AliasPrefixType { + GROUP_KEYS, + GROUP_VALUES, + PROJECT_TAG, + DEFAULT +} diff --git a/research/query_service/ir/compiler/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory b/research/query_service/ir/compiler/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory new file mode 100644 index 000000000000..d10c6addff59 --- /dev/null +++ b/research/query_service/ir/compiler/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory @@ -0,0 +1 @@ +com.alibaba.graphscope.gremlin.plugin.script.AntlrToJavaScriptEngineFactory diff --git a/research/query_service/ir/compiler/src/main/resources/conf/gremlin-server.yaml b/research/query_service/ir/compiler/src/main/resources/conf/gremlin-server.yaml new file mode 100644 index 000000000000..8f66ce9c413a --- /dev/null +++ b/research/query_service/ir/compiler/src/main/resources/conf/gremlin-server.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +host: localhost +port: 8182 +evaluationTimeout: 300000 +channelizer: com.alibaba.graphscope.gremlin.service.IrWsAndHttpChannelizer +graphs: {} +scriptEngines: { + gremlin-groovy: { + plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {classImports: [java.lang.Math], methodImports: [java.lang.Math#*]},}}} +serializers: + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, config: { ioRegistries: [org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerIoRegistryV3d0] }} # application/json + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1 } # application/vnd.graphbinary-v1.0 + - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, config: { serializeResultToString: true }} # application/vnd.graphbinary-v1.0-stringd + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV3d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v3.0+gryo-stringd + - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0} # application/vnd.gremlin-v1.0+gryo +processors: + - { className: org.apache.tinkerpop.gremlin.server.op.session.SessionOpProcessor, config: { sessionTimeout: 28800000 }} + - { className: org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor, config: { cacheExpirationTime: 600000, cacheMaxSize: 1000 }} +metrics: { + consoleReporter: {enabled: true, interval: 180000}, + csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv}, + jmxReporter: {enabled: true}, + slf4jReporter: {enabled: true, interval: 180000}} +strictTransactionManagement: false +idleConnectionTimeout: 0 +keepAliveInterval: 0 +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: { + enabled: false} \ No newline at end of file diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/ByValueTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/ByValueTest.java new file mode 100644 index 000000000000..63cc1ff12e5a --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/ByValueTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common; + +import com.alibaba.graphscope.common.jna.IrCoreLibrary; +import com.alibaba.graphscope.common.jna.type.*; + +import org.junit.Assert; +import org.junit.Test; + +public class ByValueTest { + private static IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE; + + @Test + public void cstrAsNameOrIdTest() { + String tag = "p"; + FfiNameOrId.ByValue nameOrId = irCoreLib.cstrAsNameOrId(tag); + Assert.assertEquals(FfiNameIdOpt.Name, nameOrId.opt); + Assert.assertEquals(tag, nameOrId.name); + } + + @Test + public void cstrAsConstTest() { + String value = "marko"; + FfiConst.ByValue ffiConst = irCoreLib.cstrAsConst(value); + Assert.assertEquals(FfiDataType.Str, ffiConst.dataType); + Assert.assertEquals(value, ffiConst.cstr); + } + + @Test + public void int64AsConstTest() { + long value = 10L; + FfiConst.ByValue ffiConst = irCoreLib.int64AsConst(value); + Assert.assertEquals(FfiDataType.I64, ffiConst.dataType); + Assert.assertEquals(value, ffiConst.int64); + } + + @Test + public void asLabelKeyTest() { + FfiProperty.ByValue property = irCoreLib.asLabelKey(); + Assert.assertEquals(FfiPropertyOpt.Label, property.opt); + } + + @Test + public void asIdKeyTest() { + FfiProperty.ByValue property = irCoreLib.asIdKey(); + Assert.assertEquals(FfiPropertyOpt.Id, property.opt); + } + + @Test + public void asPropertyKeyTest() { + String key = "age"; + FfiProperty.ByValue property = irCoreLib.asPropertyKey(irCoreLib.cstrAsNameOrId(key)); + Assert.assertEquals(FfiPropertyOpt.Key, property.opt); + Assert.assertEquals(FfiNameIdOpt.Name, property.key.opt); + Assert.assertEquals(key, property.key.name); + } + + @Test + public void asVarTagOnlyTest() { + String tag = "p"; + FfiVariable.ByValue variable = irCoreLib.asVarTagOnly(irCoreLib.cstrAsNameOrId(tag)); + Assert.assertEquals(FfiNameIdOpt.Name, variable.tag.opt); + Assert.assertEquals(tag, variable.tag.name); + Assert.assertEquals(FfiPropertyOpt.None, variable.property.opt); + } + + @Test + public void asVarPropertyOnlyTest() { + String key = "age"; + FfiVariable.ByValue variable = + irCoreLib.asVarPropertyOnly(irCoreLib.asPropertyKey(irCoreLib.cstrAsNameOrId(key))); + Assert.assertEquals(FfiPropertyOpt.Key, variable.property.opt); + Assert.assertEquals(FfiNameIdOpt.Name, variable.property.key.opt); + Assert.assertEquals(key, variable.property.key.name); + Assert.assertEquals(FfiNameIdOpt.None, variable.tag.opt); + } + + @Test + public void asVarTest() { + String tag = "p"; + String key = "age"; + FfiVariable.ByValue variable = + irCoreLib.asVar( + irCoreLib.cstrAsNameOrId(tag), + irCoreLib.asPropertyKey(irCoreLib.cstrAsNameOrId(key))); + Assert.assertEquals(FfiNameIdOpt.Name, variable.tag.opt); + Assert.assertEquals(tag, variable.tag.name); + Assert.assertEquals(FfiPropertyOpt.Key, variable.property.opt); + Assert.assertEquals(FfiNameIdOpt.Name, variable.property.key.opt); + Assert.assertEquals(key, variable.property.key.name); + } + + @Test + public void asNoneVarTest() { + FfiVariable.ByValue variable = irCoreLib.asNoneVar(); + Assert.assertEquals(FfiPropertyOpt.None, variable.property.opt); + Assert.assertEquals(FfiNameIdOpt.None, variable.tag.opt); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/DedupOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/DedupOpTest.java new file mode 100644 index 000000000000..8c58fd14ad88 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/DedupOpTest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.type.FfiVariable; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Function; + +public class DedupOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void dedupTest() throws IOException { + DedupOp op = new DedupOp(); + FfiVariable.ByValue dedupKey = ArgUtils.asFfiNoneVar(); + op.setDedupKeys(new OpArg(Collections.singletonList(dedupKey), Function.identity())); + + irPlan.appendInterOp(-1, op); + Assert.assertEquals(FileUtils.readJsonFromResource("dedup.json"), irPlan.getPlanAsJson()); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOpTest.java new file mode 100644 index 000000000000..90639bfb0b6f --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOpTest.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.IrCoreLibrary; +import com.alibaba.graphscope.common.jna.type.FfiDirection; +import com.alibaba.graphscope.common.jna.type.FfiNameOrId; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +public class ExpandOpTest { + private static IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE; + private IrPlan irPlan = new IrPlan(); + + @Test + public void edgeOptTest() throws IOException { + ExpandOp op = new ExpandOp(); + op.setEdgeOpt(new OpArg<>(Boolean.valueOf(true), Function.identity())); + op.setDirection(new OpArg<>(FfiDirection.Out, Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("expand_edge_opt.json"), actual); + } + + @Test + public void labelsTest() throws IOException { + ExpandOp op = new ExpandOp(); + op.setEdgeOpt(new OpArg<>(Boolean.valueOf(true), Function.identity())); + op.setDirection(new OpArg<>(FfiDirection.Out, Function.identity())); + List values = Arrays.asList(irCoreLib.cstrAsNameOrId("knows")); + op.setLabels(new OpArg(values, Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("expand_labels.json"), actual); + } + + @Test + public void aliasTest() throws IOException { + ExpandOp op = new ExpandOp(); + op.setEdgeOpt(new OpArg<>(Boolean.valueOf(true), Function.identity())); + op.setDirection(new OpArg<>(FfiDirection.Out, Function.identity())); + op.setAlias(new OpArg(ArgUtils.asFfiAlias("a", true), Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("expand_alias.json"), actual); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/GroupOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/GroupOpTest.java new file mode 100644 index 000000000000..ce805fa6efe9 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/GroupOpTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.type.*; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.javatuples.Pair; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Function; + +public class GroupOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void countTest() throws IOException { + GroupOp op = new GroupOp(); + op.setGroupByKeys(new OpArg(Collections.emptyList(), Function.identity())); + ArgAggFn aggFn = new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("values", false)); + op.setGroupByValues(new OpArg(Collections.singletonList(aggFn), Function.identity())); + + irPlan.appendInterOp(-1, op); + Assert.assertEquals(FileUtils.readJsonFromResource("count.json"), irPlan.getPlanAsJson()); + } + + @Test + public void countAsTest() throws IOException { + GroupOp op = new GroupOp(); + op.setGroupByKeys(new OpArg(Collections.emptyList(), Function.identity())); + ArgAggFn aggFn = new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("a", true)); + op.setGroupByValues(new OpArg(Collections.singletonList(aggFn), Function.identity())); + + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("count_as.json"), irPlan.getPlanAsJson()); + } + + @Test + public void groupTest() throws IOException { + GroupOp op = new GroupOp(); + Pair groupKey = + Pair.with(ArgUtils.asFfiNoneVar(), ArgUtils.asFfiAlias("keys", false)); + op.setGroupByKeys(new OpArg(Collections.singletonList(groupKey), Function.identity())); + + ArgAggFn aggFn = new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("values", false)); + op.setGroupByValues(new OpArg(Collections.singletonList(aggFn), Function.identity())); + + irPlan.appendInterOp(-1, op); + Assert.assertEquals(FileUtils.readJsonFromResource("group.json"), irPlan.getPlanAsJson()); + } + + @Test + public void groupByKeyTest() throws IOException { + GroupOp op = new GroupOp(); + Pair groupKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("keys_name", false)); + op.setGroupByKeys(new OpArg(Collections.singletonList(groupKey), Function.identity())); + + ArgAggFn aggFn = new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("values", false)); + op.setGroupByValues(new OpArg(Collections.singletonList(aggFn), Function.identity())); + + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("group_key.json"), irPlan.getPlanAsJson()); + } + + @Test + public void groupByKeyByCountTest() throws IOException { + GroupOp op = new GroupOp(); + Pair groupKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("keys_name", false)); + op.setGroupByKeys(new OpArg(Collections.singletonList(groupKey), Function.identity())); + + ArgAggFn aggFn = new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("values", false)); + op.setGroupByValues(new OpArg(Collections.singletonList(aggFn), Function.identity())); + + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("group_key_count.json"), irPlan.getPlanAsJson()); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/LimitOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/LimitOpTest.java new file mode 100644 index 000000000000..aac40f8819f1 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/LimitOpTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.function.Function; + +public class LimitOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void limitOpTest() throws IOException { + LimitOp op = new LimitOp(); + op.setLower(new OpArg<>(Integer.valueOf(1), Function.identity())); + op.setUpper(new OpArg<>(Integer.valueOf(2), Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("limit_range.json"), actual); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/OrderOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/OrderOpTest.java new file mode 100644 index 000000000000..1618c82cddfd --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/OrderOpTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.type.FfiOrderOpt; +import com.alibaba.graphscope.common.jna.type.FfiVariable; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.javatuples.Pair; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.function.Function; + +public class OrderOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void orderTest() throws IOException { + OrderOp op = new OrderOp(); + op.setOrderVarWithOrder( + new OpArg( + Arrays.asList(Pair.with(ArgUtils.asFfiNoneVar(), FfiOrderOpt.Asc)), + Function.identity())); + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("order_asc.json"), irPlan.getPlanAsJson()); + } + + @Test + public void orderByKeyTest() throws IOException { + OrderOp op = new OrderOp(); + FfiVariable.ByValue var = ArgUtils.asFfiVar("", "name"); + op.setOrderVarWithOrder( + new OpArg(Arrays.asList(Pair.with(var, FfiOrderOpt.Asc)), Function.identity())); + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("order_key.json"), irPlan.getPlanAsJson()); + } + + @Test + public void orderByKeysTest() throws IOException { + OrderOp op = new OrderOp(); + FfiVariable.ByValue v1 = ArgUtils.asFfiVar("", "name"); + FfiVariable.ByValue v2 = ArgUtils.asFfiVar("", "id"); + op.setOrderVarWithOrder( + new OpArg( + Arrays.asList( + Pair.with(v1, FfiOrderOpt.Asc), Pair.with(v2, FfiOrderOpt.Desc)), + Function.identity())); + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("order_keys.json"), irPlan.getPlanAsJson()); + } + + @Test + public void orderByLabelTest() throws IOException { + OrderOp op = new OrderOp(); + FfiVariable.ByValue var = ArgUtils.asFfiVar("", "~label"); + op.setOrderVarWithOrder( + new OpArg(Arrays.asList(Pair.with(var, FfiOrderOpt.Asc)), Function.identity())); + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("order_label.json"), irPlan.getPlanAsJson()); + } + + @Test + public void orderLimitTest() throws IOException { + OrderOp op = new OrderOp(); + FfiVariable.ByValue var = ArgUtils.asFfiNoneVar(); + op.setOrderVarWithOrder( + new OpArg(Arrays.asList(Pair.with(var, FfiOrderOpt.Asc)), Function.identity())); + op.setLower(new OpArg(Integer.valueOf(1), Function.identity())); + op.setUpper(new OpArg(Integer.valueOf(2), Function.identity())); + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("order_limit.json"), irPlan.getPlanAsJson()); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOpTest.java new file mode 100644 index 000000000000..edfdb8ba1df3 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOpTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.jna.type.FfiDirection; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.function.Function; + +public class PathExpandOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void expand_1_5_Test() throws IOException { + PathExpandOp op = new PathExpandOp(); + op.setEdgeOpt(new OpArg<>(Boolean.valueOf(false), Function.identity())); + op.setDirection(new OpArg<>(FfiDirection.Out, Function.identity())); + op.setLower(new OpArg(Integer.valueOf(1), Function.identity())); + op.setUpper(new OpArg(Integer.valueOf(5), Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("path_expand.json"), actual); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ProjectOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ProjectOpTest.java new file mode 100644 index 000000000000..4df920353e47 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ProjectOpTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.type.FfiAlias; +import com.alibaba.graphscope.common.jna.type.FfiScanOpt; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.javatuples.Pair; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +public class ProjectOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void projectKeyTest() throws IOException { + ProjectOp op = new ProjectOp(); + + String projectExpr = "@.name"; + FfiAlias.ByValue alias = ArgUtils.asFfiAlias("name", false); + List exprWithAlias = Arrays.asList(Pair.with(projectExpr, alias)); + + op.setExprWithAlias(new OpArg(exprWithAlias, Function.identity())); + irPlan.appendInterOp(-1, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("project_key.json"), irPlan.getPlanAsJson()); + } + + @Test + public void projectTagKeyTest() throws IOException { + ScanFusionOp scanOp = new ScanFusionOp(); + scanOp.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + scanOp.setAlias(new OpArg(ArgUtils.asFfiAlias("a", true), Function.identity())); + irPlan.appendInterOp(-1, scanOp); + + ProjectOp op = new ProjectOp(); + + String projectExpr = "@a.name"; + FfiAlias.ByValue alias = ArgUtils.asFfiAlias("a_name", false); + List exprWithAlias = Collections.singletonList(Pair.with(projectExpr, alias)); + + op.setExprWithAlias(new OpArg(exprWithAlias, Function.identity())); + irPlan.appendInterOp(0, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("project_tag_key.json"), irPlan.getPlanAsJson()); + } + + @Test + public void projectTagKeysTest() throws IOException { + ScanFusionOp scanOp = new ScanFusionOp(); + scanOp.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + scanOp.setAlias(new OpArg(ArgUtils.asFfiAlias("a", true), Function.identity())); + irPlan.appendInterOp(-1, scanOp); + + ProjectOp op = new ProjectOp(); + + String projectExpr = "{@a.name, @a.id}"; + FfiAlias.ByValue alias = ArgUtils.asFfiAlias("a_{name, id}", false); + List exprWithAlias = Collections.singletonList(Pair.with(projectExpr, alias)); + + op.setExprWithAlias(new OpArg(exprWithAlias, Function.identity())); + irPlan.appendInterOp(0, op); + Assert.assertEquals( + FileUtils.readJsonFromResource("project_tag_keys.json"), irPlan.getPlanAsJson()); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ScanFusionOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ScanFusionOpTest.java new file mode 100644 index 000000000000..f6780e87a952 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/ScanFusionOpTest.java @@ -0,0 +1,98 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.jna.IrCoreLibrary; +import com.alibaba.graphscope.common.jna.type.FfiConst; +import com.alibaba.graphscope.common.jna.type.FfiNameOrId; +import com.alibaba.graphscope.common.jna.type.FfiScanOpt; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +public class ScanFusionOpTest { + private IrCoreLibrary irCoreLib = IrCoreLibrary.INSTANCE; + private IrPlan irPlan = new IrPlan(); + + @Test + public void scanOptTest() throws IOException { + ScanFusionOp op = new ScanFusionOp(); + op.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("scan_opt.json"), actual); + } + + @Test + public void predicateTest() throws IOException { + ScanFusionOp op = new ScanFusionOp(); + op.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + op.setPredicate(new OpArg("@.id == 1", Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("scan_expr.json"), actual); + } + + @Test + public void labelsTest() throws IOException { + ScanFusionOp op = new ScanFusionOp(); + op.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + List values = Arrays.asList(irCoreLib.cstrAsNameOrId("person")); + op.setLabels(new OpArg(values, Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("scan_labels.json"), actual); + } + + @Test + public void idsTest() throws IOException { + ScanFusionOp op = new ScanFusionOp(); + op.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + List values = + Arrays.asList(irCoreLib.int64AsConst(1L), irCoreLib.int64AsConst(2L)); + op.setIds(new OpArg(values, Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("scan_ids.json"), actual); + } + + @Test + public void aliasTest() throws IOException { + ScanFusionOp op = new ScanFusionOp(); + op.setScanOpt(new OpArg<>(FfiScanOpt.Entity, Function.identity())); + op.setAlias(new OpArg(ArgUtils.asFfiAlias("a", true), Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("scan_alias.json"), actual); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/SelectOpTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/SelectOpTest.java new file mode 100644 index 000000000000..371d6fceba6a --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/operator/SelectOpTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.operator; + +import com.alibaba.graphscope.common.IrPlan; +import com.alibaba.graphscope.common.utils.FileUtils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.function.Function; + +public class SelectOpTest { + private IrPlan irPlan = new IrPlan(); + + @Test + public void selectOpTest() throws IOException { + SelectOp op = new SelectOp(); + op.setPredicate(new OpArg("@.id == 1 && @.name == \"marko\"", Function.identity())); + irPlan.appendInterOp(-1, op); + String actual = irPlan.getPlanAsJson(); + Assert.assertEquals(FileUtils.readJsonFromResource("select_expr.json"), actual); + } + + @After + public void after() { + if (irPlan != null) { + irPlan.close(); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/strategy/TopKStrategyTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/strategy/TopKStrategyTest.java new file mode 100644 index 000000000000..e188f0e854b3 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/common/intermediate/strategy/TopKStrategyTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.intermediate.strategy; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.LimitOp; +import com.alibaba.graphscope.common.intermediate.operator.OpArg; +import com.alibaba.graphscope.common.intermediate.operator.OrderOp; +import com.alibaba.graphscope.common.jna.type.FfiOrderOpt; + +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +public class TopKStrategyTest { + @Test + public void orderLimitTest() { + InterOpCollection opCollection = new InterOpCollection(); + + OrderOp orderOp = new OrderOp(); + List orderPair = Arrays.asList(Pair.with(ArgUtils.asFfiNoneVar(), FfiOrderOpt.Asc)); + orderOp.setOrderVarWithOrder(new OpArg(orderPair, Function.identity())); + opCollection.appendInterOp(orderOp); + + LimitOp limitOp = new LimitOp(); + limitOp.setLower(new OpArg<>(Integer.valueOf(1), Function.identity())); + limitOp.setUpper(new OpArg<>(Integer.valueOf(2), Function.identity())); + opCollection.appendInterOp(limitOp); + + TopKStrategy.INSTANCE.apply(opCollection); + OrderOp orderLimit = (OrderOp) opCollection.unmodifiableCollection().get(0); + + Assert.assertEquals(1, opCollection.unmodifiableCollection().size()); + Assert.assertEquals(1, orderLimit.getLower().get().applyArg()); + Assert.assertEquals(2, orderLimit.getUpper().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/CountStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/CountStepTest.java new file mode 100644 index 000000000000..403aa8d3b517 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/CountStepTest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.GroupOp; +import com.alibaba.graphscope.common.jna.type.FfiAggOpt; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class CountStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_count_test() { + Traversal traversal = g.V().count(); + Step step = traversal.asAdmin().getEndStep(); + GroupOp op = (GroupOp) StepTransformFactory.COUNT_STEP.apply(step); + + Assert.assertEquals(Collections.emptyList(), op.getGroupByKeys().get().applyArg()); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_count_as_test() { + Traversal traversal = g.V().count().as("a"); + Step step = traversal.asAdmin().getEndStep(); + GroupOp op = (GroupOp) StepTransformFactory.COUNT_STEP.apply(step); + + Assert.assertEquals(Collections.emptyList(), op.getGroupByKeys().get().applyArg()); + ArgAggFn expectedValue = new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("a", true)); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/DedupStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/DedupStepTest.java new file mode 100644 index 000000000000..213db46e06aa --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/DedupStepTest.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.DedupOp; +import com.alibaba.graphscope.common.jna.type.FfiVariable; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class DedupStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_dedup_test() { + Traversal traversal = g.V().dedup(); + + Step step = traversal.asAdmin().getEndStep(); + DedupOp op = (DedupOp) StepTransformFactory.DEDUP_STEP.apply(step); + + FfiVariable.ByValue expectedVar = ArgUtils.asFfiNoneVar(); + Assert.assertEquals( + Collections.singletonList(expectedVar), op.getDedupKeys().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/EmptyTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/EmptyTest.java new file mode 100644 index 000000000000..43ae9a9e845c --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/EmptyTest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.operator.ExpandOp; +import com.alibaba.graphscope.common.intermediate.operator.ScanFusionOp; +import com.alibaba.graphscope.gremlin.plugin.processor.IrStandardOpProcessor; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +// test whether Optional in InterOp is empty when parameters in traversal is not set +public class EmptyTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_has_no_id_test() { + Traversal traversal = g.V(); + IrStandardOpProcessor.applyStrategies(traversal); + Step step = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.SCAN_FUSION_STEP.apply(step); + Assert.assertEquals(false, op.getIds().isPresent()); + } + + @Test + public void g_V_has_no_label_has_no_property_test() { + Traversal traversal = g.V(); + IrStandardOpProcessor.applyStrategies(traversal); + Step step = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.SCAN_FUSION_STEP.apply(step); + Assert.assertEquals(false, op.getPredicate().isPresent()); + Assert.assertEquals(false, op.getLabels().isPresent()); + } + + @Test + public void g_V_has_label_has_no_property_test() { + Traversal traversal = g.V().hasLabel("person"); + IrStandardOpProcessor.applyStrategies(traversal); + Step step = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.SCAN_FUSION_STEP.apply(step); + Assert.assertEquals(false, op.getPredicate().isPresent()); + Assert.assertEquals(true, op.getLabels().isPresent()); + } + + @Test + public void g_V_has_no_label_has_property_test() { + Traversal traversal = g.V().has("id", 1); + IrStandardOpProcessor.applyStrategies(traversal); + Step step = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.SCAN_FUSION_STEP.apply(step); + Assert.assertEquals(true, op.getPredicate().isPresent()); + Assert.assertEquals(false, op.getLabels().isPresent()); + } + + @Test + public void g_V_outE_has_no_label_test() { + Traversal traversal = g.V().out(); + IrStandardOpProcessor.applyStrategies(traversal); + Step step = traversal.asAdmin().getEndStep(); + ExpandOp op = (ExpandOp) StepTransformFactory.VERTEX_STEP.apply(step); + Assert.assertEquals(false, op.getLabels().isPresent()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/ExprStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/ExprStepTest.java new file mode 100644 index 000000000000..244c3577eac6 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/ExprStepTest.java @@ -0,0 +1,53 @@ +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.ProjectOp; +import com.alibaba.graphscope.common.intermediate.operator.SelectOp; +import com.alibaba.graphscope.gremlin.plugin.step.ExprStep; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class ExprStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_where_expr_test() { + // g.V().where(expr("@.age")) + Traversal traversal = g.V(); + traversal + .asAdmin() + .addStep(new ExprStep(traversal.asAdmin(), "@.age", ExprStep.Type.FILTER)); + Step whereExpr = traversal.asAdmin().getEndStep(); + + SelectOp selectOp = (SelectOp) StepTransformFactory.EXPR_STEP.apply(whereExpr); + String predicate = (String) selectOp.getPredicate().get().applyArg(); + Assert.assertEquals("@.age", predicate); + } + + @Test + public void g_V_select_expr_test() { + // g.V().select(expr("@.name")) + Traversal traversal = g.V(); + traversal + .asAdmin() + .addStep(new ExprStep(traversal.asAdmin(), "@.name", ExprStep.Type.PROJECTION)); + Step whereExpr = traversal.asAdmin().getEndStep(); + + ProjectOp projectOp = (ProjectOp) StepTransformFactory.EXPR_STEP.apply(whereExpr); + + List exprWithAlias = (List) projectOp.getExprWithAlias().get().applyArg(); + Assert.assertEquals("@.name", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/GraphStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/GraphStepTest.java new file mode 100644 index 000000000000..3265da9efb0c --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/GraphStepTest.java @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.ScanFusionOp; +import com.alibaba.graphscope.common.jna.type.FfiAlias; +import com.alibaba.graphscope.common.jna.type.FfiConst; +import com.alibaba.graphscope.common.jna.type.FfiNameOrId; +import com.alibaba.graphscope.common.jna.type.FfiScanOpt; +import com.alibaba.graphscope.gremlin.plugin.processor.IrStandardOpProcessor; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class GraphStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_test() { + Traversal traversal = g.V(); + Step graphStep = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.GRAPH_STEP.apply(graphStep); + Assert.assertEquals(FfiScanOpt.Entity, op.getScanOpt().get().applyArg()); + } + + @Test + public void g_E_test() { + Traversal traversal = g.E(); + Step graphStep = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.GRAPH_STEP.apply(graphStep); + Assert.assertEquals(FfiScanOpt.Relation, op.getScanOpt().get().applyArg()); + } + + @Test + public void g_V_label_test() { + Traversal traversal = g.V().hasLabel("person"); + IrStandardOpProcessor.applyStrategies(traversal); + Step graphStep = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.SCAN_FUSION_STEP.apply(graphStep); + FfiNameOrId.ByValue ffiLabel = + ((List) op.getLabels().get().applyArg()).get(0); + Assert.assertEquals("person", ffiLabel.name); + } + + @Test + public void g_V_id_test() { + Traversal traversal = g.V(1L); + Step graphStep = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.GRAPH_STEP.apply(graphStep); + FfiConst.ByValue ffiId = ((List) op.getIds().get().applyArg()).get(0); + Assert.assertEquals(1L, ffiId.int64); + } + + @Test + public void g_V_property_test() { + Traversal traversal = g.V().has("name", "marko"); + IrStandardOpProcessor.applyStrategies(traversal); + Step graphStep = traversal.asAdmin().getStartStep(); + ScanFusionOp op = (ScanFusionOp) StepTransformFactory.SCAN_FUSION_STEP.apply(graphStep); + String expr = (String) op.getPredicate().get().applyArg(); + Assert.assertEquals("@.name && @.name == \"marko\"", expr); + } + + @Test + public void g_V_as_test() { + Traversal traversal = g.V().as("a"); + ScanFusionOp op = (ScanFusionOp) generateInterOpFromBuilder(traversal, 0); + FfiAlias.ByValue expected = ArgUtils.asFfiAlias("a", true); + Assert.assertEquals(expected, op.getAlias().get().applyArg()); + } + + @Test + public void g_E_as_test() { + Traversal traversal = g.E().as("a"); + ScanFusionOp op = (ScanFusionOp) generateInterOpFromBuilder(traversal, 0); + FfiAlias.ByValue expected = ArgUtils.asFfiAlias("a", true); + Assert.assertEquals(expected, op.getAlias().get().applyArg()); + } + + private InterOpBase generateInterOpFromBuilder(Traversal traversal, int idx) { + return (new InterOpCollectionBuilder(traversal)).build().unmodifiableCollection().get(idx); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/HasStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/HasStepTest.java new file mode 100644 index 000000000000..e2658c37eac2 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/HasStepTest.java @@ -0,0 +1,186 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.operator.SelectOp; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +public class HasStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_property_test() { + Traversal traversal = g.V().has("name", "marko"); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.name && @.name == \"marko\"", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_eq_test() { + Traversal traversal = g.V().has("name", P.eq("marko")); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.name && @.name == \"marko\"", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_neq_test() { + Traversal traversal = g.V().has("name", P.neq("marko")); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.name && @.name != \"marko\"", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_lt_test() { + Traversal traversal = g.V().has("age", P.lt(10)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age < 10", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_lte_test() { + Traversal traversal = g.V().has("age", P.lte(10)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age <= 10", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_gt_test() { + Traversal traversal = g.V().has("age", P.gt(10)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age > 10", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_gte_test() { + Traversal traversal = g.V().has("age", P.gte(10)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age >= 10", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_within_int_test() { + Traversal traversal = g.V().has("age", P.within(10)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age within [10]", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_within_ints_test() { + Traversal traversal = g.V().has("age", P.within(10, 11)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age within [10, 11]", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_without_int_test() { + Traversal traversal = g.V().has("age", P.without(10)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age without [10]", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_without_ints_test() { + Traversal traversal = g.V().has("age", P.without(10, 11)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals("@.age && @.age without [10, 11]", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_within_strs_test() { + Traversal traversal = g.V().has("name", P.within("marko", "josh")); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals( + "@.name && @.name within [\"marko\", \"josh\"]", + op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_without_strs_test() { + Traversal traversal = g.V().has("name", P.without("marko", "josh")); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals( + "@.name && @.name without [\"marko\", \"josh\"]", + op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_and_p_test() { + Traversal traversal = g.V().has("age", P.gt(27).and(P.lt(32))); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals( + "@.age && @.age > 27 && (@.age && @.age < 32)", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_or_p_test() { + Traversal traversal = g.V().has("age", P.lt(27).or(P.gt(32))); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals( + "@.age && @.age < 27 || (@.age && @.age > 32)", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_values_is_eq_test() { + Traversal traversal = g.V().values("age").is(P.eq(27)); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.IS_STEP.apply(hasStep); + Assert.assertEquals("@ == 27", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_values_is_and_p_test() { + Traversal traversal = g.V().values("age").is(P.gt(27).and(P.lt(32))); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.IS_STEP.apply(hasStep); + Assert.assertEquals("@ > 27 && (@ < 32)", op.getPredicate().get().applyArg()); + } + + @Test + public void g_V_has_has_test() { + Traversal traversal = g.V().has("name", "marko").has("id", 1); + Step hasStep = traversal.asAdmin().getEndStep(); + SelectOp op = (SelectOp) StepTransformFactory.HAS_STEP.apply(hasStep); + Assert.assertEquals( + "@.name && @.name == \"marko\" && (@.id && @.id == 1)", + op.getPredicate().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/LimitTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/LimitTest.java new file mode 100644 index 000000000000..b7406f90a836 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/LimitTest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.operator.LimitOp; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +public class LimitTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_limit_test() { + Traversal traversal = g.V().limit(1); + Step limitStep = traversal.asAdmin().getEndStep(); + LimitOp op = (LimitOp) StepTransformFactory.LIMIT_STEP.apply(limitStep); + Assert.assertEquals(2, op.getUpper().get().applyArg()); + } + + @Test + public void g_V_range_test() { + Traversal traversal = g.V().range(0, 1); + Step limitStep = traversal.asAdmin().getEndStep(); + LimitOp op = (LimitOp) StepTransformFactory.LIMIT_STEP.apply(limitStep); + Assert.assertEquals(2, op.getUpper().get().applyArg()); + Assert.assertEquals(1, op.getLower().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/MatchStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/MatchStepTest.java new file mode 100644 index 000000000000..0b3f7058131d --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/MatchStepTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.MatchSentence; +import com.alibaba.graphscope.common.intermediate.operator.MatchOp; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchStep; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class MatchStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getSentences(Traversal traversal) { + MatchStep matchStep = (MatchStep) traversal.asAdmin().getEndStep(); + MatchOp matchOp = (MatchOp) StepTransformFactory.MATCH_STEP.apply(matchStep); + return (List) matchOp.getSentences().get().applyArg(); + } + + private boolean isEqualWith( + MatchSentence sentence, + String expectedStart, + String expectedEnd, + FfiJoinKind expectedJoin, + int expectedBinderSize) { + return sentence.getStartTag().equals(ArgUtils.asFfiAlias(expectedStart, true)) + && sentence.getEndTag().equals(ArgUtils.asFfiAlias(expectedEnd, true)) + && sentence.getJoinKind() == expectedJoin + && sentence.getBinders().unmodifiableCollection().size() == expectedBinderSize; + } + + @Test + public void g_V_match_as_a_out_as_b_test() { + Traversal traversal = g.V().match(__.as("a").out().as("b")); + MatchSentence sentence = getSentences(traversal).get(0); + Assert.assertTrue(isEqualWith(sentence, "a", "b", FfiJoinKind.Inner, 1)); + } + + @Test + public void g_V_match_as_a_out_as_b_out_as_c_test() { + Traversal traversal = g.V().match(__.as("a").out().as("b"), __.as("b").out().as("c")); + MatchSentence sentence1 = getSentences(traversal).get(0); + MatchSentence sentence2 = getSentences(traversal).get(1); + Assert.assertTrue(isEqualWith(sentence1, "a", "b", FfiJoinKind.Inner, 1)); + Assert.assertTrue(isEqualWith(sentence2, "b", "c", FfiJoinKind.Inner, 1)); + } + + @Test + public void g_V_match_as_a_out_as_b_where_test() { + Traversal traversal = + g.V().match(__.as("a").out().as("b"), __.where(__.as("a").out().as("c"))); + MatchSentence sentence1 = getSentences(traversal).get(0); + MatchSentence sentence2 = getSentences(traversal).get(1); + Assert.assertTrue(isEqualWith(sentence1, "a", "b", FfiJoinKind.Inner, 1)); + Assert.assertTrue(isEqualWith(sentence2, "a", "c", FfiJoinKind.Semi, 1)); + } + + @Test + public void g_V_match_as_a_out_as_b_not_test() { + Traversal traversal = + g.V().match(__.as("a").out().as("b"), __.not(__.as("a").out().as("c"))); + MatchSentence sentence1 = getSentences(traversal).get(0); + MatchSentence sentence2 = getSentences(traversal).get(1); + Assert.assertTrue(isEqualWith(sentence1, "a", "b", FfiJoinKind.Inner, 1)); + Assert.assertTrue(isEqualWith(sentence2, "a", "c", FfiJoinKind.Anti, 1)); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/PathExpandStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/PathExpandStepTest.java new file mode 100644 index 000000000000..0de68d6f032b --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/PathExpandStepTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.operator.PathExpandOp; +import com.alibaba.graphscope.common.jna.type.FfiDirection; +import com.alibaba.graphscope.common.jna.type.FfiNameOrId; +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversalSource; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class PathExpandStepTest { + private Graph graph = TinkerFactory.createModern(); + private IrCustomizedTraversalSource g = graph.traversal(IrCustomizedTraversalSource.class); + + @Test + public void g_V_path_expand_test() { + Traversal traversal = g.V().out(__.range(1, 2)); + Step step = traversal.asAdmin().getEndStep(); + PathExpandOp op = (PathExpandOp) StepTransformFactory.PATH_EXPAND_STEP.apply(step); + + Assert.assertEquals(FfiDirection.Out, op.getDirection().get().applyArg()); + Assert.assertEquals(false, op.getIsEdge().get().applyArg()); + Assert.assertEquals(1, op.getLower().get().applyArg()); + Assert.assertEquals(2, op.getUpper().get().applyArg()); + } + + @Test + public void g_V_path_expand_label_test() { + Traversal traversal = g.V().out(__.range(1, 2), "knows"); + Step step = traversal.asAdmin().getEndStep(); + PathExpandOp op = (PathExpandOp) StepTransformFactory.PATH_EXPAND_STEP.apply(step); + + Assert.assertEquals(FfiDirection.Out, op.getDirection().get().applyArg()); + Assert.assertEquals(false, op.getIsEdge().get().applyArg()); + Assert.assertEquals(1, op.getLower().get().applyArg()); + Assert.assertEquals(2, op.getUpper().get().applyArg()); + FfiNameOrId.ByValue label = + ((List) op.getLabels().get().applyArg()).get(0); + Assert.assertEquals("knows", label.name); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/ValueMapTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/ValueMapTest.java new file mode 100644 index 000000000000..2a056a852b01 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/ValueMapTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.operator.ProjectOp; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class ValueMapTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_valueMap_strs_test() { + Traversal traversal = g.V().valueMap("name", "id"); + Step valueMapStep = traversal.asAdmin().getEndStep(); + ProjectOp op = (ProjectOp) StepTransformFactory.VALUE_MAP_STEP.apply(valueMapStep); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("{@.name, @.id}", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } + + @Test + public void g_V_values_str_test() { + Traversal traversal = g.V().values("name"); + Step valueMapStep = traversal.asAdmin().getEndStep(); + ProjectOp op = (ProjectOp) StepTransformFactory.VALUES_STEP.apply(valueMapStep); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("@.name", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/VertexStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/VertexStepTest.java new file mode 100644 index 000000000000..13c58416f4db --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/VertexStepTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin; + +import com.alibaba.graphscope.common.intermediate.operator.ExpandOp; +import com.alibaba.graphscope.common.jna.type.FfiDirection; +import com.alibaba.graphscope.common.jna.type.FfiNameOrId; +import com.alibaba.graphscope.gremlin.transform.StepTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class VertexStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + @Test + public void g_V_out() { + Traversal traversal = g.V().out(); + Step vertexStep = traversal.asAdmin().getEndStep(); + ExpandOp op = (ExpandOp) StepTransformFactory.VERTEX_STEP.apply(vertexStep); + Assert.assertEquals(FfiDirection.Out, op.getDirection().get().applyArg()); + Assert.assertEquals(false, op.getIsEdge().get().applyArg()); + } + + @Test + public void g_V_outE() { + Traversal traversal = g.V().outE(); + Step vertexStep = traversal.asAdmin().getEndStep(); + ExpandOp op = (ExpandOp) StepTransformFactory.VERTEX_STEP.apply(vertexStep); + Assert.assertEquals(FfiDirection.Out, op.getDirection().get().applyArg()); + Assert.assertEquals(true, op.getIsEdge().get().applyArg()); + } + + @Test + public void g_V_out_label() { + Traversal traversal = g.V().out("knows"); + Step vertexStep = traversal.asAdmin().getEndStep(); + ExpandOp op = (ExpandOp) StepTransformFactory.VERTEX_STEP.apply(vertexStep); + FfiNameOrId.ByValue label = + ((List) op.getLabels().get().applyArg()).get(0); + Assert.assertEquals("knows", label.name); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4/NegativeEvalTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4/NegativeEvalTest.java new file mode 100644 index 000000000000..aabcccab6d97 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4/NegativeEvalTest.java @@ -0,0 +1,236 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.exception.InvalidGremlinScriptException; +import com.alibaba.graphscope.gremlin.plugin.script.AntlrToJavaScriptEngine; +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversalSource; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.SimpleBindings; +import javax.script.SimpleScriptContext; + +public class NegativeEvalTest { + private AntlrToJavaScriptEngine scriptEngine; + private ScriptContext context; + + @Before + public void before() { + Graph graph = TinkerFactory.createModern(); + GraphTraversalSource g = graph.traversal(IrCustomizedTraversalSource.class); + Bindings globalBindings = new SimpleBindings(); + globalBindings.put("g", g); + context = new SimpleScriptContext(); + context.setBindings(globalBindings, ScriptContext.ENGINE_SCOPE); + scriptEngine = new AntlrToJavaScriptEngine(); + } + + @Test + public void g111_test() { + try { + scriptEngine.eval("g111()", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V111_test() { + try { + scriptEngine.eval("g.V111()", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_outX_test() { + try { + scriptEngine.eval("g.V().outX()", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_out_id_test() { + try { + scriptEngine.eval("g.V().out(1)", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_out_limit_str_test() { + try { + scriptEngine.eval("g.V().out().limit('xxx')", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_has_key_id_test() { + try { + scriptEngine.eval("g.V().has(1, 1)", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_bothV_test() { + try { + scriptEngine.eval("g.V().bothV()", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_valueMap_int_test() { + try { + scriptEngine.eval("g.V().valueMap(1)", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_select_none_test() { + try { + scriptEngine.eval("g.V().select()", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_select_by_keys_test() { + try { + scriptEngine.eval("g.V().select().by('name', 'id')", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_order_by_orders_test() { + try { + scriptEngine.eval("g.V().order().by(asc, desc)", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_order_by_keys_test() { + try { + scriptEngine.eval("g.V().order().by('name', 'id')", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_valueMap_none_test() { + try { + scriptEngine.eval("g.V().valueMap()", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_as_invalid_test() { + try { + scriptEngine.eval("g.V().as('a', 'b')", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_str_id_test() { + try { + scriptEngine.eval("g.V(\"1\")", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_E_str_id_test() { + try { + scriptEngine.eval("g.E(\"1\")", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } + + @Test + public void g_V_hasId_str_id_test() { + try { + scriptEngine.eval("g.V().hasId(\"1\")", context); + } catch (InvalidGremlinScriptException e) { + // expected error + return; + } + Assert.fail(); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4/PositiveEvalTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4/PositiveEvalTest.java new file mode 100644 index 000000000000..4c07426c2eb6 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/antlr4/PositiveEvalTest.java @@ -0,0 +1,869 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.antlr4; + +import com.alibaba.graphscope.gremlin.plugin.script.AntlrToJavaScriptEngine; +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversal; +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversalSource; + +import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.NotStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.SimpleBindings; +import javax.script.SimpleScriptContext; + +public class PositiveEvalTest { + private Graph graph; + private GraphTraversalSource g; + private AntlrToJavaScriptEngine scriptEngine; + private ScriptContext context; + + @Before + public void before() { + graph = TinkerFactory.createModern(); + g = graph.traversal(IrCustomizedTraversalSource.class); + Bindings globalBindings = new SimpleBindings(); + globalBindings.put("g", g); + context = new SimpleScriptContext(); + context.setBindings(globalBindings, ScriptContext.ENGINE_SCOPE); + scriptEngine = new AntlrToJavaScriptEngine(); + } + + private Object eval(String query) { + return scriptEngine.eval(query, context); + } + + @Test + public void eval_g_V_test() { + Assert.assertEquals(g.V(), eval("g.V()")); + } + + @Test + public void eval_g_E_test() { + Assert.assertEquals(g.E(), eval("g.E()")); + } + + // expand + @Test + public void eval_g_V_out_test() { + Assert.assertEquals(g.V().out(), eval("g.V().out()")); + } + + @Test + public void eval_g_V_outE_test() { + Assert.assertEquals(g.V().outE(), eval("g.V().outE()")); + } + + @Test + public void eval_g_V_in_test() { + Assert.assertEquals(g.V().in(), eval("g.V().in()")); + } + + @Test + public void eval_g_V_inE_test() { + Assert.assertEquals(g.V().inE(), eval("g.V().inE()")); + } + + @Test + public void eval_g_V_both_test() { + Assert.assertEquals(g.V().both(), eval("g.V().both()")); + } + + @Test + public void eval_g_V_bothE_test() { + Assert.assertEquals(g.V().bothE(), eval("g.V().bothE()")); + } + + @Test + public void eval_g_V_outE_label_test() { + Assert.assertEquals(g.V().outE("knows"), eval("g.V().outE('knows')")); + } + + @Test + public void eval_g_V_out_labels_test() { + Assert.assertEquals(g.V().out("knows", "has"), eval("g.V().out('knows', 'has')")); + } + + @Test + public void eval_g_V_outE_labels_test() { + Assert.assertEquals(g.V().outE("knows", "has"), eval("g.V().outE('knows', 'has')")); + } + + @Test + public void eval_g_V_in_labels_test() { + Assert.assertEquals(g.V().in("knows", "has"), eval("g.V().in('knows', 'has')")); + } + + @Test + public void eval_g_V_inE_labels_test() { + Assert.assertEquals(g.V().inE("knows", "has"), eval("g.V().inE('knows', 'has')")); + } + + @Test + public void eval_g_V_both_labels_test() { + Assert.assertEquals(g.V().both("knows", "has"), eval("g.V().both('knows', 'has')")); + } + + @Test + public void eval_g_V_bothE_labels_test() { + Assert.assertEquals(g.V().bothE("knows", "has"), eval("g.V().bothE('knows', 'has')")); + } + + @Test + public void eval_g_V_outE_label_inV_test() { + Assert.assertEquals(g.V().out("knows"), eval("g.V().outE('knows').inV()")); + } + + @Test + public void eval_g_V_inE_label_outV_test() { + Assert.assertEquals(g.V().in("knows"), eval("g.V().inE('knows').outV()")); + } + + @Test + public void eval_g_V_bothE_label_otherV_test() { + Assert.assertEquals(g.V().both("knows"), eval("g.V().bothE('knows').otherV()")); + } + + // has + + @Test + public void g_V_hasLabel_test() { + Assert.assertEquals(g.V().hasLabel("person"), eval("g.V().hasLabel('person')")); + } + + @Test + public void g_V_hasLabels_test() { + Assert.assertEquals( + g.V().hasLabel("person", "software"), eval("g.V().hasLabel('person', 'software')")); + } + + @Test + public void g_V_hasId_test() { + Assert.assertEquals(g.V().hasId(1), eval("g.V().hasId(1)")); + } + + @Test + public void g_V_hasIds_test() { + Assert.assertEquals(g.V().hasId(1, 2), eval("g.V().hasId(1, 2)")); + } + + @Test + public void g_V_has_name_str_test() { + Assert.assertEquals(g.V().has("name", "marko"), eval("g.V().has('name', 'marko')")); + } + + @Test + public void g_V_has_age_int_test() { + Assert.assertEquals(g.V().has("age", 10), eval("g.V().has('age', 10)")); + } + + // predicate + + @Test + public void g_V_has_age_P_eq_test() { + Assert.assertEquals(g.V().has("age", P.eq(10)), eval("g.V().has('age', eq(10))")); + } + + @Test + public void g_V_has_age_P_eq_1_test() { + Assert.assertEquals(g.V().has("age", P.eq(10)), eval("g.V().has('age', P.eq(10))")); + } + + @Test + public void g_V_has_age_P_neq_test() { + Assert.assertEquals(g.V().has("age", P.neq(10)), eval("g.V().has('age', neq(10))")); + } + + @Test + public void g_V_has_age_P_lt_test() { + Assert.assertEquals(g.V().has("age", P.lt(10)), eval("g.V().has('age', lt(10))")); + } + + @Test + public void g_V_has_age_P_lte_test() { + Assert.assertEquals(g.V().has("age", P.lte(10)), eval("g.V().has('age', lte(10))")); + } + + @Test + public void g_V_has_age_P_gt_test() { + Assert.assertEquals(g.V().has("age", P.gt(10)), eval("g.V().has('age', gt(10))")); + } + + @Test + public void g_V_has_age_P_gte_test() { + Assert.assertEquals(g.V().has("age", P.gte(10)), eval("g.V().has('age', gte(10))")); + } + + // limit + + @Test + public void g_V_limit_test() { + Assert.assertEquals(g.V().limit(1), eval("g.V().limit(1)")); + } + + @Test + public void g_V_hasLabel_limit_test() { + Assert.assertEquals(g.V().limit(1), eval("g.V().limit(1)")); + } + + @Test + public void g_V_limit_hasLabel_test() { + Assert.assertEquals( + g.V().hasLabel("person").limit(1), eval("g.V().hasLabel('person').limit(1)")); + } + + @Test + public void g_V_out_limit_test() { + Assert.assertEquals(g.V().out().limit(1), eval("g.V().out().limit(1)")); + } + + @Test + public void g_V_hasLabel_out_limit_test() { + Assert.assertEquals( + g.V().hasLabel("person").out().limit(1), + eval("g.V().hasLabel('person').out().limit(1)")); + } + + @Test + public void g_V_limit_out_out_limit_test() { + Assert.assertEquals( + g.V().limit(1).out().out().limit(1), eval("g.V().limit(1).out().out().limit(1)")); + } + + @Test + public void g_V_valueMap_str_test() { + Assert.assertEquals(g.V().valueMap("name"), eval("g.V().valueMap('name')")); + } + + @Test + public void g_V_valueMap_strs_test() { + Assert.assertEquals(g.V().valueMap("name", "id"), eval("g.V().valueMap('name', 'id')")); + } + + @Test + public void g_V_select_a_test() { + Traversal expected = g.V().as("a").select("a"); + Assert.assertEquals(expected, eval("g.V().as(\"a\").select(\"a\")")); + } + + @Test + public void g_V_select_a_by_str_test() { + Traversal expected = g.V().as("a").select("a").by("name"); + Assert.assertEquals(expected, eval("g.V().as(\"a\").select(\"a\").by(\"name\")")); + } + + @Test + public void g_V_select_a_by_valueMap_key_test() { + Traversal expected = g.V().as("a").select("a").by(__.valueMap("name")); + Assert.assertEquals(expected, eval("g.V().as(\"a\").select(\"a\").by(valueMap(\"name\"))")); + } + + @Test + public void g_V_select_a_by_valueMap_key_1_test() { + Traversal expected = g.V().as("a").select("a").by(__.valueMap("name")); + Assert.assertEquals( + expected, eval("g.V().as(\"a\").select(\"a\").by(__.valueMap(\"name\"))")); + } + + @Test + public void g_V_select_a_by_valueMap_keys_test() { + Traversal expected = g.V().as("a").select("a").by(__.valueMap("name", "id")); + Assert.assertEquals( + expected, eval("g.V().as(\"a\").select(\"a\").by(valueMap(\"name\", \"id\"))")); + } + + @Test + public void g_V_select_a_b_test() { + Assert.assertEquals( + g.V().as("a").out().as("b").select("a", "b"), + eval("g.V().as(\"a\").out().as(\"b\").select(\"a\", \"b\")")); + } + + @Test + public void g_V_select_a_b_by_strs_test() { + Assert.assertEquals( + g.V().as("a").out().as("b").select("a", "b").by("name").by("id"), + eval( + "g.V().as(\"a\").out().as(\"b\").select(\"a\"," + + " \"b\").by(\"name\").by(\"id\")")); + } + + @Test + public void g_V_select_a_b_c_test() { + Assert.assertEquals( + g.V().as("a").out().as("b").out().as("c").select("a", "b", "c"), + eval( + "g.V().as(\"a\").out().as(\"b\").out().as(\"c\").select(\"a\", \"b\"," + + " \"c\")")); + } + + @Test + public void g_V_select_a_b_by_str_test() { + Assert.assertEquals( + g.V().as("a").out().as("b").select("a", "b").by("name"), + eval("g.V().as(\"a\").out().as(\"b\").select(\"a\", \"b\").by(\"name\")")); + } + + @Test + public void g_V_select_a_b_by_valueMap_key_test() { + Assert.assertEquals( + g.V().as("a").out().as("b").select("a", "b").by(__.valueMap("name")), + eval( + "g.V().as(\"a\").out().as(\"b\").select(\"a\"," + + " \"b\").by(valueMap(\"name\"))")); + } + + @Test + public void g_V_select_a_b_by_valueMap_keys_test() { + Assert.assertEquals( + g.V().as("a") + .out() + .as("b") + .select("a", "b") + .by(__.valueMap("name", "id")) + .by(__.valueMap("age")), + eval( + "g.V().as(\"a\").out().as(\"b\").select(\"a\", \"b\").by(valueMap(\"name\"," + + " \"id\")).by(valueMap(\"age\"))")); + } + + @Test + public void g_V_order_test() { + Assert.assertEquals(g.V().order(), eval("g.V().order()")); + } + + @Test + public void g_V_order_by_asc_test() { + Assert.assertEquals(g.V().order().by(Order.asc), eval("g.V().order().by(asc)")); + } + + @Test + public void g_V_order_by_desc_test() { + Assert.assertEquals(g.V().order().by(Order.desc), eval("g.V().order().by(desc)")); + } + + @Test + public void g_V_order_by_key_asc_test() { + Assert.assertEquals( + g.V().order().by("name", Order.asc), eval("g.V().order().by('name', asc)")); + } + + @Test + public void g_V_order_by_key_desc_test() { + Assert.assertEquals( + g.V().order().by("name", Order.desc), eval("g.V().order().by('name', desc)")); + } + + @Test + public void g_V_order_by_keys_test() { + Assert.assertEquals( + g.V().order().by("name", Order.desc).by("id", Order.asc), + eval("g.V().order().by('name', desc).by('id', asc)")); + } + + @Test + public void g_V_order_by_key_test() { + Assert.assertEquals(g.V().order().by("name"), eval("g.V().order().by(\"name\")")); + } + + @Test + public void g_V_order_by_values_test() { + Assert.assertEquals( + g.V().order().by(__.values("name")), eval("g.V().order().by(__.values(\"name\"))")); + } + + @Test + public void g_V_order_by_select_test() { + Assert.assertEquals( + g.V().as("a").order().by(__.select("a")), + eval("g.V().as(\"a\").order().by(__.select(\"a\"))")); + } + + @Test + public void g_V_order_by_select_asc_test() { + Assert.assertEquals( + g.V().as("a").order().by(__.select("a"), Order.asc), + eval("g.V().as(\"a\").order().by(__.select(\"a\"), asc)")); + } + + @Test + public void g_V_has_property_test() { + Assert.assertEquals(g.V().has("name", "marko"), eval("g.V().has('name', 'marko')")); + } + + @Test + public void g_V_has_eq_test() { + Assert.assertEquals( + g.V().has("name", P.eq("marko")), eval("g.V().has('name', eq('marko'))")); + } + + @Test + public void g_V_has_neq_test() { + Assert.assertEquals( + g.V().has("name", P.neq("marko")), eval("g.V().has('name', neq('marko'))")); + } + + @Test + public void g_V_has_lt_test() { + Assert.assertEquals(g.V().has("age", P.lt(10)), eval("g.V().has('age', lt(10))")); + } + + @Test + public void g_V_has_lte_test() { + Assert.assertEquals(g.V().has("age", P.lte(10)), eval("g.V().has('age', lte(10))")); + } + + @Test + public void g_V_has_gt_test() { + Assert.assertEquals(g.V().has("age", P.gt(10)), eval("g.V().has('age', gt(10))")); + } + + @Test + public void g_V_has_gte_test() { + Assert.assertEquals(g.V().has("age", P.gte(10)), eval("g.V().has('age', gte(10))")); + } + + @Test + public void g_V_has_within_int_test() { + Assert.assertEquals(g.V().has("age", P.within(10)), eval("g.V().has('age', within(10))")); + } + + @Test + public void g_V_has_within_ints_test() { + Assert.assertEquals( + g.V().has("age", P.within(10, 11)), eval("g.V().has('age', within(10, 11))")); + } + + @Test + public void g_V_has_within_ints_1_test() { + Assert.assertEquals( + g.V().has("age", P.within(10, 11)), eval("g.V().has('age', within([10, 11]))")); + } + + @Test + public void g_V_has_without_int_test() { + Assert.assertEquals(g.V().has("age", P.without(10)), eval("g.V().has('age', without(10))")); + } + + @Test + public void g_V_has_without_ints_test() { + Assert.assertEquals( + g.V().has("age", P.without(10, 11)), eval("g.V().has('age', without(10, 11))")); + } + + @Test + public void g_V_has_within_strs_test() { + Assert.assertEquals( + g.V().has("name", P.within("marko", "josh")), + eval("g.V().has('name', within('marko', 'josh'))")); + } + + @Test + public void g_V_has_without_strs_test() { + Assert.assertEquals( + g.V().has("name", P.without("marko", "josh")), + eval("g.V().has('name', without('marko', 'josh'))")); + } + + @Test + public void g_V_has_and_p_test() { + Assert.assertEquals( + g.V().has("age", P.gt(25).and(P.lt(32))), + eval("g.V().has(\"age\", P.gt(25).and(P.lt(32)))")); + } + + @Test + public void g_V_has_or_p_test() { + Assert.assertEquals( + g.V().has("age", P.gt(25).or(P.lt(32))), + eval("g.V().has(\"age\", P.gt(25).or(P.lt(32)))")); + } + + @Test + public void g_V_has_label_property_test() { + Assert.assertEquals( + g.V().has("person", "age", 25), eval("g.V().has(\"person\", \"age\", 25)")); + } + + @Test + public void g_V_has_label_property_eq_test() { + Assert.assertEquals( + g.V().has("person", "age", P.eq(25)), + eval("g.V().has(\"person\", \"age\", P.eq(25))")); + } + + @Test + public void g_V_values_is_val_test() { + Assert.assertEquals(g.V().values("name").is(27), eval("g.V().values(\"name\").is(27)")); + } + + @Test + public void g_V_values_is_p_test() { + Assert.assertEquals( + g.V().values("name").is(P.eq(27)), eval("g.V().values(\"name\").is(P.eq(27))")); + } + + @Test + public void g_V_as_test() { + Assert.assertEquals(g.V().as("a"), eval("g.V().as('a')")); + } + + @Test + public void g_V_out_as_test() { + Assert.assertEquals(g.V().out().as("abc"), eval("g.V().out().as('abc'))")); + } + + @Test + public void g_V_has_as_test() { + Assert.assertEquals( + g.V().has("name", P.without("marko", "josh")).as("c"), + eval("g.V().has('name', without('marko', 'josh')).as('c')")); + } + + @Test + public void g_V_group_test() { + Assert.assertEquals(g.V().group(), eval("g.V().group()")); + } + + @Test + public void g_V_group_by_key_str_test() { + Assert.assertEquals(g.V().group().by("name"), eval("g.V().group().by(\"name\")")); + } + + @Test + public void g_V_group_by_key_values_test() { + Assert.assertEquals( + g.V().group().by(__.values("name")), eval("g.V().group().by(values(\"name\"))")); + } + + @Test + public void g_V_group_by_key_values_1_test() { + Assert.assertEquals( + g.V().group().by(__.values("name")), eval("g.V().group().by(__.values(\"name\"))")); + } + + @Test + public void g_V_group_by_key_values_as_test() { + Assert.assertEquals( + g.V().group().by(__.values("name").as("name")), + eval("g.V().group().by(values(\"name\").as(\"name\"))")); + } + + @Test + public void g_V_group_by_key_str_by_value_count_test() { + Assert.assertEquals( + g.V().group().by("name").by(__.count()), + eval("g.V().group().by(\"name\").by(count())")); + } + + @Test + public void g_V_group_by_key_values_by_value_fold_test() { + Assert.assertEquals( + g.V().group().by(__.values("name")).by(__.fold()), + eval("g.V().group().by(values(\"name\")).by(fold())")); + } + + @Test + public void g_V_group_by_key_values_by_value_fold_1_test() { + Assert.assertEquals( + g.V().group().by(__.values("name")).by(__.fold()), + eval("g.V().group().by(values(\"name\")).by(__.fold())")); + } + + @Test + public void g_V_group_by_key_str_by_value_count_as_test() { + Assert.assertEquals( + g.V().group().by("name").by(__.count().as("a")), + eval("g.V().group().by(\"name\").by(count().as(\"a\"))")); + } + + @Test + public void g_V_group_by_key_values_by_value_fold_as_test() { + Assert.assertEquals( + g.V().group().by(__.values("name")).by(__.fold().as("a")), + eval("g.V().group().by(values(\"name\")).by(fold().as(\"a\"))")); + } + + @Test + public void g_V_groupCount_test() { + Assert.assertEquals(g.V().groupCount(), eval("g.V().groupCount()")); + } + + @Test + public void g_V_groupCount_by_str_test() { + Assert.assertEquals(g.V().groupCount().by("name"), eval("g.V().groupCount().by(\"name\")")); + } + + @Test + public void g_V_groupCount_by_values_test() { + Assert.assertEquals( + g.V().groupCount().by(__.values("name")), + eval("g.V().groupCount().by(values(\"name\"))")); + } + + @Test + public void g_V_groupCount_by_values_as_test() { + Assert.assertEquals( + g.V().groupCount().by(__.values("name").as("a")), + eval("g.V().groupCount().by(__.values(\"name\").as(\"a\"))")); + } + + @Test + public void g_V_dedup_test() { + Assert.assertEquals(g.V().dedup(), eval("g.V().dedup()")); + } + + @Test + public void g_V_count_test() { + Assert.assertEquals(g.V().count(), eval("g.V().count()")); + } + + @Test + public void g_V_count_as_test() { + Assert.assertEquals(g.V().count().as("a"), eval("g.V().count().as(\"a\")")); + } + + @Test + public void g_V_values_test() { + Assert.assertEquals(g.V().values("name"), eval("g.V().values(\"name\")")); + } + + @Test + public void g_V_where_eq_a_test() { + Assert.assertEquals(g.V().where(P.eq("a")), eval("g.V().where(P.eq(\"a\"))")); + } + + @Test + public void g_V_where_eq_a_age_test() { + Assert.assertEquals( + g.V().where(P.eq("a")).by("age"), eval("g.V().where(P.eq(\"a\")).by(\"age\")")); + } + + @Test + public void g_V_where_a_eq_b_test() { + Assert.assertEquals(g.V().where("a", P.eq("b")), eval("g.V().where(\"a\", P.eq(\"b\"))")); + } + + @Test + public void g_V_where_a_eq_b_age_test() { + Assert.assertEquals( + g.V().where("a", P.eq("b")).by("age"), + eval("g.V().where(\"a\", P.eq(\"b\")).by(\"age\")")); + } + + @Test + public void g_V_where_a_eq_b_value_age_test() { + Assert.assertEquals( + g.V().where("a", P.eq("b")).by("age").by(__.values("age")), + eval("g.V().where(\"a\", P.eq(\"b\")).by(\"age\").by(values(\"age\"))")); + } + + @Test + public void g_V_where_a_eq_b_nested_value_age_test() { + Assert.assertEquals( + g.V().where("a", P.eq("b")).by("age").by(__.values("age")), + eval("g.V().where(\"a\", P.eq(\"b\")).by(\"age\").by(__.values(\"age\"))")); + } + + @Test + public void g_V_path_expand_test() { + Assert.assertEquals( + ((IrCustomizedTraversalSource) g).V().out(__.range(1, 2)), + eval("g.V().out('1..2')")); + } + + @Test + public void g_V_path_expand_label_test() { + Assert.assertEquals( + ((IrCustomizedTraversalSource) g).V().out(__.range(1, 2), "knows"), + eval("g.V().out('1..2', \"knows\"))")); + } + + @Test + public void g_V_path_expand_labels_test() { + Assert.assertEquals( + ((IrCustomizedTraversalSource) g).V().out(__.range(1, 2), "knows", "person"), + eval("g.V().out(\"1..2\", \"knows\", \"person\"))")); + } + + @Test + public void g_V_outE_as_otherV_test() { + Assert.assertEquals(g.V().outE().as("a").otherV(), eval("g.V().outE().as(\"a\").otherV()")); + } + + @Test + public void g_V_outE_as_inV_test() { + Assert.assertEquals(g.V().outE().as("a").inV(), eval("g.V().outE().as(\"a\").inV()")); + } + + @Test + public void g_V_outE_as_outV_test() { + Assert.assertEquals(g.V().outE().as("a").outV(), eval("g.V().outE().as(\"a\").outV()")); + } + + @Test + public void g_V_where_out_out_test() { + Assert.assertEquals(g.V().where(__.out().out()), eval("g.V().where(__.out().out())")); + } + + @Test + public void g_V_where_as_out_test() { + Assert.assertEquals( + g.V().as("a").where(__.as("a").out().out()), + eval("g.V().as(\"a\").where(__.as(\"a\").out().out())")); + } + + @Test + public void g_V_where_out_as_test() { + Assert.assertEquals( + g.V().as("a").where(__.out().out().as("a")), + eval("g.V().as(\"a\").where(__.out().out().as(\"a\"))")); + } + + @Test + public void g_V_where_not_test() { + Traversal.Admin traversal = (Traversal.Admin) eval("g.V().where(__.not(__.out()))"); + Assert.assertEquals(NotStep.class, traversal.getEndStep().getClass()); + } + + @Test + public void g_V_not_out_test() { + Assert.assertEquals(g.V().not(__.out().out()), eval("g.V().not(__.out().out())")); + } + + @Test + public void g_V_not_values_test() { + Assert.assertEquals(g.V().not(__.values("name")), eval("g.V().not(__.values(\"name\"))")); + } + + @Test + public void g_V_select_by_none_test() { + Assert.assertEquals( + g.V().as("a").select("a").by(), eval("g.V().as(\"a\").select(\"a\").by()")); + } + + @Test + public void g_V_order_by_none_test() { + Assert.assertEquals(g.V().order().by(), eval("g.V().order().by()")); + } + + @Test + public void g_V_where_by_none_test() { + Assert.assertEquals( + g.V().as("a").where(P.eq("a")).by(), + eval("g.V().as(\"a\").where(P.eq(\"a\")).by()")); + } + + @Test + public void g_V_group_key_by_none_value_by_none_test() { + Assert.assertEquals(g.V().group().by().by(), eval("g.V().group().by().by()")); + } + + @Test + public void g_V_union_test() { + Assert.assertEquals(g.V().union(__.out()), eval("g.V().union(__.out())")); + } + + @Test + public void g_V_union_out_test() { + Assert.assertEquals( + g.V().union(__.out(), __.out()), eval("g.V().union(__.out(), __.out())")); + } + + @Test + public void g_V_id_list_test() { + Assert.assertEquals(g.V(1, 2, 3), eval("g.V([1, 2, 3])")); + } + + // subtask + @Test + public void g_V_select_a_by_out_count_test() { + Assert.assertEquals( + g.V().as("a").select("a").by(__.out().count()), + eval("g.V().as(\"a\").select(\"a\").by(__.out().count())")); + } + + @Test + public void g_V_select_a_b_by_out_count_by_count_test() { + Assert.assertEquals( + g.V().as("a").select("a", "b").by(__.out().count()).by(__.count()), + eval("g.V().as(\"a\").select(\"a\", \"b\").by(__.out().count()).by(__.count())")); + } + + @Test + public void g_V_order_by_out_count_test() { + Assert.assertEquals( + g.V().order().by(__.out().count()), eval("g.V().order().by(__.out().count())")); + } + + @Test + public void g_V_order_by_out_count_by_count_test() { + Assert.assertEquals( + g.V().order().by(__.out().count()).by(__.count(), Order.desc), + eval("g.V().order().by(__.out().count()).by(__.count(), Order.desc)")); + } + + @Test + public void g_V_group_by_out_count_test() { + Assert.assertEquals( + g.V().group().by(__.out().count()), eval("g.V().group().by(__.out().count())")); + } + + @Test + public void g_V_groupCount_by_out_count_test() { + Assert.assertEquals( + g.V().groupCount().by(__.out().count()), + eval("g.V().groupCount().by(__.out().count())")); + } + + @Test + public void g_V_where_by_out_count() { + Assert.assertEquals( + g.V().as("a").out().as("b").where("a", P.eq("b")).by(__.out().count()), + eval( + "g.V().as(\"a\").out().as(\"b\").where(\"a\"," + + " P.eq(\"b\")).by(__.out().count())")); + } + + @Test + public void g_V_match_test() { + Assert.assertEquals( + g.V().match(__.as("a").out().as("b"), __.as("b").out().as("c")), + eval("g.V().match(__.as(\"a\").out().as(\"b\"), __.as(\"b\").out().as(\"c\"))")); + } + + @Test + public void g_V_out_endV_test() { + IrCustomizedTraversalSource g1 = (IrCustomizedTraversalSource) g; + Traversal traversal = ((IrCustomizedTraversal) (g1.V().out("1..5"))).endV(); + EdgeVertexStep endVStep = (EdgeVertexStep) traversal.asAdmin().getEndStep(); + Assert.assertEquals(Direction.IN, endVStep.getDirection()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/GroupStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/GroupStepTest.java new file mode 100644 index 000000000000..b00e352b7258 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/GroupStepTest.java @@ -0,0 +1,302 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.subtask; + +import com.alibaba.graphscope.common.intermediate.ArgAggFn; +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.GroupOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.jna.type.FfiAggOpt; +import com.alibaba.graphscope.common.jna.type.FfiAlias; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.common.jna.type.FfiVariable; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.__; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +public class GroupStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getApplyWithGroup(Traversal traversal) { + TraversalParent parent = (TraversalParent) traversal.asAdmin().getEndStep(); + return TraversalParentTransformFactory.GROUP_BY_STEP.apply(parent); + } + + @Test + public void g_V_group_test() { + Traversal traversal = g.V().group(); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiNoneVar(), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_key_test() { + Traversal traversal = g.V().group().by("name"); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_values_test() { + Traversal traversal = g.V().group().by(__.values("name")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_values_as_test() { + Traversal traversal = g.V().group().by(__.values("name").as("a")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("a", true)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_key_by_count_test() { + Traversal traversal = g.V().group().by("name").by(__.count()); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_key_by_count_as_test() { + Traversal traversal = g.V().group().by("name").by(__.count().as("b")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("b", true)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_key_by_fold_test() { + Traversal traversal = g.V().group().by("name").by(__.fold()); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_key_by_fold_as_test() { + Traversal traversal = g.V().group().by("name").by(__.fold().as("b")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("b", true)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_a_key_by_fold_as_test() { + Traversal traversal = + g.V().as("a").group().by(__.select("a").by("name")).by(__.fold().as("b")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("a", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("b", true)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_groupCount_test() { + Traversal traversal = g.V().groupCount(); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiNoneVar(), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_groupCount_by_key_test() { + Traversal traversal = g.V().groupCount().by("name"); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_groupCount_by_values_test() { + Traversal traversal = g.V().groupCount().by(__.values("name")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_groupCount_by_values_as_test() { + Traversal traversal = g.V().groupCount().by(__.values("name").as("a")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("", "name"), ArgUtils.asFfiAlias("a", true)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_groupCount_by_a_key_test() { + Traversal traversal = g.V().as("a").groupCount().by(__.select("a").by("name")); + GroupOp op = (GroupOp) getApplyWithGroup(traversal).get(0); + + Pair expectedKey = + Pair.with(ArgUtils.asFfiVar("a", "name"), ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.Count, ArgUtils.asFfiAlias("~values_1_0", false)); + + Assert.assertEquals( + Collections.singletonList(expectedKey), op.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), op.getGroupByValues().get().applyArg()); + } + + @Test + public void g_V_group_by_out_count_test() { + Traversal traversal = g.V().group().by(__.out().count()); + List ops = getApplyWithGroup(traversal); + Assert.assertEquals(2, ops.size()); + + ApplyOp applyOp = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(2, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~keys_1_0", false), applyOp.getAlias().get().applyArg()); + + GroupOp groupOp = (GroupOp) ops.get(1); + Pair expectedKey = + Pair.with( + ArgUtils.asFfiVar("~keys_1_0", ""), + ArgUtils.asFfiAlias("~keys_1_0", false)); + ArgAggFn expectedValue = + new ArgAggFn(FfiAggOpt.ToList, ArgUtils.asFfiAlias("~values_1_0", false)); + Assert.assertEquals( + Collections.singletonList(expectedKey), groupOp.getGroupByKeys().get().applyArg()); + Assert.assertEquals( + Collections.singletonList(expectedValue), + groupOp.getGroupByValues().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/NotTraversalTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/NotTraversalTest.java new file mode 100644 index 000000000000..816f45b6dfd6 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/NotTraversalTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.subtask; + +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.SelectOp; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class NotTraversalTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getApplyOrSelect(Traversal traversal) { + TraversalParent parent = (TraversalParent) traversal.asAdmin().getEndStep(); + return TraversalParentTransformFactory.NOT_TRAVERSAL_STEP.apply(parent); + } + + @Test + public void g_V_not_values() { + Traversal traversal = g.V().not(__.values("name")); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("!@.name", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_not_a() { + Traversal traversal = g.V().as("a").not(__.select("a")); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("!@a", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_not_a_values() { + Traversal traversal = g.V().as("a").not(__.select("a").by(__.values("name"))); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("!@a.name", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_not_out_out() { + Traversal traversal = g.V().not(__.out().out()); + ApplyOp applyOp = (ApplyOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals(FfiJoinKind.Anti, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(2, subOps.unmodifiableCollection().size()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/OrderStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/OrderStepTest.java new file mode 100644 index 000000000000..8424b931bc3a --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/OrderStepTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.subtask; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.OrderOp; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.common.jna.type.FfiOrderOpt; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +public class OrderStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getApplyWithOrder(Traversal traversal) { + TraversalParent parent = (TraversalParent) traversal.asAdmin().getEndStep(); + return TraversalParentTransformFactory.ORDER_BY_STEP.apply(parent); + } + + @Test + public void g_V_order_test() { + Traversal traversal = g.V().order(); + OrderOp op = (OrderOp) getApplyWithOrder(traversal).get(0); + + List expected = Arrays.asList(Pair.with(ArgUtils.asFfiNoneVar(), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, op.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_label_test() { + Traversal traversal = g.V().order().by("~label"); + OrderOp op = (OrderOp) getApplyWithOrder(traversal).get(0); + + List expected = + Arrays.asList(Pair.with(ArgUtils.asFfiVar("", "~label"), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, op.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_key_test() { + Traversal traversal = g.V().order().by("name"); + OrderOp op = (OrderOp) getApplyWithOrder(traversal).get(0); + + List expected = + Arrays.asList(Pair.with(ArgUtils.asFfiVar("", "name"), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, op.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_values_test() { + Traversal traversal = g.V().order().by("name"); + OrderOp op = (OrderOp) getApplyWithOrder(traversal).get(0); + + List expected = + Arrays.asList(Pair.with(ArgUtils.asFfiVar("", "name"), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, op.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_a_test() { + Traversal traversal = g.V().as("a").order().by(__.select("a")); + OrderOp op = (OrderOp) getApplyWithOrder(traversal).get(0); + + List expected = Arrays.asList(Pair.with(ArgUtils.asFfiVar("a", ""), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, op.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_a_name_test() { + Traversal traversal = g.V().as("a").order().by(__.select("a").by("name")); + OrderOp op = (OrderOp) getApplyWithOrder(traversal).get(0); + + List expected = + Arrays.asList(Pair.with(ArgUtils.asFfiVar("a", "name"), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, op.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_out_count_test() { + Traversal traversal = g.V().order().by(__.out().count()); + List ops = getApplyWithOrder(traversal); + Assert.assertEquals(2, ops.size()); + + ApplyOp applyOp = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(2, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~alias_1_0", false), applyOp.getAlias().get().applyArg()); + + OrderOp orderOp = (OrderOp) ops.get(1); + List expected = + Arrays.asList(Pair.with(ArgUtils.asFfiVar("~alias_1_0", ""), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, orderOp.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_name_by_out_count_test() { + Traversal traversal = g.V().order().by("name").by(__.out().count()); + List ops = getApplyWithOrder(traversal); + Assert.assertEquals(2, ops.size()); + + ApplyOp applyOp = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(2, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~alias_1_1", false), applyOp.getAlias().get().applyArg()); + + OrderOp orderOp = (OrderOp) ops.get(1); + List expected = + Arrays.asList( + Pair.with(ArgUtils.asFfiVar("", "name"), FfiOrderOpt.Asc), + Pair.with(ArgUtils.asFfiVar("~alias_1_1", ""), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, orderOp.getOrderVarWithOrder().get().applyArg()); + } + + @Test + public void g_V_order_by_out_out_count_by_out_count_test() { + Traversal traversal = + g.V().order().by(__.out().out().count(), Order.desc).by(__.out().count()); + List ops = getApplyWithOrder(traversal); + Assert.assertEquals(3, ops.size()); + + ApplyOp apply1 = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, apply1.getJoinKind().get().applyArg()); + InterOpCollection subOps = (InterOpCollection) apply1.getSubOpCollection().get().applyArg(); + Assert.assertEquals(3, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~alias_1_0", false), apply1.getAlias().get().applyArg()); + + ApplyOp apply2 = (ApplyOp) ops.get(1); + Assert.assertEquals(FfiJoinKind.Inner, apply2.getJoinKind().get().applyArg()); + subOps = (InterOpCollection) apply2.getSubOpCollection().get().applyArg(); + Assert.assertEquals(2, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~alias_1_1", false), apply2.getAlias().get().applyArg()); + + OrderOp orderOp = (OrderOp) ops.get(2); + List expected = + Arrays.asList( + Pair.with(ArgUtils.asFfiVar("~alias_1_0", ""), FfiOrderOpt.Desc), + Pair.with(ArgUtils.asFfiVar("~alias_1_1", ""), FfiOrderOpt.Asc)); + Assert.assertEquals(expected, orderOp.getOrderVarWithOrder().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/SelectStepTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/SelectStepTest.java new file mode 100644 index 000000000000..c114955eeace --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/SelectStepTest.java @@ -0,0 +1,224 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.subtask; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.ProjectOp; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class SelectStepTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getApplyWithProject(Traversal traversal) { + TraversalParent parent = (TraversalParent) traversal.asAdmin().getEndStep(); + return TraversalParentTransformFactory.PROJECT_BY_STEP.apply(parent); + } + + private InterOpBase generateInterOpFromBuilder(Traversal traversal, int idx) { + return (new InterOpCollectionBuilder(traversal)).build().unmodifiableCollection().get(idx); + } + + @Test + public void g_V_select_one_test() { + Traversal traversal = g.V().as("a").select("a"); + ProjectOp op = (ProjectOp) getApplyWithProject(traversal).get(0); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("@a", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } + + @Test + public void g_V_select_one_as_test() { + Traversal traversal = g.V().as("a").select("a").as("b"); + ProjectOp op = (ProjectOp) generateInterOpFromBuilder(traversal, 1); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("@a", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("b", true), exprWithAlias.get(0).getValue1()); + } + + @Test + public void g_V_select_one_by_key_test() { + Traversal traversal = g.V().as("a").select("a").by("name"); + ProjectOp op = (ProjectOp) getApplyWithProject(traversal).get(0); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("@a.name", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } + + @Test + public void g_V_select_one_by_valueMap_key_test() { + Traversal traversal = g.V().as("a").select("a").by(__.valueMap("name")); + ProjectOp op = (ProjectOp) getApplyWithProject(traversal).get(0); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("{@a.name}", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } + + @Test + public void g_V_select_one_by_valueMap_keys_test() { + Traversal traversal = g.V().as("a").select("a").by(__.valueMap("name", "id")); + ProjectOp op = (ProjectOp) getApplyWithProject(traversal).get(0); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + Assert.assertEquals("{@a.name, @a.id}", exprWithAlias.get(0).getValue0()); + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), exprWithAlias.get(0).getValue1()); + } + + @Test + public void g_V_select_test() { + Traversal traversal = g.V().as("a").out().as("b").select("a", "b"); + ProjectOp op = (ProjectOp) getApplyWithProject(traversal).get(0); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + + Pair firstEntry = exprWithAlias.get(0); + Assert.assertEquals("@a", firstEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("a_2_0", false), firstEntry.getValue1()); + + Pair sndEntry = exprWithAlias.get(1); + Assert.assertEquals("@b", sndEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("b_2_1", false), sndEntry.getValue1()); + } + + @Test + public void g_V_select_key_test() { + Traversal traversal = g.V().as("a").out().as("b").select("a", "b").by("name"); + ProjectOp op = (ProjectOp) getApplyWithProject(traversal).get(0); + + List exprWithAlias = (List) op.getExprWithAlias().get().applyArg(); + + Pair firstEntry = exprWithAlias.get(0); + Assert.assertEquals("@a.name", firstEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("a_2_0", false), firstEntry.getValue1()); + + Pair sndEntry = exprWithAlias.get(1); + Assert.assertEquals("@b.name", sndEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("b_2_1", false), sndEntry.getValue1()); + } + + @Test + public void g_V_select_a_by_out_count_test() { + Traversal traversal = g.V().as("a").select("a").by(__.out().count()); + List ops = getApplyWithProject(traversal); + + Assert.assertEquals(2, ops.size()); + + ApplyOp applyOp = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(3, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("a_1_0", false), applyOp.getAlias().get().applyArg()); + + ProjectOp projectOp = (ProjectOp) ops.get(1); + List exprWithAlias = (List) projectOp.getExprWithAlias().get().applyArg(); + Assert.assertEquals(1, exprWithAlias.size()); + + Pair firstEntry = exprWithAlias.get(0); + // expression + Assert.assertEquals("@a_1_0", firstEntry.getValue0()); + // alias + Assert.assertEquals(ArgUtils.asFfiNoneAlias(), firstEntry.getValue1()); + } + + @Test + public void g_V_select_a_b_by_name_by_out_count_test() { + Traversal traversal = + g.V().as("a").out().as("b").select("a", "b").by("name").by(__.out().count()); + List ops = getApplyWithProject(traversal); + + Assert.assertEquals(2, ops.size()); + + ApplyOp applyOp = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(3, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("b_2_1", false), applyOp.getAlias().get().applyArg()); + + ProjectOp projectOp = (ProjectOp) ops.get(1); + List exprWithAlias = (List) projectOp.getExprWithAlias().get().applyArg(); + Assert.assertEquals(2, exprWithAlias.size()); + + Pair firstEntry = exprWithAlias.get(0); + Assert.assertEquals("@a.name", firstEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("a_2_0", false), firstEntry.getValue1()); + + Pair sndEntry = exprWithAlias.get(1); + Assert.assertEquals("@b_2_1", sndEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("b_2_1", false), sndEntry.getValue1()); + } + + @Test + public void g_V_select_a_b_by_out_out_count_test() { + Traversal traversal = + g.V().as("a").out().as("b").select("a", "b").by(__.out().out().count()); + List ops = getApplyWithProject(traversal); + + Assert.assertEquals(3, ops.size()); + + ApplyOp apply1 = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, apply1.getJoinKind().get().applyArg()); + InterOpCollection subOps = (InterOpCollection) apply1.getSubOpCollection().get().applyArg(); + Assert.assertEquals(4, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("a_2_0", false), apply1.getAlias().get().applyArg()); + + ApplyOp apply2 = (ApplyOp) ops.get(1); + Assert.assertEquals(FfiJoinKind.Inner, apply2.getJoinKind().get().applyArg()); + subOps = (InterOpCollection) apply2.getSubOpCollection().get().applyArg(); + Assert.assertEquals(4, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("b_2_1", false), apply2.getAlias().get().applyArg()); + + ProjectOp projectOp = (ProjectOp) ops.get(2); + List exprWithAlias = (List) projectOp.getExprWithAlias().get().applyArg(); + Assert.assertEquals(2, exprWithAlias.size()); + + Pair firstEntry = exprWithAlias.get(0); + Assert.assertEquals("@a_2_0", firstEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("a_2_0", false), firstEntry.getValue1()); + + Pair sndEntry = exprWithAlias.get(1); + Assert.assertEquals("@b_2_1", sndEntry.getValue0()); + Assert.assertEquals(ArgUtils.asFfiAlias("b_2_1", false), sndEntry.getValue1()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/WherePredicateTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/WherePredicateTest.java new file mode 100644 index 000000000000..44848a9f320d --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/WherePredicateTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.subtask; + +import com.alibaba.graphscope.common.intermediate.ArgUtils; +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.SelectOp; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class WherePredicateTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getApplyWithSelect(Traversal traversal) { + TraversalParent parent = (TraversalParent) traversal.asAdmin().getEndStep(); + return TraversalParentTransformFactory.WHERE_BY_STEP.apply(parent); + } + + @Test + public void g_V_where_eq_a() { + Traversal traversal = g.V().as("a").out().where(P.eq("a")); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals("@ == @a", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a_eq_b() { + Traversal traversal = g.V().as("a").out().as("b").where("a", P.eq("b")); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals("@a == @b", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a_eq_b_or_eq_c() { + Traversal traversal = + g.V().as("a").out().as("b").out().as("c").where("a", P.eq("b").or(P.eq("c"))); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals("@a == @b || (@a == @c)", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_eq_a_age() { + Traversal traversal = g.V().as("a").out().where(P.eq("a")).by("age"); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals( + "@.age && @a.age && @.age == @a.age", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_eq_a_values_age() { + Traversal traversal = g.V().as("a").out().where(P.eq("a")).by(__.values("age")); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals( + "@.age && @a.age && @.age == @a.age", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a_id_eq_b_age() { + Traversal traversal = g.V().as("a").out().as("b").where("a", P.eq("b")).by("id").by("age"); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals( + "@a.id && @b.age && @a.id == @b.age", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a_id_eq_b_age_or_c_id() { + Traversal traversal = + g.V().as("a") + .out() + .as("b") + .out() + .as("c") + .where("a", P.eq("b").or(P.eq("c"))) + .by("id") + .by("age") + .by("id"); + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(0); + + Assert.assertEquals( + "@a.id && @b.age && @a.id == @b.age || (@a.id && @c.id && @a.id == @c.id)", + selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a_eq_b_by_out_count() { + Traversal traversal = + g.V().as("a").out().as("b").where("a", P.eq("b")).by(__.out().count()); + List ops = getApplyWithSelect(traversal); + + ApplyOp apply1 = (ApplyOp) ops.get(0); + Assert.assertEquals(FfiJoinKind.Inner, apply1.getJoinKind().get().applyArg()); + InterOpCollection subOps = (InterOpCollection) apply1.getSubOpCollection().get().applyArg(); + Assert.assertEquals(3, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~alias_2_0", false), apply1.getAlias().get().applyArg()); + + ApplyOp apply2 = (ApplyOp) ops.get(1); + Assert.assertEquals(FfiJoinKind.Inner, apply2.getJoinKind().get().applyArg()); + subOps = (InterOpCollection) apply2.getSubOpCollection().get().applyArg(); + Assert.assertEquals(3, subOps.unmodifiableCollection().size()); + Assert.assertEquals( + ArgUtils.asFfiAlias("~alias_2_1", false), apply2.getAlias().get().applyArg()); + + SelectOp selectOp = (SelectOp) getApplyWithSelect(traversal).get(2); + Assert.assertEquals("@~alias_2_0 == @~alias_2_1", selectOp.getPredicate().get().applyArg()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/WhereTraversalTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/WhereTraversalTest.java new file mode 100644 index 000000000000..2da3c77f6d9a --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/gremlin/subtask/WhereTraversalTest.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.gremlin.subtask; + +import com.alibaba.graphscope.common.intermediate.InterOpCollection; +import com.alibaba.graphscope.common.intermediate.operator.ApplyOp; +import com.alibaba.graphscope.common.intermediate.operator.InterOpBase; +import com.alibaba.graphscope.common.intermediate.operator.SelectOp; +import com.alibaba.graphscope.common.jna.type.FfiJoinKind; +import com.alibaba.graphscope.gremlin.transform.TraversalParentTransformFactory; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class WhereTraversalTest { + private Graph graph = TinkerFactory.createModern(); + private GraphTraversalSource g = graph.traversal(); + + private List getApplyOrSelect(Traversal traversal) { + TraversalParent parent = (TraversalParent) traversal.asAdmin().getEndStep(); + return TraversalParentTransformFactory.WHERE_TRAVERSAL_STEP.apply(parent); + } + + @Test + public void g_V_where_values() { + Traversal traversal = g.V().where(__.values("name")); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("@.name", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a() { + Traversal traversal = g.V().as("a").where(__.select("a")); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("@a", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_a_values() { + Traversal traversal = g.V().as("a").where(__.select("a").by(__.values("name"))); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("@a.name", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_as_a() { + Traversal traversal = g.V().as("a").where(__.as("a")); + SelectOp selectOp = (SelectOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals("@a", selectOp.getPredicate().get().applyArg()); + } + + @Test + public void g_V_where_out_out() { + Traversal traversal = g.V().where(__.out().out()); + ApplyOp applyOp = (ApplyOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals(FfiJoinKind.Semi, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(2, subOps.unmodifiableCollection().size()); + } + + @Test + public void g_V_where_as_out_as() { + Traversal traversal = g.V().as("a").out().as("b").where(__.as("a").out().as("b")); + ApplyOp applyOp = (ApplyOp) getApplyOrSelect(traversal).get(0); + + Assert.assertEquals(FfiJoinKind.Semi, applyOp.getJoinKind().get().applyArg()); + InterOpCollection subOps = + (InterOpCollection) applyOp.getSubOpCollection().get().applyArg(); + Assert.assertEquals(3, subOps.unmodifiableCollection().size()); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/IrGremlinTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/IrGremlinTest.java new file mode 100644 index 000000000000..b1816f862260 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/IrGremlinTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.integration; + +import com.alibaba.graphscope.gremlin.integration.graph.RemoteTestGraph; +import com.alibaba.graphscope.gremlin.integration.graph.RemoteTestGraphProvider; + +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.junit.runner.RunWith; + +@RunWith(IrGremlinTestSuite.class) +@GraphProviderClass(provider = RemoteTestGraphProvider.class, graph = RemoteTestGraph.class) +public class IrGremlinTest {} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/IrGremlinTestSuite.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/IrGremlinTestSuite.java new file mode 100644 index 000000000000..accaf91fe40f --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/IrGremlinTestSuite.java @@ -0,0 +1,117 @@ +/* + * This file is referred and derived from project apache/tinkerpop + * + * https://github.com/apache/tinkerpop/blob/master/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java + * + * which has the following license: + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.alibaba.graphscope.integration; + +import org.apache.tinkerpop.gremlin.AbstractGremlinSuite; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.RepeatTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.*; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerBuilder; + +public class IrGremlinTestSuite extends AbstractGremlinSuite { + + private static final Class[] allTests = + new Class[] { + // branch + RepeatTest.Traversals.class, + UnionTest.Traversals.class, + + // filter + CyclicPathTest.Traversals.class, + DedupTest.Traversals.class, + FilterTest.Traversals.class, + HasTest.Traversals.class, + IsTest.Traversals.class, + RangeTest.Traversals.class, + SimplePathTest.Traversals.class, + WhereTest.Traversals.class, + + // map + org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest.Traversals.class, + GraphTest.Traversals.class, + OrderTest.Traversals.class, + PathTest.Traversals.class, + PropertiesTest.Traversals.class, + SelectTest.Traversals.class, + VertexTest.Traversals.class, + UnfoldTest.Traversals.class, + ValueMapTest.Traversals.class, + GroupTest.Traversals.class, + GroupCountTest.Traversals.class, + + // match + MatchTest.CountMatchTraversals.class, + }; + + private static final Class[] testsToEnforce = + new Class[] { + // branch + RepeatTest.Traversals.class, + UnionTest.Traversals.class, + + // filter + CyclicPathTest.Traversals.class, + DedupTest.Traversals.class, + FilterTest.Traversals.class, + HasTest.Traversals.class, + IsTest.Traversals.class, + RangeTest.Traversals.class, + SimplePathTest.Traversals.class, + WhereTest.Traversals.class, + + // map + org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest.Traversals.class, + GraphTest.Traversals.class, + OrderTest.Traversals.class, + PathTest.Traversals.class, + PropertiesTest.Traversals.class, + SelectTest.Traversals.class, + VertexTest.Traversals.class, + UnfoldTest.Traversals.class, + ValueMapTest.Traversals.class, + GroupTest.Traversals.class, + GroupCountTest.Traversals.class, + + // match + MatchTest.CountMatchTraversals.class, + }; + + public IrGremlinTestSuite(final Class klass, final RunnerBuilder builder) + throws InitializationError { + super(klass, builder, allTests, testsToEnforce, false, TraversalEngine.Type.STANDARD); + } + + public IrGremlinTestSuite( + final Class klass, final RunnerBuilder builder, final Class[] testsToExecute) + throws InitializationError { + super(klass, builder, testsToExecute, testsToEnforce, true, TraversalEngine.Type.STANDARD); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/IrLdbcTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/IrLdbcTest.java new file mode 100644 index 000000000000..703a17699458 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/IrLdbcTest.java @@ -0,0 +1,11 @@ +package com.alibaba.graphscope.integration.ldbc; + +import com.alibaba.graphscope.gremlin.integration.graph.RemoteTestGraph; +import com.alibaba.graphscope.gremlin.integration.graph.RemoteTestGraphProvider; + +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.junit.runner.RunWith; + +@RunWith(IrLdbcTestSuite.class) +@GraphProviderClass(provider = RemoteTestGraphProvider.class, graph = RemoteTestGraph.class) +public class IrLdbcTest {} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/IrLdbcTestSuite.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/IrLdbcTestSuite.java new file mode 100644 index 000000000000..482220536612 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/IrLdbcTestSuite.java @@ -0,0 +1,30 @@ +package com.alibaba.graphscope.integration.ldbc; + +import org.apache.tinkerpop.gremlin.AbstractGremlinSuite; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerBuilder; + +public class IrLdbcTestSuite extends AbstractGremlinSuite { + + private static final Class[] allTests = + new Class[] { + LdbcQueryTest.Traversals.class, + }; + + private static final Class[] testsToEnforce = + new Class[] { + LdbcQueryTest.Traversals.class, + }; + + public IrLdbcTestSuite(final Class klass, final RunnerBuilder builder) + throws InitializationError { + super(klass, builder, allTests, testsToEnforce, false, TraversalEngine.Type.STANDARD); + } + + public IrLdbcTestSuite( + final Class klass, final RunnerBuilder builder, final Class[] testsToExecute) + throws InitializationError { + super(klass, builder, testsToExecute, testsToEnforce, true, TraversalEngine.Type.STANDARD); + } +} diff --git a/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/LdbcQueryTest.java b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/LdbcQueryTest.java new file mode 100644 index 000000000000..f26541ee4b34 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/java/com/alibaba/graphscope/integration/ldbc/LdbcQueryTest.java @@ -0,0 +1,861 @@ +package com.alibaba.graphscope.integration.ldbc; + +import com.alibaba.graphscope.gremlin.plugin.traversal.IrCustomizedTraversal; +import com.google.common.collect.Sets; + +import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest; +import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Column; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class LdbcQueryTest extends AbstractGremlinProcessTest { + public abstract Traversal> get_ldbc_1_test(); + + public abstract Traversal> get_ldbc_2_test(); + + public abstract Traversal get_ldbc_3_test(); + + public abstract Traversal> get_ldbc_4_test(); + + public abstract Traversal> get_ldbc_5_test(); + + public abstract Traversal> get_ldbc_6_test(); + + public abstract Traversal> get_ldbc_7_test(); + + public abstract Traversal> get_ldbc_8_test(); + + public abstract Traversal> get_ldbc_9_test(); + + public abstract Traversal> get_ldbc_11_test(); + + public abstract Traversal> get_ldbc_12_test(); + + @Test + public void run_ldbc_1_test() { + Traversal> traversal = this.get_ldbc_1_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{p=2, a={firstName=[Chau], lastName=[Nguyen], id=[4848]}}", + "{p=2, a={firstName=[Chau], lastName=[Do], id=[9101]}}", + "{p=2, a={firstName=[Chau], lastName=[Nguyen], id=[2199023265573]}}", + "{p=2, a={firstName=[Chau], lastName=[Loan], id=[6597069771031]}}", + "{p=2, a={firstName=[Chau], lastName=[Nguyen], id=[8796093031224]}}", + "{p=2, a={firstName=[Chau], lastName=[Ho], id=[10995116285282]}}", + "{p=2, a={firstName=[Chau], lastName=[Loan], id=[13194139544258]}}", + "{p=2, a={firstName=[Chau], lastName=[Ha], id=[15393162793500]}}", + "{p=2, a={firstName=[Chau], lastName=[Ho], id=[19791209303405]}}", + "{p=2, a={firstName=[Chau], lastName=[Nguyen], id=[26388279068635]}}", + "{p=2, a={firstName=[Chau], lastName=[Loan], id=[26388279076217]}}", + "{p=2, a={firstName=[Chau], lastName=[Nguyen], id=[28587302322743]}}", + "{p=2, a={firstName=[Chau], lastName=[Ho], id=[28587302323020]}}", + "{p=2, a={firstName=[Chau], lastName=[Ho], id=[32985348842021]}}", + "{p=3, a={firstName=[Chau], lastName=[Loan], id=[10995116284332]}}", + "{p=3, a={firstName=[Chau], lastName=[Ha], id=[15393162789090]}}", + "{p=3, a={firstName=[Chau], lastName=[Nguyen], id=[26388279072379]}}", + "{p=3, a={firstName=[Chau], lastName=[Ho], id=[32985348840129]}}"); + + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_2_test() { + Traversal> traversal = this.get_ldbc_2_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{p={lastName=[Khan], firstName=[Kunal], id=[30786325587937]}," + + " m={id=[2061587339072], creationDate=[20120803072025654]," + + " content=[fine]}}", + "{p={lastName=[Rao], firstName=[Abhishek], id=[8796093031885]}," + + " m={id=[2061587080174], creationDate=[20120803070839866]," + + " content=[About Paul Martin, aham for the inteAbout Ernest" + + " Hemingway, which he wrote FAbo]}}", + "{p={lastName=[Khan], firstName=[John], id=[8796093029267]}," + + " m={id=[2061585035664], creationDate=[20120803070658893]," + + " content=[About Amitabh Bachchan, l. In additAbout Chen Shui-bian," + + " en Yi-hsiunAbout Denzel Wash]}}", + "{p={lastName=[Khan], firstName=[John], id=[8796093029267]}," + + " m={imageFile=[], id=[2061587033910]," + + " creationDate=[20120803070442531], content=[About Abbas I of Persia," + + " took back land from the Portuguese and the Mughals. Abbas was a" + + " great builder and moved his kingdom's capital from Qazvin to" + + " Isfahan. In his later years, the shah became suspicious of his own" + + " so]}}", + "{p={lastName=[Khan], firstName=[Arjun], id=[30786325579845]}," + + " m={id=[2061589148849], creationDate=[20120803065019796]," + + " content=[thanks]}}", + "{p={lastName=[Rao], firstName=[Abhishek], id=[8796093031885]}," + + " m={id=[2061590844627], creationDate=[20120803064241072]," + + " content=[About Thomas Aquinas, he Great Books and seminar method." + + " It haAbout Edward Elga]}}", + "{p={lastName=[Khan], firstName=[John], id=[8796093029267]}," + + " m={id=[2061585035666], creationDate=[20120803064146457]," + + " content=[great]}}", + "{p={lastName=[Bolier], firstName=[Albert], id=[8796093031407]}," + + " m={id=[2061588995483], creationDate=[20120803063619558]," + + " content=[thx]}}", + "{p={lastName=[Rao], firstName=[Abhishek], id=[8796093031885]}," + + " m={id=[2061587080596], creationDate=[20120803062639721]," + + " content=[About Walt Whitman, n was concernAbout At Fillmore East," + + " 500 Greatest About Italy, Coun]}}", + "{p={lastName=[Khan], firstName=[Shweta], id=[7725]}," + + " m={id=[2061588924543], creationDate=[20120803061904810]," + + " content=[great]}}", + "{p={lastName=[Rao], firstName=[Arjun], id=[10995116279390]}," + + " m={id=[2061585663942], creationDate=[20120803061330637]," + + " content=[About William the Conqueror, ed, but William was able to" + + " put them dowAbout Anne]}}", + "{p={lastName=[Garcia], firstName=[Isabel], id=[10995116286316]}," + + " m={id=[2061590620731], creationDate=[20120803060002996]," + + " content=[maybe]}}", + "{p={lastName=[Khan], firstName=[Arjun], id=[24189255811940]}," + + " m={id=[2061586499110], creationDate=[20120803053430934]," + + " content=[About Samuel Johnson, t, literary critic, biAbout Francis" + + " Bacon, proper methodology ]}}", + "{p={lastName=[Zhang], firstName=[Yang], id=[13194139535025]}," + + " m={id=[2061588568513], creationDate=[20120803050920738]," + + " content=[About Harry S. Truman, edom and human rights seemAbout" + + " John Steinbeck, and E]}}", + "{p={lastName=[Kumar], firstName=[Deepak], id=[17592186053700]}," + + " m={id=[2061588703136], creationDate=[20120803045119016]," + + " content=[great]}}", + "{p={lastName=[Garcia], firstName=[Isabel], id=[10995116286316]}," + + " m={id=[2061587017186], creationDate=[20120803043821969]," + + " content=[ok]}}", + "{p={lastName=[Rao], firstName=[Arjun], id=[10995116279390]}," + + " m={id=[2061584412526], creationDate=[20120803043634096]," + + " content=[great]}}", + "{p={lastName=[Zhang], firstName=[Yang], id=[13194139535025]}," + + " m={id=[2061589148863], creationDate=[20120803034926704]," + + " content=[ok]}}", + "{p={lastName=[Rao], firstName=[Arjun], id=[10995116279390]}," + + " m={id=[2061585353358], creationDate=[20120803033714516]," + + " content=[About Chen Shui-bian, ended more than fiAbout Henry Clay," + + " grams. In 1957,]}}", + "{p={lastName=[Rao], firstName=[Abhishek], id=[8796093031885]}," + + " m={id=[2061587080731], creationDate=[20120803033107843]," + + " content=[About Augustine of Hippo, ity of God, distincAbout Harold" + + " Arlen, , a numbe]}}"); + + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_3_test() { + Traversal traversal = this.get_ldbc_3_test(); + this.printTraversalForm(traversal); + // V[72066390130957625]: id: 8796093029689, firstName: Eun-Hye, lastName: Yoon + Assert.assertEquals(72066390130957625L, traversal.next().id()); + Assert.assertFalse(traversal.hasNext()); + } + + @Test + public void run_ldbc_4_test() { + Traversal> traversal = this.get_ldbc_4_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{postCount=3, tagName=Ehud_Olmert}", + "{postCount=1, tagName=Be-Bop-A-Lula}", + "{postCount=1, tagName=Kingdom_of_Sardinia}", + "{postCount=1, tagName=The_Singles:_The_First_Ten_Years}"); + + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_5_test() { + Traversal> traversal = this.get_ldbc_5_test(); + this.printTraversalForm(traversal); + + String expected = + "{v[288230788468638061]=2, v[288231613102324388]=2, v[288231887980212232]=2," + + " v[288230788468574546]=1, v[288230788468598862]=1, v[288230925907600292]=1," + + " v[288230925907605637]=1, v[288231338224392113]=1, v[288231475663357917]=1," + + " v[288231475663417090]=1, v[288231750541328016]=1, v[288231887980204174]=1," + + " v[288231887981252988]=1, v[288232162858188293]=1, v[288232162858200268]=1," + + " v[288232300297113293]=1, v[288232300297120851]=1, v[288232437736032472]=1," + + " v[288232437736038238]=1}"; + Map result = traversal.next(); + Assert.assertEquals(expected, result.toString()); + Assert.assertFalse(traversal.hasNext()); + } + + @Test + public void run_ldbc_6_test() { + Traversal> traversal = this.get_ldbc_6_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{keys=Tom_Gehrels, values=28}", + "{keys=Sammy_Sosa, values=9}", + "{keys=Charles_Dickens, values=5}", + "{keys=Genghis_Khan, values=5}", + "{keys=Ivan_Ljubičić, values=5}", + "{keys=Marc_Gicquel, values=5}", + "{keys=Freddie_Mercury, values=4}", + "{keys=Peter_Hain, values=4}", + "{keys=Robert_Fripp, values=4}", + "{keys=Boris_Yeltsin, values=3}"); + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_7_test() { + Traversal> traversal = this.get_ldbc_7_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{likedate=20120912143240024, liker={firstName=[Jean-Pierre]," + + " lastName=[Kanam], id=[17592186049473]}," + + " message={id=[2199024319581], content=[About Charles V, Holy Roman" + + " Emperor, rmation. In addition to thAbout Bob Dyla]}}", + "{likedate=20120911131917097, liker={firstName=[Ajuma], lastName=[Leakey]," + + " id=[32985348842700]}, message={id=[1786707240991], content=[About" + + " Costa Rica, a to the southeast, the Pacific Ocean to theAbout" + + " Napoleon III, pula]}}", + "{likedate=20120910053218016, liker={firstName=[Mohamed], lastName=[Wong]," + + " id=[32985348835903]}, message={id=[1786706525707], content=[About" + + " Christopher Lee, strong voice and imposing height. He has performed" + + " ro]}}", + "{likedate=20120905054751531, liker={firstName=[Alfonso]," + + " lastName=[Gonzalez], id=[32985348842314]}," + + " message={id=[1786707314163], content=[About Anne, Queen of Great" + + " Britain, the resAbout Bob Dylan, ed in MaAbout Frank Zappa]}}", + "{likedate=20120904190103709, liker={firstName=[Sirak], lastName=[Dego]," + + " id=[32985348835356]}, message={id=[1511833096614], content=[About" + + " Gustav Mahler, (Hofoper). DAbout Karl Marx, list politicaAbout" + + " Pablo Picasso, a ]}}", + "{likedate=20120904154553202, liker={firstName=[Charles], lastName=[Bona]," + + " id=[32985348837885]}, message={id=[1786707240991], content=[About" + + " Costa Rica, a to the southeast, the Pacific Ocean to theAbout" + + " Napoleon III, pula]}}", + "{likedate=20120904131816796, liker={firstName=[Aditya], lastName=[Khan]," + + " id=[32985348841531]}, message={id=[1786707314163], content=[About" + + " Anne, Queen of Great Britain, the resAbout Bob Dylan, ed in" + + " MaAbout Frank Zappa]}}", + "{likedate=20120904114300885, liker={firstName=[John], lastName=[Kumar]," + + " id=[32985348836287]}, message={id=[1786706617430], content=[About" + + " Mickey Rooney, iner whose film, telAbout Michael Moore, ions. He" + + " has als]}}", + "{likedate=20120902221041145, liker={firstName=[Ivan], lastName=[Santiago]," + + " id=[32985348841493]}, message={id=[1786707240991], content=[About" + + " Costa Rica, a to the southeast, the Pacific Ocean to theAbout" + + " Napoleon III, pula]}}", + "{likedate=20120902190352859, liker={firstName=[Baruch], lastName=[Dego]," + + " id=[4139]}, message={id=[2061584849006], content=[About Left" + + " Behind, game Left Behind: Eternal Forces and its sequels, Left ]}}", + "{likedate=20120902133650353, liker={firstName=[Hossein]," + + " lastName=[Karimi], id=[32985348842048]}," + + " message={id=[2061584849006], content=[About Left Behind, game Left" + + " Behind: Eternal Forces and its sequels, Left ]}}", + "{likedate=20120901170918823, liker={firstName=[Megumi], lastName=[Suzuki]," + + " id=[28587302326663]}, message={id=[1511830609010], content=[About" + + " Mariano Rivera, nt relievers in major league history. PAbout Naima," + + " an ]}}", + "{likedate=20120901130728341, liker={firstName=[Deepak], lastName=[Bose]," + + " id=[24189255812226]}, message={id=[2061584849006], content=[About" + + " Left Behind, game Left Behind: Eternal Forces and its sequels, Left" + + " ]}}", + "{likedate=20120901113114136, liker={firstName=[Zeki], lastName=[Arikan]," + + " id=[19791209300402]}, message={id=[2061584849006], content=[About" + + " Left Behind, game Left Behind: Eternal Forces and its sequels, Left" + + " ]}}", + "{likedate=20120901095813249, liker={firstName=[Manuel], lastName=[Cosio]," + + " id=[28587302332758]}, message={id=[2061584849006], content=[About" + + " Left Behind, game Left Behind: Eternal Forces and its sequels, Left" + + " ]}}", + "{likedate=20120901072309081, liker={firstName=[Jacques]," + + " lastName=[Arnaud], id=[15393162795133]}," + + " message={id=[2061584849006], content=[About Left Behind, game Left" + + " Behind: Eternal Forces and its sequels, Left ]}}", + "{likedate=20120831225619539, liker={firstName=[Ching], lastName=[Hoang]," + + " id=[32985348833559]}, message={id=[1236951518889], content=[About" + + " Pope Paul VI, cese, yet denying hiAbout Bono, tress reduction. It" + + " About Lou Reed, an American rock muAbout ]}}", + "{likedate=20120831154258335, liker={firstName=[Ahmad Rafiq]," + + " lastName=[Akbar], id=[17592186053137]}," + + " message={id=[2061584849006], content=[About Left Behind, game Left" + + " Behind: Eternal Forces and its sequels, Left ]}}", + "{likedate=20120831135801287, liker={firstName=[Ismail], lastName=[Aziz]," + + " id=[10995116282290]}, message={id=[2061584849006], content=[About" + + " Left Behind, game Left Behind: Eternal Forces and its sequels, Left" + + " ]}}", + "{likedate=20120831122429543, liker={firstName=[Ali], lastName=[Lo]," + + " id=[8796093029002]}, message={id=[2061584849006], content=[About" + + " Left Behind, game Left Behind: Eternal Forces and its sequels, Left" + + " ]}}"); + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_8_test() { + Traversal> traversal = this.get_ldbc_8_test(); + this.printTraversalForm(traversal); + int counter = 0; + + Set expected = + Sets.newHashSet( + "{comment={id=[1786706501264], creationDate=[20120508164718581]," + + " content=[About Denmark, ual struggle for control of the Baltic Sea;" + + " before the digging of the Kiel]}, commenter={firstName=[Isabel]," + + " lastName=[Garcia], id=[10995116286316]}}", + "{comment={id=[1786706501272], creationDate=[20120417052950802]," + + " content=[About Jorge Luis Borges, ia. Scholars have also" + + " suggesteAbout Hong Kong, en described as E]}," + + " commenter={firstName=[Abhishek], lastName=[Rao]," + + " id=[8796093031885]}}", + "{comment={id=[1786711226851], creationDate=[20120421111140819]," + + " content=[About Thomas Edison, and factories – About Buddy Holly, d" + + " a pioneer of roAbout Samu]}, commenter={firstName=[Arif]," + + " lastName=[Lesmana], id=[5976]}}", + "{comment={id=[1786711226855], creationDate=[20120421005529694]," + + " content=[About John Rhys-Davies, ohn Rhys-Davies (About Dog Man" + + " Star, it is considered Ab]}, commenter={firstName=[Jan]," + + " lastName=[Anton], id=[10995116288583]}}", + "{comment={id=[1924145454735], creationDate=[20120509045720143]," + + " content=[cool]}, commenter={firstName=[Deepak], lastName=[Kumar]," + + " id=[4398046513018]}}", + "{comment={id=[1374389640939], creationDate=[20111019210610461]," + + " content=[duh]}, commenter={firstName=[Abhishek], lastName=[Rao]," + + " id=[8796093031885]}}", + "{comment={id=[1786711226853], creationDate=[20120421210719836]," + + " content=[roflol]}, commenter={firstName=[Eddie], lastName=[Garcia]," + + " id=[5330]}}", + "{comment={id=[1374389640931], creationDate=[20111019163439495]," + + " content=[no way!]}, commenter={firstName=[Abhishek]," + + " lastName=[Rao], id=[8796093031885]}}", + "{comment={id=[1786711226854], creationDate=[20120421030229805]," + + " content=[About Samuel Taylor Coleridge, ic, literaryAbout Thomas" + + " Edison, dern industrA]}, commenter={firstName=[Mads]," + + " lastName=[Haugland], id=[13194139543018]}}", + "{comment={id=[2061584551353], creationDate=[20120902192046400]," + + " content=[ok]}, commenter={firstName=[John], lastName=[Iyar]," + + " id=[28587302328223]}}", + "{comment={id=[1786706501274], creationDate=[20120416184458353]," + + " content=[duh]}, commenter={firstName=[John], lastName=[Wilson]," + + " id=[1490]}}", + "{comment={id=[1786711226865], creationDate=[20120421221926988]," + + " content=[About Euripides, ives that wereAbout John Rhys-Davies, d" + + " the voices oAbout Henry D]}, commenter={firstName=[Antonio]," + + " lastName=[Garcia], id=[13194139539603]}}", + "{comment={id=[2061584551352], creationDate=[20120903111223692]," + + " content=[maybe]}, commenter={firstName=[Ashin], lastName=[Karat]," + + " id=[17592186044810]}}", + "{comment={id=[1786706501276], creationDate=[20120417004630904]," + + " content=[About Felix Mendelssohn, Franz Liszt, RichaAbout Tony" + + " Blair, our Prime Mi]}, commenter={firstName=[Deepak]," + + " lastName=[Kumar], id=[4398046513018]}}", + "{comment={id=[1374389640938], creationDate=[20111019193909069]," + + " content=[cool]}, commenter={firstName=[Albert], lastName=[Bolier]," + + " id=[8796093031407]}}", + "{comment={id=[1374389640946], creationDate=[20111020022520841]," + + " content=[ok]}, commenter={firstName=[Isabel], lastName=[Garcia]," + + " id=[10995116286316]}}", + "{comment={id=[2061584551354], creationDate=[20120902192731040]," + + " content=[About Lil Wayne, m. Lil Wayne released his debut rock" + + " album, Rebirth, in 2]}, commenter={firstName=[Farrukh]," + + " lastName=[Znaimer], id=[17592186048413]}}", + "{comment={id=[1511830965950], creationDate=[20111221171052493], content=[I" + + " see]}, commenter={firstName=[Rahul], lastName=[Reddy]," + + " id=[19791209303129]}}", + "{comment={id=[1511828737702], creationDate=[20111206230226884]," + + " content=[thanks]}, commenter={firstName=[K.], lastName=[Sharma]," + + " id=[9773]}}", + "{comment={id=[1374389640934], creationDate=[20111019225544527]," + + " content=[About Desiderius Erasmus, ined committed to reforming" + + " theAbout George Washington, his wea]}," + + " commenter={firstName=[Yang], lastName=[Zhang]," + + " id=[13194139535025]}}"); + + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(expected.contains(bindings.toString())); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_9_test() { + Traversal> traversal = this.get_ldbc_9_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{post={id=[1511829711860], creationDate=[20111216235809425]," + + " content=[About Augustine of Hippo, osopher and theologian from" + + " Roman Africa. About Che Gueva]}, friends={lastName=[Wang]," + + " firstName=[Xiaolu], id=[2199023260919]}}", + "{post={id=[1511830666887], creationDate=[20111216235709064]," + + " content=[good]}, friends={lastName=[Yamada], firstName=[Prince]," + + " id=[2199023260291]}}", + "{post={imageFile=[], id=[1511831473649], creationDate=[20111216235526728]," + + " content=[About Ho Chi Minh, ource Consulting, Economist" + + " Intelligence Unit and ECA International, Ho Chi Minh City is ranked" + + " 132 on the list of]}, friends={lastName=[Brown], firstName=[Jack]," + + " id=[10995116288703]}}", + "{post={id=[1511834999905], creationDate=[20111216235442124]," + + " content=[yes]}, friends={lastName=[Hosseini], firstName=[Hossein]," + + " id=[19791209310913]}}", + "{post={id=[1511828923913], creationDate=[20111216235355410]," + + " content=[ok]}, friends={lastName=[Khan], firstName=[Kiran]," + + " id=[8796093029365]}}", + "{post={id=[1511829478450], creationDate=[20111216235154542]," + + " content=[dia.org American Samoa competed at the 2004 Summer" + + " Olympics in Athens, Greece.About Ame]}," + + " friends={lastName=[Amenábar], firstName=[Carlos]," + + " id=[10995116279387]}}", + "{post={id=[1511829728931], creationDate=[20111216234926822]," + + " content=[cool]}, friends={lastName=[Khan], firstName=[Babar]," + + " id=[8796093030398]}}", + "{post={id=[1511828864535], creationDate=[20111216234741888]," + + " content=[LOL]}, friends={lastName=[Codreanu], firstName=[Victor]," + + " id=[17592186047200]}}", + "{post={imageFile=[], id=[1511829728929], creationDate=[20111216234716228]," + + " content=[About Adolf Hitler, is views were more or less formed" + + " during three perioAbout Aristotle, odern advent ]}," + + " friends={lastName=[Kazadi], firstName=[Jean van de]," + + " id=[8796093026337]}}", + "{post={id=[1511829863227], creationDate=[20111216234401470]," + + " content=[ok]}, friends={lastName=[David], firstName=[Mihai]," + + " id=[17592186045238]}}", + "{post={id=[1511828832823], creationDate=[20111216234140954]," + + " content=[About Henry Kissinger, ecretary of State in the" + + " administratioAbout Luxembo]}, friends={lastName=[Donati]," + + " firstName=[Giuseppe], id=[8796093028051]}}", + "{post={id=[1511831505173], creationDate=[20111216233920753], content=[no" + + " way!]}, friends={lastName=[Chen], firstName=[Lin], id=[9850]}}", + "{post={id=[1511829861734], creationDate=[20111216233654529]," + + " content=[thx]}, friends={lastName=[Redl], firstName=[Eva]," + + " id=[21990232557420]}}", + "{post={id=[1511831310812], creationDate=[20111216233627372]," + + " content=[About Jackson Browne, notable songs throughout About Rudy" + + " Giuliani, t to run and remain activAbout Jefferson Davis, the]}," + + " friends={lastName=[Xu], firstName=[Anson], id=[17592186049298]}}", + "{post={id=[1511835042602], creationDate=[20111216233450452], content=[no" + + " way!]}, friends={lastName=[Abouba], firstName=[Hamani]," + + " id=[15393162791608]}}", + "{post={imageFile=[photo1511833666259.jpg], id=[1511833666259]," + + " creationDate=[20111216233439099], content=[]}," + + " friends={lastName=[Yamada], firstName=[Prince]," + + " id=[2199023260291]}}", + "{post={imageFile=[photo1511833666258.jpg], id=[1511833666258]," + + " creationDate=[20111216233438099], content=[]}," + + " friends={lastName=[Yamada], firstName=[Prince]," + + " id=[2199023260291]}}", + "{post={imageFile=[photo1511833666257.jpg], id=[1511833666257]," + + " creationDate=[20111216233437099], content=[]}," + + " friends={lastName=[Yamada], firstName=[Prince]," + + " id=[2199023260291]}}", + "{post={imageFile=[photo1511833666256.jpg], id=[1511833666256]," + + " creationDate=[20111216233436099], content=[]}," + + " friends={lastName=[Yamada], firstName=[Prince]," + + " id=[2199023260291]}}", + "{post={imageFile=[photo1511833666255.jpg], id=[1511833666255]," + + " creationDate=[20111216233435099], content=[]}," + + " friends={lastName=[Yamada], firstName=[Prince]," + + " id=[2199023260291]}}"); + + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_11_test() { + Traversal> traversal = this.get_ldbc_11_test(); + this.printTraversalForm(traversal); + int counter = 0; + + List expected = + Arrays.asList( + "{works=2002, orgname=Lao_Airlines, friends={lastName=[Pham]," + + " firstName=[Eve-Mary Thai], id=[6597069767125]}}", + "{works=2002, orgname=Lao_Airlines, friends={lastName=[Hafez]," + + " firstName=[Atef], id=[28587302330691]}}", + "{works=2004, orgname=Lao_Airlines, friends={lastName=[Vorachith]," + + " firstName=[Cy], id=[5869]}}", + "{works=2005, orgname=Lao_Air, friends={lastName=[Vang], firstName=[Mee]," + + " id=[8796093022909]}}", + "{works=2005, orgname=Lao_Airlines, friends={lastName=[Charoenpura]," + + " firstName=[Jetsada], id=[10995116285549]}}", + "{works=2006, orgname=Lao_Airlines, friends={lastName=[Anwar]," + + " firstName=[A.], id=[24189255815555]}}", + "{works=2007, orgname=Lao_Air, friends={lastName=[Li], firstName=[Ben]," + + " id=[2199023266276]}}", + "{works=2007, orgname=Lao_Airlines, friends={lastName=[Sysavanh]," + + " firstName=[Pao], id=[8796093027636]}}", + "{works=2008, orgname=Lao_Air, friends={lastName=[Vongvichit]," + + " firstName=[Mee], id=[1259]}}", + "{works=2009, orgname=Lao_Air, friends={lastName=[Achiou], firstName=[Ali]," + + " id=[2199023258003]}}"); + + while (traversal.hasNext()) { + Map bindings = traversal.next(); + Assert.assertTrue(bindings.toString().equals(expected.get(counter))); + ++counter; + } + + Assert.assertEquals(expected.size(), (long) counter); + } + + @Test + public void run_ldbc_12_test() { + Traversal> traversal = this.get_ldbc_12_test(); + this.printTraversalForm(traversal); + + String expected = + "{v[72061992084440954]=5, v[72066390130957203]=5, v[72070788177462961]=5," + + " v[72057594037929426]=4, v[72057594037935661]=4, v[72066390130959821]=4," + + " v[72066390130959343]=3, v[72068589154214252]=2, v[72068589154207326]=1," + + " v[72081783293739876]=1}"; + Map result = traversal.next(); + Assert.assertEquals(expected, result.toString()); + Assert.assertFalse(traversal.hasNext()); + } + + public static class Traversals extends LdbcQueryTest { + + @Override + public Traversal> get_ldbc_1_test() { + return ((IrCustomizedTraversal) + g.V().hasLabel("PERSON") + .has("id", 30786325583618L) + .both("1..4", "KNOWS") + .as("p")) + .endV() + .has("id", P.neq(30786325583618L)) + .has("firstName", P.eq("Chau")) + .as("a") + .order() + .by(__.select("p").by("~len"), Order.asc) + .dedup() + .order() + .by(__.select("p").by("~len"), Order.asc) + .by(__.select("a").by("id")) + .by(__.select("a").by("lastName")) + .limit(20) + .select("a", "p") + .by(__.valueMap("id", "firstName", "lastName")) + .by("~len"); + } + + @Override + public Traversal> get_ldbc_2_test() { + return g.V().hasLabel("PERSON") + .has("id", 17592186044810L) + .both("KNOWS") + .as("p") + .in("HASCREATOR") + .has("creationDate", P.lte(20120803072025654L)) + .order() + .by("creationDate", Order.desc) + .by("id", Order.asc) + .limit(20) + .as("m") + .select("p", "m") + .by(__.valueMap("id", "firstName", "lastName")) + .by(__.valueMap("id", "imageFile", "creationDate", "content")); + } + + @Override + public Traversal get_ldbc_3_test() { + return g.V().hasLabel("PERSON") + .has("id", 17592186055119L) + .union(__.both("KNOWS"), __.both("KNOWS").both("KNOWS")) + .dedup() + .where(__.out("ISLOCATEDIN").has("name", P.without("Laos", "United_States"))) + .where( + __.in("HASCREATOR") + .has( + "creationDate", + P.gt(20110601000000000L).and(P.lt(20110713000000000L))) + .out("ISLOCATEDIN") + .has("name", P.eq("Laos").or(P.eq("Scotland"))) + .values("name") + .dedup() + .count() + .is(2)) + .order() + .by( + __.in("HASCREATOR") + .has( + "creationDate", + P.gt(20110601000000000L).and(P.lt(20110713000000000L))) + .out("ISLOCATEDIN") + .has("name", "Laos") + .count(), + Order.desc) + .by("id", Order.asc) + .limit(20); + } + + @Override + public Traversal> get_ldbc_4_test() { + return g.V().hasLabel("PERSON") + .has("id", 15393162790846L) + .as("person") + .both("KNOWS") + .in("HASCREATOR") + .hasLabel("POST") + .as("post") + .has("creationDate", P.gte(20120801000000000L).and(P.lt(20120830000000000L))) + .out("HASTAG") + .as("tag") + .select("person") + .not( + __.both("KNOWS") + .in("HASCREATOR") + .hasLabel("POST") + .has("creationDate", P.lt(20120801000000000L)) + .out("HASTAG") + .where(P.eq("tag"))) + .select("tag") + .groupCount() + .order() + .by(__.select(Column.values), Order.desc) + .by(__.select(Column.keys).values("name"), Order.asc) + .limit(10) + .select(Column.keys) + .values("name") + .as("tagName") + .select(Column.values) + .as("postCount") + .select("tagName", "postCount"); + } + + @Override + public Traversal> get_ldbc_5_test() { + return ((IrCustomizedTraversal) + g.V().hasLabel("PERSON") + .has("id", 21990232560302L) + .both("1..3", "KNOWS")) + .endV() + .dedup() + .as("p") + .inE("HASMEMBER") + .has("joinDate", P.gt(20120901000000000L)) + .outV() + .as("forum") + .out("CONTAINEROF") + .hasLabel("POST") + .out("HASCREATOR") + .where(P.eq("p")) + .select("forum") + .groupCount() + .order() + .by(__.select(Column.values), Order.desc) + .by(__.select(Column.keys).values("id"), Order.asc) + .limit(20); + } + + @Override + public Traversal> get_ldbc_6_test() { + return g.V().hasId(72088380363511554L) + .union(__.both("KNOWS"), __.both("KNOWS").both("KNOWS")) + .dedup() + .has("id", P.neq(30786325583618L)) + .in("HASCREATOR") + .hasLabel("POST") + .as("_t") + .out("HASTAG") + .has("name", P.eq("Angola")) + .select("_t") + .dedup() + .out("HASTAG") + .has("name", P.neq("Angola")) + .groupCount() + .order() + .by(__.select(Column.values), Order.desc) + .by(__.select(Column.keys).values("name"), Order.asc) + .limit(10) + .select(Column.keys) + .values("name") + .as("keys") + .select(Column.values) + .as("values") + .select("keys", "values"); + } + + @Override + public Traversal> get_ldbc_7_test() { + return g.V().hasLabel("PERSON") + .has("id", 17592186053137L) + .in("HASCREATOR") + .as("message") + .inE("LIKES") + .as("like") + .values("creationDate") + .as("likedate") + .select("like") + .outV() + .as("liker") + .order() + .by(__.select("likedate"), Order.desc) + .by("id", Order.asc) + .limit(20) + .select("message", "likedate", "liker") + .by(__.valueMap("id", "content", "imageFile")) + .by() + .by(__.valueMap("id", "firstName", "lastName")); + } + + @Override + public Traversal> get_ldbc_8_test() { + return g.V().hasLabel("PERSON") + .has("id", 17592186044810L) + .in("HASCREATOR") + .in("REPLYOF") + .hasLabel("COMMENT") + .as("comment") + .order() + .by("creationDate", Order.desc) + .by("id", Order.asc) + .limit(20) + .out("HASCREATOR") + .as("commenter") + .select("commenter", "comment") + .by(__.valueMap("id", "firstName", "lastName")) + .by(__.valueMap("creationDate", "id", "content")); + } + + @Override + public Traversal> get_ldbc_9_test() { + return ((IrCustomizedTraversal) + g.V().hasLabel("PERSON") + .has("id", 13194139542834L) + .both("1..3", "KNOWS")) + .endV() + .dedup() + .has("id", P.neq(13194139542834L)) + .as("friends") + .in("HASCREATOR") + .has("creationDate", P.lt(20111217000000000L)) + .as("post") + .order() + .by("creationDate", Order.desc) + .by("id", Order.asc) + .limit(20) + .select("friends", "post") + .by(__.valueMap("id", "firstName", "lastName")) + .by(__.valueMap("id", "content", "imageFile", "creationDate")); + } + + @Override + public Traversal> get_ldbc_11_test() { + return ((IrCustomizedTraversal) + g.V().hasLabel("PERSON") + .has("id", 30786325583618L) + .as("root") + .both("1..3", "KNOWS")) + .endV() + .dedup() + .has("id", P.neq(30786325583618L)) + .as("friends") + .outE("WORKAT") + .has("workFrom", P.lt(2010)) + .as("startWork") + .values("workFrom") + .as("works") + .select("startWork") + .inV() + .as("comp") + .values("name") + .as("orgname") + .select("comp") + .out("ISLOCATEDIN") + .has("name", "Laos") + .select("friends") + .order() + .by(__.select("works"), Order.asc) + .by("id", Order.asc) + .by(__.select("orgname"), Order.desc) + .limit(10) + .select("friends", "orgname", "works") + .by(__.valueMap("id", "firstName", "lastName")) + .by() + .by(); + } + + @Override + public Traversal> get_ldbc_12_test() { + return g.V().hasLabel("PERSON") + .has("id", 17592186044810L) + .both("KNOWS") + .as("friend") + .in("HASCREATOR") + .hasLabel("COMMENT") + .where( + __.out("REPLYOF") + .hasLabel("POST") + .out("HASTAG") + .out("HASTYPE") + .has("name", P.eq("BasketballPlayer"))) + .select("friend") + .groupCount() + .order() + .by(__.select(Column.values), Order.desc) + .by(__.select(Column.keys).values("id"), Order.asc) + .limit(20); + } + } +} diff --git a/research/query_service/ir/compiler/src/test/resources/auxilia_alias.json b/research/query_service/ir/compiler/src/test/resources/auxilia_alias.json new file mode 100644 index 000000000000..10e35a804dc7 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/auxilia_alias.json @@ -0,0 +1,25 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Auxilia": { + "params": { + "table_names": [], + "columns": [], + "limit": null, + "predicate": null, + "requirements": [] + }, + "alias": { + "item": { + "Name": "a" + } + } + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/auxilia_properties.json b/research/query_service/ir/compiler/src/test/resources/auxilia_properties.json new file mode 100644 index 000000000000..37a0c891ded4 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/auxilia_properties.json @@ -0,0 +1,33 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Auxilia": { + "tag": null, + "params": { + "table_names": [], + "columns": [ + { + "item": { + "Name": "age" + } + }, + { + "item": { + "Name": "name" + } + } + ], + "limit": null, + "predicate": null, + "requirements": [] + }, + "alias": null + } + } + }, + "children": [] + } + ] +} \ No newline at end of file diff --git a/research/query_service/ir/compiler/src/test/resources/count.json b/research/query_service/ir/compiler/src/test/resources/count.json new file mode 100644 index 000000000000..f4db0682b831 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/count.json @@ -0,0 +1,25 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "GroupBy": { + "mappings": [], + "functions": [ + { + "vars": [], + "aggregate": 3, + "alias": { + "item": { + "Name": "values" + } + } + } + ] + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/count_as.json b/research/query_service/ir/compiler/src/test/resources/count_as.json new file mode 100644 index 000000000000..409a8ee12f55 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/count_as.json @@ -0,0 +1,25 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "GroupBy": { + "mappings": [], + "functions": [ + { + "vars": [], + "aggregate": 3, + "alias": { + "item": { + "Name": "a" + } + } + } + ] + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/dedup.json b/research/query_service/ir/compiler/src/test/resources/dedup.json new file mode 100644 index 000000000000..a5bf4b39bcdf --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/dedup.json @@ -0,0 +1,19 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Dedup": { + "keys": [ + { + "tag": null, + "property": null + } + ] + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/expand_alias.json b/research/query_service/ir/compiler/src/test/resources/expand_alias.json new file mode 100644 index 000000000000..2920e62e16da --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/expand_alias.json @@ -0,0 +1,29 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Edge": { + "v_tag": null, + "direction": 0, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "is_edge": true, + "alias": { + "item": { + "Name": "a" + } + } + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/expand_edge_opt.json b/research/query_service/ir/compiler/src/test/resources/expand_edge_opt.json new file mode 100644 index 000000000000..e96f668e5a6e --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/expand_edge_opt.json @@ -0,0 +1,25 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Edge": { + "v_tag": null, + "direction": 0, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "is_edge": true, + "alias": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/expand_labels.json b/research/query_service/ir/compiler/src/test/resources/expand_labels.json new file mode 100644 index 000000000000..40a6af15f8fb --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/expand_labels.json @@ -0,0 +1,31 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Edge": { + "v_tag": null, + "direction": 0, + "params": { + "tables": [ + { + "item": { + "Name": "knows" + } + } + ], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "is_edge": true, + "alias": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/group.json b/research/query_service/ir/compiler/src/test/resources/group.json new file mode 100644 index 000000000000..50dd559d76d1 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/group.json @@ -0,0 +1,37 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "GroupBy": { + "mappings": [ + { + "key": { + "tag": null, + "property": null + }, + "alias": { + "item": { + "Name": "keys" + } + } + } + ], + "functions": [ + { + "vars": [], + "aggregate": 5, + "alias": { + "item": { + "Name": "values" + } + } + } + ] + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/group_key.json b/research/query_service/ir/compiler/src/test/resources/group_key.json new file mode 100644 index 000000000000..91370c328fc8 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/group_key.json @@ -0,0 +1,45 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "GroupBy": { + "mappings": [ + { + "key": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + }, + "alias": { + "item": { + "Name": "keys_name" + } + } + } + ], + "functions": [ + { + "vars": [], + "aggregate": 5, + "alias": { + "item": { + "Name": "values" + } + } + } + ] + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/group_key_count.json b/research/query_service/ir/compiler/src/test/resources/group_key_count.json new file mode 100644 index 000000000000..c5d2fc9c532d --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/group_key_count.json @@ -0,0 +1,45 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "GroupBy": { + "mappings": [ + { + "key": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + }, + "alias": { + "item": { + "Name": "keys_name" + } + } + } + ], + "functions": [ + { + "vars": [], + "aggregate": 3, + "alias": { + "item": { + "Name": "values" + } + } + } + ] + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/limit_range.json b/research/query_service/ir/compiler/src/test/resources/limit_range.json new file mode 100644 index 000000000000..41c85f527171 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/limit_range.json @@ -0,0 +1,17 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Limit": { + "range": { + "lower": 1, + "upper": 2 + } + } + } + }, + "children": [] + } + ] +} \ No newline at end of file diff --git a/research/query_service/ir/compiler/src/test/resources/order_asc.json b/research/query_service/ir/compiler/src/test/resources/order_asc.json new file mode 100644 index 000000000000..3f8349aec2e8 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/order_asc.json @@ -0,0 +1,23 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "OrderBy": { + "pairs": [ + { + "key": { + "tag": null, + "property": null + }, + "order": 1 + } + ], + "limit": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/order_key.json b/research/query_service/ir/compiler/src/test/resources/order_key.json new file mode 100644 index 000000000000..9c5666d08f1b --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/order_key.json @@ -0,0 +1,31 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "OrderBy": { + "pairs": [ + { + "key": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + }, + "order": 1 + } + ], + "limit": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/order_keys.json b/research/query_service/ir/compiler/src/test/resources/order_keys.json new file mode 100644 index 000000000000..df6e3a7a0353 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/order_keys.json @@ -0,0 +1,46 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "OrderBy": { + "pairs": [ + { + "key": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + }, + "order": 1 + }, + { + "key": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "id" + } + } + } + } + }, + "order": 2 + } + ], + "limit": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/order_label.json b/research/query_service/ir/compiler/src/test/resources/order_label.json new file mode 100644 index 000000000000..809ebd67f961 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/order_label.json @@ -0,0 +1,27 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "OrderBy": { + "pairs": [ + { + "key": { + "tag": null, + "property": { + "item": { + "Label": {} + } + } + }, + "order": 1 + } + ], + "limit": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/order_limit.json b/research/query_service/ir/compiler/src/test/resources/order_limit.json new file mode 100644 index 000000000000..66f4f9a864c0 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/order_limit.json @@ -0,0 +1,26 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "OrderBy": { + "pairs": [ + { + "key": { + "tag": null, + "property": null + }, + "order": 1 + } + ], + "limit": { + "lower": 1, + "upper": 2 + } + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/path_expand.json b/research/query_service/ir/compiler/src/test/resources/path_expand.json new file mode 100644 index 000000000000..9f0408941e7a --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/path_expand.json @@ -0,0 +1,34 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Path": { + "base": { + "v_tag": null, + "direction": 0, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "is_edge": false, + "alias": null + }, + "start_tag": null, + "is_whole_path": false, + "alias": null, + "hop_range": { + "lower": 1, + "upper": 5 + } + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/project_key.json b/research/query_service/ir/compiler/src/test/resources/project_key.json new file mode 100644 index 000000000000..d64fd5f37da3 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/project_key.json @@ -0,0 +1,43 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Project": { + "mappings": [ + { + "expr": { + "operators": [ + { + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + } + } + } + ] + }, + "alias": { + "item": { + "Name": "name" + } + } + } + ], + "is_append": true + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/project_tag.json b/research/query_service/ir/compiler/src/test/resources/project_tag.json new file mode 100644 index 000000000000..16627f845f97 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/project_tag.json @@ -0,0 +1,39 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Project": { + "mappings": [ + { + "expr": { + "operators": [ + { + "item": { + "Var": { + "tag": { + "item": { + "Name": "a" + } + }, + "property": null + } + } + } + ] + }, + "alias": { + "item": { + "Name": "project_a" + } + } + } + ], + "is_append": true + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/project_tag_key.json b/research/query_service/ir/compiler/src/test/resources/project_tag_key.json new file mode 100644 index 000000000000..ece4e8eaeefd --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/project_tag_key.json @@ -0,0 +1,73 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": { + "item": { + "Name": "a" + } + }, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "idx_predicate": null + } + } + }, + "children": [ + 1 + ] + }, + { + "opr": { + "opr": { + "Project": { + "mappings": [ + { + "expr": { + "operators": [ + { + "item": { + "Var": { + "tag": { + "item": { + "Name": "a" + } + }, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + } + } + } + ] + }, + "alias": { + "item": { + "Name": "a_name" + } + } + } + ], + "is_append": true + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/project_tag_keys.json b/research/query_service/ir/compiler/src/test/resources/project_tag_keys.json new file mode 100644 index 000000000000..568ea034b1c2 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/project_tag_keys.json @@ -0,0 +1,93 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": { + "item": { + "Name": "a" + } + }, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "idx_predicate": null + } + } + }, + "children": [ + 1 + ] + }, + { + "opr": { + "opr": { + "Project": { + "mappings": [ + { + "expr": { + "operators": [ + { + "item": { + "VarMap": { + "keys": [ + { + "tag": { + "item": { + "Name": "a" + } + }, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + }, + { + "tag": { + "item": { + "Name": "a" + } + }, + "property": { + "item": { + "Key": { + "item": { + "Name": "id" + } + } + } + } + } + ] + } + } + } + ] + }, + "alias": { + "item": { + "Name": "a_{name, id}" + } + } + } + ], + "is_append": true + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/scan_alias.json b/research/query_service/ir/compiler/src/test/resources/scan_alias.json new file mode 100644 index 000000000000..6636e6989e61 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/scan_alias.json @@ -0,0 +1,28 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": { + "item": { + "Name": "a" + } + }, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "idx_predicate": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/scan_expr.json b/research/query_service/ir/compiler/src/test/resources/scan_expr.json new file mode 100644 index 000000000000..bcc2c2d57229 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/scan_expr.json @@ -0,0 +1,57 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": { + "operators": [ + { + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "id" + } + } + } + } + } + } + }, + { + "item": { + "Logical": 0 + } + }, + { + "item": { + "Const": { + "item": { + "I64": 1 + } + } + } + } + ] + }, + "extra": {} + }, + "idx_predicate": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/scan_ids.json b/research/query_service/ir/compiler/src/test/resources/scan_ids.json new file mode 100644 index 000000000000..8a0934a35a9e --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/scan_ids.json @@ -0,0 +1,61 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "idx_predicate": { + "or_predicates": [ + { + "predicates": [ + { + "key": { + "item": { + "Id": {} + } + }, + "value": { + "item": { + "I64": 1 + } + }, + "cmp": null + } + ] + }, + { + "predicates": [ + { + "key": { + "item": { + "Id": {} + } + }, + "value": { + "item": { + "I64": 2 + } + }, + "cmp": null + } + ] + } + ] + } + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/scan_labels.json b/research/query_service/ir/compiler/src/test/resources/scan_labels.json new file mode 100644 index 000000000000..37626d21da78 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/scan_labels.json @@ -0,0 +1,30 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [ + { + "item": { + "Name": "person" + } + } + ], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "idx_predicate": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/scan_opt.json b/research/query_service/ir/compiler/src/test/resources/scan_opt.json new file mode 100644 index 000000000000..b656eefe13fb --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/scan_opt.json @@ -0,0 +1,24 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Scan": { + "scan_opt": 0, + "alias": null, + "params": { + "tables": [], + "columns": [], + "is_all_columns": false, + "limit": null, + "predicate": null, + "extra": {} + }, + "idx_predicate": null + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/compiler/src/test/resources/select_expr.json b/research/query_service/ir/compiler/src/test/resources/select_expr.json new file mode 100644 index 000000000000..5d4003d98966 --- /dev/null +++ b/research/query_service/ir/compiler/src/test/resources/select_expr.json @@ -0,0 +1,82 @@ +{ + "nodes": [ + { + "opr": { + "opr": { + "Select": { + "predicate": { + "operators": [ + { + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "id" + } + } + } + } + } + } + }, + { + "item": { + "Logical": 0 + } + }, + { + "item": { + "Const": { + "item": { + "I64": 1 + } + } + } + }, + { + "item": { + "Logical": 8 + } + }, + { + "item": { + "Var": { + "tag": null, + "property": { + "item": { + "Key": { + "item": { + "Name": "name" + } + } + } + } + } + } + }, + { + "item": { + "Logical": 0 + } + }, + { + "item": { + "Const": { + "item": { + "Str": "marko" + } + } + } + } + ] + } + } + } + }, + "children": [] + } + ] +} diff --git a/research/query_service/ir/core/Cargo.toml b/research/query_service/ir/core/Cargo.toml new file mode 100644 index 000000000000..7f94ff4db4c7 --- /dev/null +++ b/research/query_service/ir/core/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "ir_core" +version = "0.1.0" +edition = "2018" + +[lib] +crate-type = ["rlib", "cdylib"] + +[dependencies] +dyn_type = {path = "../../../dyn_type"} +env_logger = "0.9.0" +ir_common = {path = "../common"} +lazy_static = "1.3.0" +log = "0.4" +pegasus_client = {path = "../../../engine/pegasus/clients/rust/client"} +pegasus_server = {path = "../../../engine/pegasus/server-v0"} +pegasus = {path = "../../../engine/pegasus/pegasus"} +prost = "0.9" +serde = "1.0" +serde_json = "1.0" +vec_map = { version = "0.8.2", features = ["serde"] } + +[features] +default = [] +proto_inplace = ["ir_common/proto_inplace"] + diff --git a/research/query_service/ir/core/resource/ldbc_schema.json b/research/query_service/ir/core/resource/ldbc_schema.json new file mode 100644 index 000000000000..6fdf424e65c7 --- /dev/null +++ b/research/query_service/ir/core/resource/ldbc_schema.json @@ -0,0 +1,837 @@ +{ + "entities": [ + { + "label": { + "id": 3, + "name": "POST" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 1, + "name": "imageFile" + }, + "data_type": 4 + }, + { + "key": { + "id": 2, + "name": "creationDate" + }, + "data_type": 2 + }, + { + "key": { + "id": 3, + "name": "locationIP" + }, + "data_type": 4 + }, + { + "key": { + "id": 4, + "name": "browserUsed" + }, + "data_type": 4 + }, + { + "key": { + "id": 5, + "name": "language" + }, + "data_type": 4 + }, + { + "key": { + "id": 6, + "name": "content" + }, + "data_type": 4 + }, + { + "key": { + "id": 7, + "name": "length" + }, + "data_type": 1 + } + ] + }, + { + "label": { + "id": 11, + "name": "COMPANY" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 2, + "name": "COMMENT" + }, + "columns": [ + { + "key": { + "id": 4, + "name": "browserUsed" + }, + "data_type": 4 + }, + { + "key": { + "id": 6, + "name": "content" + }, + "data_type": 4 + }, + { + "key": { + "id": 3, + "name": "locationIP" + }, + "data_type": 4 + }, + { + "key": { + "id": 7, + "name": "length" + }, + "data_type": 1 + } + ] + }, + { + "label": { + "id": 7, + "name": "TAG" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 10, + "name": "CONTINENT" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 0, + "name": "PLACE" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 4, + "name": "FORUM" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 10, + "name": "title" + }, + "data_type": 4 + }, + { + "key": { + "id": 2, + "name": "creationDate" + }, + "data_type": 2 + } + ] + }, + { + "label": { + "id": 6, + "name": "TAGCLASS" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 1, + "name": "PERSON" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 11, + "name": "firstName" + }, + "data_type": 4 + }, + { + "key": { + "id": 12, + "name": "lastName" + }, + "data_type": 4 + }, + { + "key": { + "id": 13, + "name": "gender" + }, + "data_type": 4 + }, + { + "key": { + "id": 14, + "name": "birthday" + }, + "data_type": 2 + }, + { + "key": { + "id": 2, + "name": "creationDate" + }, + "data_type": 2 + }, + { + "key": { + "id": 3, + "name": "locationIP" + }, + "data_type": 4 + }, + { + "key": { + "id": 4, + "name": "browserUsed" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 5, + "name": "ORGANISATION" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 9, + "name": "CITY" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 12, + "name": "UNIVERSITY" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 8, + "name": "COUNTRY" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "id" + }, + "data_type": 2 + }, + { + "key": { + "id": 8, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 9, + "name": "url" + }, + "data_type": 4 + } + ] + } + ], + "relations": [ + { + "label": { + "id": 15, + "name": "STUDYAT" + }, + "entity_pairs": [ + { + "src": { + "id": 1, + "name": "PERSON" + }, + "dst": { + "id": 12, + "name": "UNIVERSITY" + } + } + ], + "columns": [ + { + "key": { + "id": 15, + "name": "classYear" + }, + "data_type": 1 + } + ] + }, + { + "label": { + "id": 16, + "name": "WORKAT" + }, + "entity_pairs": [ + { + "src": { + "id": 1, + "name": "PERSON" + }, + "dst": { + "id": 11, + "name": "COMPANY" + } + } + ], + "columns": [ + { + "key": { + "id": 16, + "name": "workFrom" + }, + "data_type": 1 + } + ] + }, + { + "label": { + "id": 17, + "name": "ISPARTOF" + }, + "entity_pairs": [ + { + "src": { + "id": 9, + "name": "CITY" + }, + "dst": { + "id": 8, + "name": "COUNTRY" + } + }, + { + "src": { + "id": 8, + "name": "COUNTRY" + }, + "dst": { + "id": 10, + "name": "CONTINENT" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 22, + "name": "ISSUBCLASSOF" + }, + "entity_pairs": [ + { + "src": { + "id": 6, + "name": "TAGCLASS" + }, + "dst": { + "id": 6, + "name": "TAGCLASS" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 6, + "name": "HASMEMBER" + }, + "entity_pairs": [ + { + "src": { + "id": 4, + "name": "FORUM" + }, + "dst": { + "id": 1, + "name": "PERSON" + } + } + ], + "columns": [ + { + "key": { + "id": 17, + "name": "joinDate" + }, + "data_type": 2 + } + ] + }, + { + "label": { + "id": 3, + "name": "REPLYOF" + }, + "entity_pairs": [ + { + "src": { + "id": 2, + "name": "COMMENT" + }, + "dst": { + "id": 2, + "name": "COMMENT" + } + }, + { + "src": { + "id": 2, + "name": "COMMENT" + }, + "dst": { + "id": 3, + "name": "POST" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 13, + "name": "LIKES" + }, + "entity_pairs": [ + { + "src": { + "id": 1, + "name": "PERSON" + }, + "dst": { + "id": 2, + "name": "COMMENT" + } + }, + { + "src": { + "id": 1, + "name": "PERSON" + }, + "dst": { + "id": 3, + "name": "POST" + } + } + ], + "columns": [ + { + "key": { + "id": 2, + "name": "creationDate" + }, + "data_type": 2 + } + ] + }, + { + "label": { + "id": 10, + "name": "HASINTEREST" + }, + "entity_pairs": [ + { + "src": { + "id": 1, + "name": "PERSON" + }, + "dst": { + "id": 7, + "name": "TAG" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 21, + "name": "HASTYPE" + }, + "entity_pairs": [ + { + "src": { + "id": 7, + "name": "TAG" + }, + "dst": { + "id": 6, + "name": "TAGCLASS" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 12, + "name": "KNOWS" + }, + "entity_pairs": [ + { + "src": { + "id": 1, + "name": "PERSON" + }, + "dst": { + "id": 1, + "name": "PERSON" + } + } + ], + "columns": [ + { + "key": { + "id": 2, + "name": "creationDate" + }, + "data_type": 2 + } + ] + }, + { + "label": { + "id": 0, + "name": "HASCREATOR" + }, + "entity_pairs": [ + { + "src": { + "id": 2, + "name": "COMMENT" + }, + "dst": { + "id": 1, + "name": "PERSON" + } + }, + { + "src": { + "id": 3, + "name": "POST" + }, + "dst": { + "id": 1, + "name": "PERSON" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 5, + "name": "CONTAINEROF" + }, + "entity_pairs": [ + { + "src": { + "id": 4, + "name": "FORUM" + }, + "dst": { + "id": 3, + "name": "POST" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 7, + "name": "HASMODERATOR" + }, + "entity_pairs": [ + { + "src": { + "id": 4, + "name": "FORUM" + }, + "dst": { + "id": 1, + "name": "PERSON" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 11, + "name": "ISLOCATEDIN" + }, + "entity_pairs": [ + { + "src": { + "id": 12, + "name": "UNIVERSITY" + }, + "dst": { + "id": 9, + "name": "CITY" + } + }, + { + "src": { + "id": 11, + "name": "COMPANY" + }, + "dst": { + "id": 8, + "name": "COUNTRY" + } + } + ], + "columns": [] + }, + { + "label": { + "id": 1, + "name": "HASTAG" + }, + "entity_pairs": [ + { + "src": { + "id": 4, + "name": "FORUM" + }, + "dst": { + "id": 7, + "name": "TAG" + } + } + ], + "columns": [] + } + ], + "is_table_id": true, + "is_column_id": false +} diff --git a/research/query_service/ir/core/resource/modern_schema.json b/research/query_service/ir/core/resource/modern_schema.json new file mode 100644 index 000000000000..a4cbc813563b --- /dev/null +++ b/research/query_service/ir/core/resource/modern_schema.json @@ -0,0 +1,106 @@ +{ + "entities": [ + { + "label": { + "id": 1, + "name": "software" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 2, + "name": "lang" + }, + "data_type": 4 + } + ] + }, + { + "label": { + "id": 0, + "name": "person" + }, + "columns": [ + { + "key": { + "id": 0, + "name": "name" + }, + "data_type": 4 + }, + { + "key": { + "id": 1, + "name": "age" + }, + "data_type": 1 + } + ] + } + ], + "relations": [ + { + "label": { + "id": 0, + "name": "knows" + }, + "entity_pairs": [ + { + "src": { + "id": 0, + "name": "person" + }, + "dst": { + "id": 0, + "name": "person" + } + } + ], + "columns": [ + { + "key": { + "id": 3, + "name": "weight" + }, + "data_type": 3 + } + ] + }, + { + "label": { + "id": 1, + "name": "created" + }, + "entity_pairs": [ + { + "src": { + "id": 0, + "name": "person" + }, + "dst": { + "id": 0, + "name": "software" + } + } + ], + "columns": [ + { + "key": { + "id": 3, + "name": "weight" + }, + "data_type": 3 + } + ] + } + ], + "is_table_id": true, + "is_column_id": false +} \ No newline at end of file diff --git a/research/query_service/ir/core/src/error.rs b/research/query_service/ir/core/src/error.rs new file mode 100644 index 000000000000..9253bb3cc6ee --- /dev/null +++ b/research/query_service/ir/core/src/error.rs @@ -0,0 +1,91 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use std::fmt; + +use ir_common::error::ParsePbError; +use ir_common::expr_parse::error::ExprError; +use ir_common::NameOrId; +use prost::EncodeError; + +/// Record any error while transforming ir to a pegasus physical plan +#[derive(Debug, Clone)] +pub enum IrError { + // Logical Errors + TableNotExist(NameOrId), + ColumnNotExist(NameOrId), + ParentNodeNotExist(u32), + TagNotExist(NameOrId), + ParsePbError(ParsePbError), + ParseExprError(ExprError), + InvalidPattern(String), + + // Physical Errors + PbEncodeError(EncodeError), + MissingData(String), + InvalidRange(i32, i32), + + // Common Errors + Unsupported(String), +} + +pub type IrResult = Result; + +impl fmt::Display for IrError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + IrError::TableNotExist(table) => { + write!(f, "the given table(label): {:?} does not exist", table) + } + IrError::ColumnNotExist(col) => { + write!(f, "the given column(property): {:?} does not exist", col) + } + IrError::TagNotExist(tag) => write!(f, "the given tag: {:?} does not exist", tag), + IrError::ParentNodeNotExist(node) => { + write!(f, "the given parent node: {:?} does not exist", node) + } + IrError::ParsePbError(err) => write!(f, "parse pb error: {:?}", err), + IrError::ParseExprError(err) => write!(f, "parse expression error: {:?}", err), + IrError::InvalidPattern(s) => write!(f, "invalid pattern: {:?}", s), + IrError::PbEncodeError(err) => write!(f, "encoding protobuf error: {:?}", err), + IrError::MissingData(s) => write!(f, "missing required data: {:?}", s), + IrError::InvalidRange(lo, up) => { + write!(f, "invalid range ({:?}, {:?})", lo, up) + } + IrError::Unsupported(s) => write!(f, "{:?}: is not supported", s), + } + } +} + +impl std::error::Error for IrError {} + +impl From for IrError { + fn from(err: ParsePbError) -> Self { + Self::ParsePbError(err) + } +} + +impl From for IrError { + fn from(err: EncodeError) -> Self { + Self::PbEncodeError(err) + } +} + +impl From for IrError { + fn from(err: ExprError) -> Self { + Self::ParseExprError(err) + } +} diff --git a/research/query_service/ir/core/src/lib.rs b/research/query_service/ir/core/src/lib.rs new file mode 100644 index 000000000000..78766e45cdf2 --- /dev/null +++ b/research/query_service/ir/core/src/lib.rs @@ -0,0 +1,37 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::io; + +pub use crate::plan::ffi::*; + +pub mod error; +pub mod plan; + +#[macro_use] +extern crate lazy_static; + +#[macro_use] +extern crate log; + +pub trait JsonIO { + /// Write the logical plan to a json via the given `writer`. + fn into_json(self, writer: W) -> io::Result<()>; + + /// Read the logical plan from a json via the given `reader` + fn from_json(reader: R) -> io::Result + where + Self: Sized; +} diff --git a/research/query_service/ir/core/src/plan/ffi.rs b/research/query_service/ir/core/src/plan/ffi.rs new file mode 100644 index 000000000000..f191cf9d7ee5 --- /dev/null +++ b/research/query_service/ir/core/src/plan/ffi.rs @@ -0,0 +1,2042 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! +//! The ffi module gives the C-like apis for the Gaia client to build the plan from the +//! query semantics, and to connect with the distributed service of Gaia. +//! +//! We instruct how to use these apis as follows. +//! +//! First of all, call `cbindgen` to generate the header of apis for C-binded caller, as: +//! `cbindgen --crate ir_core --output /path/to/c-caller/ir_core.h` +//! +//! Secondly, build the dynamic ir_core library, as: `cargo build --release`, +//! which will generate the `libir_core.dylib` under `./target/release`. +//! Copy it to `/path/to/c-caller`. +//! +//! Thirdly, write the C-code for building the ir plan, as: +//! +//! # Example +//! +//! # #include +//! # using namespace std; +//! # int main(int argc, char** argv) { +//! # const void* ptr_plan = init_logical_plan(); +//! # const void* ptr_project = init_project_operator(); +//! # add_project_mapping(ptr_project, "@name", int_as_name_or_id(0)); +//! # int opr_id = 0; +//! # append_project_operator(ptr_plan, ptr_project, 0, &opr_id); +//! # cout << "the id is: " << opr_id << endl; +//! +//! # const void* ptr_select = init_select_operator(); +//! # set_select_predicate(ptr_select, "@age > 20 && @name == \"John\""); +//! # append_select_operator(ptr_plan, ptr_select, opr_id, &opr_id); +//! # cout << "the id is: " << opr_id << endl; +//! +//! # write_plan_to_json(ptr_plan, "./plan.json"); +//! # destroy_logical_plan(ptr_plan); +//! # } +//! +//! Save the codes as , and build like: +//! `g++ -o test test.cc -std=c++11 -L. -lir_core` + +use std::convert::{TryFrom, TryInto}; +use std::ffi::{c_void, CStr}; +use std::fs::File; +use std::os::raw::c_char; + +use ir_common::expr_parse::str_to_expr_pb; +use ir_common::generated::algebra as pb; +use ir_common::generated::common as common_pb; +use pegasus::BuildJobError; +use pegasus_client::builder::JobBuilder; +use prost::Message; + +use crate::error::IrError; +use crate::plan::logical::LogicalPlan; +use crate::plan::meta::set_schema_from_json; +use crate::plan::physical::AsPhysical; +use crate::JsonIO; + +#[repr(i32)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ResultCode { + Success = 0, + /// Parse an expression error + ParseExprError = 1, + /// Missing necessary data + MissingDataError = 2, + /// The error while transforming from C-like string, aka char* + CStringError = 3, + /// The provided data type is unknown + UnknownTypeError = 4, + /// The provided range is invalid + InvalidRangeError = 5, + /// The given index is negative + NegativeIndexError = 6, + /// Build Physical Plan Error + BuildJobError = 7, + /// Parse protobuf error + ParsePbError = 8, + /// The parent of an operator cannot be found + ParentNotFoundError = 9, + /// A column (property) does not exist in the store + ColumnNotExistError = 10, + /// A table (label) does not exist in the store + TableNotExistError = 11, + /// A queried tag has not been specified + TagNotExistError = 12, + UnSupported = 13, + Others = 16, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FfiError { + code: ResultCode, + msg: *const c_char, +} + +impl FfiError { + pub fn new(code: ResultCode, msg: String) -> Self { + let result = string_to_cstr(msg); + if let Ok(msg) = result { + Self { code, msg } + } else { + result.err().unwrap() + } + } + + pub fn success() -> Self { + Self { code: ResultCode::Success, msg: std::ptr::null::() } + } +} + +impl std::fmt::Display for FfiError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("FfiError") + .field("code", &self.code) + .field("msg", &cstr_to_string(self.msg)) + .finish() + } +} + +impl std::error::Error for FfiError {} + +impl From for FfiError { + fn from(err: IrError) -> Self { + match err { + IrError::TableNotExist(t) => { + FfiError::new(ResultCode::TableNotExistError, format!("table {:?} does not exist", t)) + } + IrError::ColumnNotExist(col) => { + FfiError::new(ResultCode::ColumnNotExistError, format!("column {:?} does not exist", col)) + } + IrError::ParentNodeNotExist(p) => FfiError::new( + ResultCode::ParentNotFoundError, + format!("parent node {:?} does not exist", p), + ), + IrError::TagNotExist(t) => FfiError::new( + ResultCode::TagNotExistError, + format!("the queried tag {:?} is not specified", t), + ), + IrError::ParsePbError(err) => FfiError::new(ResultCode::ParsePbError, err.to_string()), + IrError::ParseExprError(err) => FfiError::new(ResultCode::ParseExprError, err.to_string()), + IrError::InvalidPattern(s) => FfiError::new(ResultCode::Others, s), + IrError::PbEncodeError(err) => FfiError::new(ResultCode::Others, err.to_string()), + IrError::MissingData(d) => { + FfiError::new(ResultCode::MissingDataError, format!("required data {:?} is missing", d)) + } + IrError::InvalidRange(l, u) => FfiError::new( + ResultCode::InvalidRangeError, + format!("the range ({:?}, {:?}) is invalid", l, u), + ), + IrError::Unsupported(err) => FfiError::new(ResultCode::UnSupported, err.to_string()), + } + } +} + +impl From for FfiError { + fn from(err: BuildJobError) -> Self { + Self::new(ResultCode::BuildJobError, err.to_string()) + } +} + +#[repr(C)] +pub struct FfiData { + /// The pointer to the raw data returned from Ffi functions + ptr: *mut u8, + /// The length of the data (in bytes) allocated for `Self::ptr` + len: usize, + /// To recode any error captured from building logical/physical plan + error: FfiError, +} + +impl From for FfiData { + fn from(error: IrError) -> Self { + FfiData { ptr: std::ptr::null_mut(), len: 0, error: error.into() } + } +} + +impl From for FfiData { + fn from(error: BuildJobError) -> Self { + FfiData { ptr: std::ptr::null_mut(), len: 0, error: error.into() } + } +} + +impl From for FfiData { + fn from(error: FfiError) -> Self { + FfiData { ptr: std::ptr::null_mut(), len: 0, error } + } +} + +pub(crate) fn cstr_to_string(cstr: *const c_char) -> Result { + if !cstr.is_null() { + let str_result = unsafe { CStr::from_ptr(cstr) }.to_str(); + if let Ok(str) = str_result { + Ok(str.to_string()) + } else { + Err(FfiError::new( + ResultCode::CStringError, + "error parsing C string into Rust string".to_string(), + )) + } + } else { + Ok("".to_string()) + } +} + +pub(crate) fn string_to_cstr(str: String) -> Result<*const c_char, FfiError> { + let str_result = std::ffi::CString::new(str); + if let Ok(str) = str_result { + let str_ptr = str.as_ptr(); + // **NOTE** must release the string pointer on the C side. + std::mem::forget(str); + + Ok(str_ptr) + } else { + Err(FfiError::new(ResultCode::CStringError, "error parsing Rust string into C string".to_string())) + } +} + +pub(crate) fn cstr_to_expr_pb(cstr: *const c_char) -> Result { + let str = cstr_to_string(cstr); + if str.is_err() { + Err(str.err().unwrap()) + } else { + let expr = str_to_expr_pb(str.unwrap()).map_err(|err| IrError::from(err))?; + Ok(expr) + } +} + +#[repr(i32)] +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum FfiNameIdOpt { + None = 0, + Name = 1, + Id = 2, +} + +impl Default for FfiNameIdOpt { + fn default() -> Self { + Self::None + } +} + +#[repr(C)] +pub struct FfiNameOrId { + opt: FfiNameIdOpt, + name: *const c_char, + name_id: i32, +} + +impl Default for FfiNameOrId { + fn default() -> Self { + Self { opt: FfiNameIdOpt::default(), name: std::ptr::null() as *const c_char, name_id: 0 } + } +} + +impl TryFrom for Option { + type Error = FfiError; + + fn try_from(ffi: FfiNameOrId) -> Result { + match &ffi.opt { + FfiNameIdOpt::None => Ok(None), + FfiNameIdOpt::Name => Ok(Some(common_pb::NameOrId { + item: Some(common_pb::name_or_id::Item::Name(cstr_to_string(ffi.name)?)), + })), + FfiNameIdOpt::Id => { + Ok(Some(common_pb::NameOrId { item: Some(common_pb::name_or_id::Item::Id(ffi.name_id)) })) + } + } + } +} + +impl TryFrom for common_pb::NameOrIdKey { + type Error = FfiError; + + fn try_from(ffi: FfiNameOrId) -> Result { + match &ffi.opt { + FfiNameIdOpt::None => Ok(common_pb::NameOrIdKey { key: None }), + FfiNameIdOpt::Name => Ok(common_pb::NameOrIdKey { + key: Some(common_pb::NameOrId { + item: Some(common_pb::name_or_id::Item::Name(cstr_to_string(ffi.name)?)), + }), + }), + FfiNameIdOpt::Id => Ok(common_pb::NameOrIdKey { + key: Some(common_pb::NameOrId { item: Some(common_pb::name_or_id::Item::Id(ffi.name_id)) }), + }), + } + } +} + +#[repr(i32)] +#[derive(Copy, Clone)] +pub enum FfiPropertyOpt { + None = 0, + Id = 1, + Label = 2, + Len = 3, + Key = 4, +} + +impl Default for FfiPropertyOpt { + fn default() -> Self { + Self::None + } +} + +#[repr(C)] +#[derive(Default)] +pub struct FfiProperty { + opt: FfiPropertyOpt, + key: FfiNameOrId, +} + +impl TryFrom for Option { + type Error = FfiError; + + fn try_from(ffi: FfiProperty) -> Result { + let result = match &ffi.opt { + FfiPropertyOpt::None => None, + FfiPropertyOpt::Id => { + Some(common_pb::Property { item: Some(common_pb::property::Item::Id(common_pb::IdKey {})) }) + } + FfiPropertyOpt::Label => Some(common_pb::Property { + item: Some(common_pb::property::Item::Label(common_pb::LabelKey {})), + }), + FfiPropertyOpt::Len => Some(common_pb::Property { + item: Some(common_pb::property::Item::Len(common_pb::LengthKey {})), + }), + FfiPropertyOpt::Key => { + if let Some(key) = ffi.key.try_into()? { + Some(common_pb::Property { item: Some(common_pb::property::Item::Key(key)) }) + } else { + None + } + } + }; + + Ok(result) + } +} + +#[repr(C)] +#[derive(Default)] +pub struct FfiVariable { + tag: FfiNameOrId, + property: FfiProperty, +} + +impl TryFrom for common_pb::Variable { + type Error = FfiError; + + fn try_from(ffi: FfiVariable) -> Result { + let (tag, property) = (ffi.tag.try_into()?, ffi.property.try_into()?); + Ok(Self { tag, property }) + } +} + +#[repr(C)] +#[derive(Default)] +pub struct FfiAlias { + alias: FfiNameOrId, + is_query_given: bool, +} + +impl TryFrom for Option { + type Error = FfiError; + + fn try_from(ffi: FfiAlias) -> Result { + Self::try_from(ffi.alias) + } +} + +/// Build a none-`NameOrId` +#[no_mangle] +pub extern "C" fn none_name_or_id() -> FfiNameOrId { + FfiNameOrId { opt: FfiNameIdOpt::None, name: std::ptr::null(), name_id: 0 } +} + +/// Transform a c-like string into `NameOrId` +#[no_mangle] +pub extern "C" fn cstr_as_name_or_id(cstr: *const c_char) -> FfiNameOrId { + FfiNameOrId { opt: FfiNameIdOpt::Name, name: cstr, name_id: 0 } +} + +/// Transform an integer into `NameOrId`. +#[no_mangle] +pub extern "C" fn int_as_name_or_id(integer: i32) -> FfiNameOrId { + FfiNameOrId { opt: FfiNameIdOpt::Id, name: std::ptr::null(), name_id: integer } +} + +/// Build an `None` property +#[no_mangle] +pub extern "C" fn as_none_key() -> FfiProperty { + FfiProperty::default() +} + +/// Build an id property +#[no_mangle] +pub extern "C" fn as_id_key() -> FfiProperty { + FfiProperty { opt: FfiPropertyOpt::Id, key: FfiNameOrId::default() } +} + +/// Build a label property +#[no_mangle] +pub extern "C" fn as_label_key() -> FfiProperty { + FfiProperty { opt: FfiPropertyOpt::Label, key: FfiNameOrId::default() } +} + +/// Build a length property +#[no_mangle] +pub extern "C" fn as_len_key() -> FfiProperty { + FfiProperty { opt: FfiPropertyOpt::Len, key: FfiNameOrId::default() } +} + +/// Build a keyed property from a given key +#[no_mangle] +pub extern "C" fn as_property_key(key: FfiNameOrId) -> FfiProperty { + FfiProperty { opt: FfiPropertyOpt::Key, key } +} + +/// Build a variable with tag only +#[no_mangle] +pub extern "C" fn as_var_tag_only(tag: FfiNameOrId) -> FfiVariable { + FfiVariable { tag, property: FfiProperty::default() } +} + +/// Build a variable with property only +#[no_mangle] +pub extern "C" fn as_var_property_only(property: FfiProperty) -> FfiVariable { + FfiVariable { tag: FfiNameOrId::default(), property } +} + +/// Build a variable with tag and property +#[no_mangle] +pub extern "C" fn as_var(tag: FfiNameOrId, property: FfiProperty) -> FfiVariable { + FfiVariable { tag, property } +} + +/// Build a default variable with `None` tag and property +#[no_mangle] +pub extern "C" fn as_none_var() -> FfiVariable { + FfiVariable::default() +} + +fn destroy_ptr(ptr: *const c_void) { + if !ptr.is_null() { + unsafe { + let _ = Box::from_raw(ptr as *mut M); + } + } +} + +#[derive(Clone, Copy, Debug)] +#[repr(i32)] +pub enum FfiDataType { + Unknown = 0, + Boolean = 1, + I32 = 2, + I64 = 3, + F64 = 4, + Str = 5, + I32Array = 6, + I64Array = 7, + F64Array = 8, + StrArray = 9, +} + +#[derive(Clone)] +#[repr(C)] +pub struct FfiConst { + data_type: FfiDataType, + boolean: bool, + int32: i32, + int64: i64, + float64: f64, + cstr: *const c_char, + raw: *const c_void, +} + +impl Default for FfiConst { + fn default() -> Self { + FfiConst { + data_type: FfiDataType::Unknown, + boolean: false, + int32: 0, + int64: 0, + float64: 0.0, + cstr: std::ptr::null::(), + raw: std::ptr::null::(), + } + } +} + +impl TryFrom for common_pb::Value { + type Error = FfiError; + + fn try_from(ffi: FfiConst) -> Result { + match &ffi.data_type { + FfiDataType::Boolean => Ok(common_pb::Value::from(ffi.boolean)), + FfiDataType::I32 => Ok(common_pb::Value::from(ffi.int32)), + FfiDataType::I64 => Ok(common_pb::Value::from(ffi.int64)), + FfiDataType::F64 => Ok(common_pb::Value::from(ffi.float64)), + FfiDataType::Str => { + let str = cstr_to_string(ffi.cstr); + if str.is_ok() { + Ok(common_pb::Value::from(str.unwrap())) + } else { + Err(str.err().unwrap()) + } + } + // TODO(longbin) add support for other type + _ => Err(FfiError::new( + ResultCode::UnknownTypeError, + format!("unknown data type {:?}", ffi.data_type), + )), + } + } +} + +#[no_mangle] +pub extern "C" fn boolean_as_const(boolean: bool) -> FfiConst { + let mut ffi = FfiConst::default(); + ffi.data_type = FfiDataType::Boolean; + ffi.boolean = boolean; + ffi +} + +#[no_mangle] +pub extern "C" fn int32_as_const(int32: i32) -> FfiConst { + let mut ffi = FfiConst::default(); + ffi.data_type = FfiDataType::I32; + ffi.int32 = int32; + ffi +} + +#[no_mangle] +pub extern "C" fn int64_as_const(int64: i64) -> FfiConst { + let mut ffi = FfiConst::default(); + ffi.data_type = FfiDataType::I64; + ffi.int64 = int64; + ffi +} + +#[no_mangle] +pub extern "C" fn f64_as_const(float64: f64) -> FfiConst { + let mut ffi = FfiConst::default(); + ffi.data_type = FfiDataType::F64; + ffi.float64 = float64; + ffi +} + +#[no_mangle] +pub extern "C" fn cstr_as_const(cstr: *const c_char) -> FfiConst { + let mut ffi = FfiConst::default(); + ffi.data_type = FfiDataType::Str; + ffi.cstr = cstr; + ffi +} + +/// Set schema via a json-formatted cstring. +#[no_mangle] +pub extern "C" fn set_schema(cstr_json: *const c_char) -> FfiError { + let result = cstr_to_string(cstr_json); + if let Ok(json) = result { + set_schema_from_json(json.as_bytes()); + + FfiError::success() + } else { + result.err().unwrap() + } +} + +/// Initialize a logical plan, which expose a pointer for c-like program to access the +/// entry of the logical plan. This pointer, however, is owned by Rust, and the caller +/// **must not** process any operation, which includes but not limited to deallocate it. +/// We have provided the [`destroy_logical_plan`] api for deallocating the pointer of the logical plan. +#[no_mangle] +pub extern "C" fn init_logical_plan() -> *const c_void { + use super::meta::STORE_META; + let mut plan = Box::new(LogicalPlan::default()); + if let Ok(meta) = STORE_META.read() { + if let Some(schema) = &meta.schema { + plan.meta = plan + .meta + .with_store_conf(schema.is_table_id(), schema.is_column_id()); + } + } + Box::into_raw(plan) as *const c_void +} + +/// To destroy a logical plan. +#[no_mangle] +pub extern "C" fn destroy_logical_plan(ptr_plan: *const c_void) { + destroy_ptr::(ptr_plan) +} + +/// To release a FfiError +#[no_mangle] +pub extern "C" fn destroy_ffi_error(error: FfiError) { + if !error.msg.is_null() { + let _ = unsafe { std::ffi::CString::from_raw(error.msg as *mut c_char) }; + } +} + +/// To release a FfiData +#[no_mangle] +pub extern "C" fn destroy_ffi_data(data: FfiData) { + if !data.ptr.is_null() { + let _ = unsafe { Vec::from_raw_parts(data.ptr, data.len, data.len) }; + } + if !data.error.msg.is_null() { + let _ = unsafe { std::ffi::CString::from_raw(data.error.msg as *mut c_char) }; + } +} + +/// To build a physical plan from the logical plan. +#[no_mangle] +pub extern "C" fn build_physical_plan( + ptr_plan: *const c_void, num_workers: u32, num_servers: u32, +) -> FfiData { + let mut plan = unsafe { Box::from_raw(ptr_plan as *mut LogicalPlan) }; + if num_workers > 1 || num_servers > 1 { + plan.meta = plan.meta.with_partition(); + } + let mut plan_meta = plan.meta.clone(); + let mut builder = JobBuilder::default(); + let build_result = plan.add_job_builder(&mut builder, &mut plan_meta); + let result = if build_result.is_ok() { + let req_result = builder.build(); + if let Ok(req) = req_result { + let mut req_bytes = req.encode_to_vec().into_boxed_slice(); + let data = + FfiData { ptr: req_bytes.as_mut_ptr(), len: req_bytes.len(), error: FfiError::success() }; + std::mem::forget(req_bytes); + + data + } else { + req_result.err().unwrap().into() + } + } else { + build_result.err().unwrap().into() + }; + + std::mem::forget(plan); + + result +} + +fn append_operator( + ptr_plan: *const c_void, operator: pb::logical_plan::Operator, parent_ids: Vec, id: *mut i32, +) -> FfiError { + let mut plan = unsafe { Box::from_raw(ptr_plan as *mut LogicalPlan) }; + let result = plan.append_operator_as_node( + operator, + parent_ids + .into_iter() + .filter_map(|x| if x >= 0 { Some(x as u32) } else { None }) + .collect(), + ); + if result.is_err() { + std::mem::forget(plan); + result.err().unwrap().into() + } else { + // Do not let rust drop the pointer before explicitly calling `destroy_logical_plan` + std::mem::forget(plan); + unsafe { *id = result.unwrap() as i32 }; + FfiError::success() + } +} + +#[no_mangle] +pub extern "C" fn write_plan_to_json(ptr_plan: *const c_void, cstr_file: *const c_char) { + let box_plan = unsafe { Box::from_raw(ptr_plan as *mut LogicalPlan) }; + let plan = box_plan.as_ref().clone(); + let file = cstr_to_string(cstr_file).expect("C String to Rust String error!"); + plan.into_json(File::create(&file).expect(&format!("Create json file: {:?} error", file))) + .expect("Write to json error"); + + std::mem::forget(box_plan); +} + +/// Define the target operator/parameter while setting certain parameters +#[allow(dead_code)] +#[derive(PartialEq, Copy, Clone)] +enum Target { + Select, + Scan, + GetV, + EdgeExpand, + PathExpand, + Limit, + As, + OrderBy, + Apply, + Sink, + Params, +} + +/// Set the size range limitation for certain operators +fn set_range(ptr: *const c_void, lower: i32, upper: i32, target: Target) -> FfiError { + if lower < 0 || upper < 0 || upper < lower { + FfiError::new( + ResultCode::InvalidRangeError, + format!("the range ({:?}, {:?}) is invalid", lower, upper), + ) + } else { + match target { + Target::Limit => { + let mut limit = unsafe { Box::from_raw(ptr as *mut pb::Limit) }; + limit.range = Some(pb::Range { lower, upper }); + std::mem::forget(limit); + } + Target::OrderBy => { + let mut orderby = unsafe { Box::from_raw(ptr as *mut pb::OrderBy) }; + orderby.limit = Some(pb::Range { lower, upper }); + std::mem::forget(orderby); + } + Target::Params => { + let mut params = unsafe { Box::from_raw(ptr as *mut pb::QueryParams) }; + params.limit = Some(pb::Range { lower, upper }); + std::mem::forget(params); + } + Target::PathExpand => { + let mut pathxpd = unsafe { Box::from_raw(ptr as *mut pb::PathExpand) }; + pathxpd.hop_range = Some(pb::Range { lower, upper }); + std::mem::forget(pathxpd); + } + _ => unreachable!(), + } + + FfiError::success() + } +} + +fn set_alias(ptr: *const c_void, alias: FfiAlias, target: Target) -> FfiError { + let alias_pb = alias.try_into(); + if alias_pb.is_ok() { + match target { + Target::Scan => { + let mut scan = unsafe { Box::from_raw(ptr as *mut pb::Scan) }; + scan.alias = alias_pb.unwrap(); + std::mem::forget(scan); + } + Target::EdgeExpand => { + let mut edgexpd = unsafe { Box::from_raw(ptr as *mut pb::EdgeExpand) }; + edgexpd.alias = alias_pb.unwrap(); + std::mem::forget(edgexpd); + } + Target::PathExpand => { + let mut pathxpd = unsafe { Box::from_raw(ptr as *mut pb::PathExpand) }; + pathxpd.alias = alias_pb.unwrap(); + std::mem::forget(pathxpd); + } + Target::GetV => { + let mut getv = unsafe { Box::from_raw(ptr as *mut pb::GetV) }; + getv.alias = alias_pb.unwrap(); + std::mem::forget(getv); + } + Target::Apply => { + let mut apply = unsafe { Box::from_raw(ptr as *mut pb::Apply) }; + apply.alias = alias_pb.unwrap(); + std::mem::forget(apply); + } + Target::As => { + let mut as_opr = unsafe { Box::from_raw(ptr as *mut pb::As) }; + as_opr.alias = alias_pb.unwrap(); + std::mem::forget(as_opr); + } + _ => unreachable!(), + } + FfiError::success() + } else { + alias_pb.err().unwrap() + } +} + +/// To set an operator's predicate. +fn set_predicate(ptr: *const c_void, cstr_predicate: *const c_char, target: Target) -> FfiError { + let predicate_pb = cstr_to_expr_pb(cstr_predicate); + if predicate_pb.is_ok() { + match target { + Target::Select => { + let mut select = unsafe { Box::from_raw(ptr as *mut pb::Select) }; + select.predicate = predicate_pb.ok(); + std::mem::forget(select); + } + Target::Params => { + let mut params = unsafe { Box::from_raw(ptr as *mut pb::QueryParams) }; + params.predicate = predicate_pb.ok(); + std::mem::forget(params); + } + _ => unreachable!(), + } + FfiError::success() + } else { + predicate_pb.err().unwrap() + } +} + +fn set_tag(ptr: *const c_void, tag: FfiNameOrId, target: Target) -> FfiError { + let pb = tag.try_into(); + if pb.is_ok() { + match target { + Target::EdgeExpand => { + let mut expand = unsafe { Box::from_raw(ptr as *mut pb::EdgeExpand) }; + expand.v_tag = pb.unwrap(); + std::mem::forget(expand); + } + Target::GetV => { + let mut getv = unsafe { Box::from_raw(ptr as *mut pb::GetV) }; + getv.tag = pb.unwrap(); + std::mem::forget(getv); + } + Target::PathExpand => { + let mut pathxpd = unsafe { Box::from_raw(ptr as *mut pb::PathExpand) }; + pathxpd.start_tag = pb.unwrap(); + std::mem::forget(pathxpd); + } + _ => unreachable!(), + } + FfiError::success() + } else { + pb.err().unwrap() + } +} + +mod params { + use std::collections::HashMap; + + use super::*; + + /// To initialize a query parameters + #[no_mangle] + pub extern "C" fn init_query_params() -> *const c_void { + let query_params = Box::new(pb::QueryParams { + tables: vec![], + columns: vec![], + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + }); + + Box::into_raw(query_params) as *const c_void + } + + #[no_mangle] + pub extern "C" fn add_params_table(ptr_params: *const c_void, table: FfiNameOrId) -> FfiError { + let mut params = unsafe { Box::from_raw(ptr_params as *mut pb::QueryParams) }; + let pb = table.try_into(); + if pb.is_ok() { + if let Some(table) = pb.unwrap() { + params.tables.push(table) + } + std::mem::forget(params); + + FfiError::success() + } else { + pb.err().unwrap() + } + } + + #[no_mangle] + pub extern "C" fn add_params_column(ptr_params: *const c_void, col: FfiNameOrId) -> FfiError { + let mut params = unsafe { Box::from_raw(ptr_params as *mut pb::QueryParams) }; + let pb = col.try_into(); + if pb.is_ok() { + if let Some(col) = pb.unwrap() { + params.columns.push(col) + } + std::mem::forget(params); + + FfiError::success() + } else { + pb.err().unwrap() + } + } + + #[no_mangle] + pub extern "C" fn set_params_range(ptr_params: *const c_void, lower: i32, upper: i32) -> FfiError { + set_range(ptr_params, lower, upper, Target::Params) + } + + #[no_mangle] + pub extern "C" fn set_params_predicate( + ptr_params: *const c_void, cstr_pred: *const c_char, + ) -> FfiError { + set_predicate(ptr_params, cstr_pred, Target::Params) + } + + /// Set getting all columns + #[no_mangle] + pub extern "C" fn set_params_is_all_columns(ptr_params: *const c_void) -> FfiError { + let mut params = unsafe { Box::from_raw(ptr_params as *mut pb::QueryParams) }; + params.is_all_columns = true; + std::mem::forget(params); + + FfiError::success() + } +} + +mod project { + use super::*; + /// To initialize a project operator. + #[no_mangle] + pub extern "C" fn init_project_operator(is_append: bool) -> *const c_void { + let project = Box::new(pb::Project { mappings: vec![], is_append }); + Box::into_raw(project) as *const c_void + } + + /// To add a mapping for the project operator, which maps a c-like string to represent an + /// expression, to a `NameOrId` parameter that represents an alias. + #[no_mangle] + pub extern "C" fn add_project_expr_alias( + ptr_project: *const c_void, cstr_expr: *const c_char, alias: FfiAlias, + ) -> FfiError { + let mut result = FfiError::success(); + let mut project = unsafe { Box::from_raw(ptr_project as *mut pb::Project) }; + let expr_pb = cstr_to_expr_pb(cstr_expr); + let alias_pb = Option::::try_from(alias); + + if !expr_pb.is_ok() { + result = expr_pb.err().unwrap(); + } else if !alias_pb.is_ok() { + result = alias_pb.err().unwrap(); + } else { + let attribute = pb::project::ExprAlias { expr: expr_pb.ok(), alias: alias_pb.unwrap() }; + project.mappings.push(attribute); + } + std::mem::forget(project); + + result + } + + /// Append a project operator to the logical plan. To do so, one specifies the following arguments: + /// * `ptr_plan`: A rust-owned pointer created by `init_logical_plan()`. + /// * `ptr_project`: A rust-owned pointer created by `init_project_operator()`. + /// * `parent_id`: The unique parent operator's index in the logical plan, which must be present + /// except when a negative id is provided to bypass the setting of parent operator. + /// * `id`: An index pointer that gonna hold the index of this operator. + /// + /// If it is successful to be appended to the logical plan, the `ptr_project` will be + /// automatically released by the rust program. Therefore, the caller needs not to deallocate + /// the pointer, and must **not** use it thereafter. + /// + /// Otherwise, user can manually call [`destroy_project_operator()`] to release the pointer. + /// + /// # Return + /// * Returning [`FfiError`] to capture any error. + /// + /// **Note**: All following `append_xx_operator()` apis have the same usage as this one. + /// + #[no_mangle] + pub extern "C" fn append_project_operator( + ptr_plan: *const c_void, ptr_project: *const c_void, parent_id: i32, id: *mut i32, + ) -> FfiError { + let project = unsafe { Box::from_raw(ptr_project as *mut pb::Project) }; + append_operator(ptr_plan, project.as_ref().clone().into(), vec![parent_id], id) + } + + #[no_mangle] + pub extern "C" fn destroy_project_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod select { + use super::*; + + /// To initialize a select operator + #[no_mangle] + pub extern "C" fn init_select_operator() -> *const c_void { + let select = Box::new(pb::Select { predicate: None }); + Box::into_raw(select) as *const c_void + } + + /// To set a select operator's metadata, which is a predicate represented as a c-string. + #[no_mangle] + pub extern "C" fn set_select_predicate( + ptr_select: *const c_void, cstr_predicate: *const c_char, + ) -> FfiError { + set_predicate(ptr_select, cstr_predicate, Target::Select) + } + + /// Append a select operator to the logical plan + #[no_mangle] + pub extern "C" fn append_select_operator( + ptr_plan: *const c_void, ptr_select: *const c_void, parent_id: i32, id: *mut i32, + ) -> FfiError { + let select = unsafe { Box::from_raw(ptr_select as *mut pb::Select) }; + append_operator(ptr_plan, select.as_ref().clone().into(), vec![parent_id], id) + } + + #[no_mangle] + pub extern "C" fn destroy_select_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod join { + use super::*; + + #[allow(dead_code)] + #[repr(i32)] + #[derive(Copy, Clone, Debug)] + pub enum FfiJoinKind { + /// Inner join + Inner = 0, + /// Left outer join + LeftOuter = 1, + /// Right outer join + RightOuter = 2, + /// Full outer join + FullOuter = 3, + /// Left semi-join, right alternative can be naturally adapted + Semi = 4, + /// Left anti-join, right alternative can be naturally adapted + Anti = 5, + /// aka. Cartesian product + Times = 6, + } + + /// To initialize a join operator + #[no_mangle] + pub extern "C" fn init_join_operator(join_kind: FfiJoinKind) -> *const c_void { + let kind = unsafe { std::mem::transmute(join_kind) }; + let join = Box::new(pb::Join { left_keys: vec![], right_keys: vec![], kind }); + Box::into_raw(join) as *const c_void + } + + /// To add a join operator's metadata, which is a pair of left and right keys. + /// In the join processing, a pair of data will be output if the corresponding fields + /// regarding left and right keys are **equivalent**. + #[no_mangle] + pub extern "C" fn add_join_key_pair( + ptr_join: *const c_void, left_key: FfiVariable, right_key: FfiVariable, + ) -> FfiError { + let mut result = FfiError::success(); + let mut join = unsafe { Box::from_raw(ptr_join as *mut pb::Join) }; + let left_key_pb = left_key.try_into(); + let right_key_pb = right_key.try_into(); + if left_key_pb.is_err() { + result = left_key_pb.err().unwrap(); + } else if right_key_pb.is_err() { + result = right_key_pb.err().unwrap(); + } else { + join.left_keys.push(left_key_pb.unwrap()); + join.right_keys.push(right_key_pb.unwrap()); + } + std::mem::forget(join); + + result + } + + /// Append a join operator to the logical plan. Note that both left and right parent ids + /// for join must be non-negative, and they must refer some nodes in the logical plan + #[no_mangle] + pub extern "C" fn append_join_operator( + ptr_plan: *const c_void, ptr_join: *const c_void, parent_left: i32, parent_right: i32, id: *mut i32, + ) -> FfiError { + if parent_left < 0 || parent_right < 0 { + FfiError::new( + ResultCode::NegativeIndexError, + format!("invalid left parent {:?}, or right {:?}", parent_left, parent_right), + ) + } else { + let join = unsafe { Box::from_raw(ptr_join as *mut pb::Join) }; + append_operator(ptr_plan, join.as_ref().clone().into(), vec![parent_left, parent_right], id) + } + } + + #[no_mangle] + pub extern "C" fn destroy_join_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod union { + use super::*; + + /// To initialize a union operator + #[no_mangle] + pub extern "C" fn init_union_operator() -> *const c_void { + let union = Box::new(pb::Union { parents: vec![] }); + Box::into_raw(union) as *const c_void + } + + /// Add the subtask parent id to Union + #[no_mangle] + pub extern "C" fn add_union_parent(ptr_union: *const c_void, parent_id: i32) -> FfiError { + let mut union = unsafe { Box::from_raw(ptr_union as *mut pb::Union) }; + union.parents.push(parent_id); + std::mem::forget(union); + + FfiError::success() + } + + /// Append a Union operator to the logical plan + #[no_mangle] + pub extern "C" fn append_union_operator( + ptr_plan: *const c_void, ptr_union: *const c_void, id: *mut i32, + ) -> FfiError { + let union_opr = unsafe { Box::from_raw(ptr_union as *mut pb::Union) }; + append_operator(ptr_plan, union_opr.as_ref().clone().into(), union_opr.parents, id) + } + + #[no_mangle] + pub extern "C" fn destroy_union_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod groupby { + use super::*; + + /// To initialize a groupby operator + #[no_mangle] + pub extern "C" fn init_groupby_operator() -> *const c_void { + let group = Box::new(pb::GroupBy { mappings: vec![], functions: vec![] }); + Box::into_raw(group) as *const c_void + } + + #[allow(dead_code)] + #[repr(i32)] + #[derive(Clone, Copy)] + pub enum FfiAggOpt { + Sum = 0, + Min = 1, + Max = 2, + Count = 3, + CountDistinct = 4, + ToList = 5, + ToSet = 6, + Avg = 7, + } + + #[repr(C)] + pub struct FfiAggFn { + vars: *const FfiVariable, + aggregate: FfiAggOpt, + alias: FfiAlias, + } + + impl TryFrom for pb::group_by::AggFunc { + type Error = FfiError; + + fn try_from(value: FfiAggFn) -> Result { + let mut agg_fn_pb = pb::group_by::AggFunc { + vars: vec![], + aggregate: unsafe { std::mem::transmute::(value.aggregate) }, + alias: None, + }; + let (vars, alias) = (value.vars as *mut Vec, value.alias); + let vars: Box> = unsafe { Box::from_raw(vars) }; + for var in vars.into_iter() { + agg_fn_pb.vars.push(var.try_into()?) + } + agg_fn_pb.alias = alias.try_into()?; + + Ok(agg_fn_pb) + } + } + + /// Initialize an aggregate function with empty value to aggregate. + /// To add value to aggregate, call `add_agg_value()` + #[no_mangle] + pub extern "C" fn init_agg_fn(aggregate: FfiAggOpt, alias: FfiAlias) -> FfiAggFn { + let vars: Box> = Box::new(vec![]); + FfiAggFn { vars: Box::into_raw(vars) as *const FfiVariable, aggregate, alias } + } + + #[no_mangle] + pub extern "C" fn add_agg_value(agg_fn: &mut FfiAggFn, agg_var: FfiVariable) { + let mut vars = unsafe { Box::from_raw(agg_fn.vars as *mut Vec) }; + vars.push(agg_var); + std::mem::forget(vars); + } + + /// Add the key (and its alias if any) according to which the grouping is conducted + #[no_mangle] + pub extern "C" fn add_groupby_key_alias( + ptr_groupby: *const c_void, key: FfiVariable, alias: FfiAlias, + ) -> FfiError { + let mut result = FfiError::success(); + let mut group = unsafe { Box::from_raw(ptr_groupby as *mut pb::GroupBy) }; + let key_pb = key.try_into(); + let alias_pb = alias.try_into(); + + if key_pb.is_ok() && alias_pb.is_ok() { + group + .mappings + .push(pb::group_by::KeyAlias { key: key_pb.ok(), alias: alias_pb.unwrap() }); + } else { + result = key_pb.err().unwrap(); + } + std::mem::forget(group); + + result + } + + /// Add the aggregate function for each group. + #[no_mangle] + pub extern "C" fn add_groupby_agg_fn(ptr_groupby: *const c_void, agg_fn: FfiAggFn) -> FfiError { + let mut result = FfiError::success(); + let mut group = unsafe { Box::from_raw(ptr_groupby as *mut pb::GroupBy) }; + let agg_fn_pb = agg_fn.try_into(); + + if agg_fn_pb.is_ok() { + group + .as_mut() + .functions + .push(agg_fn_pb.unwrap()); + } else { + result = agg_fn_pb.err().unwrap(); + } + std::mem::forget(group); + + result + } + + /// Append a groupby operator to the logical plan + #[no_mangle] + pub extern "C" fn append_groupby_operator( + ptr_plan: *const c_void, ptr_groupby: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let group = unsafe { Box::from_raw(ptr_groupby as *mut pb::GroupBy) }; + append_operator(ptr_plan, group.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_groupby_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod orderby { + use super::*; + + #[allow(dead_code)] + #[repr(i32)] + #[derive(Clone, Copy)] + pub enum FfiOrderOpt { + Shuffle = 0, + Asc = 1, + Desc = 2, + } + + /// To initialize an orderby operator + #[no_mangle] + pub extern "C" fn init_orderby_operator() -> *const c_void { + let order = Box::new(pb::OrderBy { pairs: vec![], limit: None }); + Box::into_raw(order) as *const c_void + } + + /// Add the pair for conducting ordering. + #[no_mangle] + pub extern "C" fn add_orderby_pair( + ptr_orderby: *const c_void, var: FfiVariable, order_opt: FfiOrderOpt, + ) -> FfiError { + let mut result = FfiError::success(); + let mut orderby = unsafe { Box::from_raw(ptr_orderby as *mut pb::OrderBy) }; + let key_result = var.try_into(); + if key_result.is_ok() { + let order = match order_opt { + FfiOrderOpt::Shuffle => 0, + FfiOrderOpt::Asc => 1, + FfiOrderOpt::Desc => 2, + }; + orderby + .pairs + .push(pb::order_by::OrderingPair { key: key_result.ok(), order }); + } else { + result = key_result.err().unwrap(); + } + std::mem::forget(orderby); + + result + } + + /// Set the size limit of the orderby operator, which will turn it into topk + #[no_mangle] + pub extern "C" fn set_orderby_limit(ptr_orderby: *const c_void, lower: i32, upper: i32) -> FfiError { + set_range(ptr_orderby, lower, upper, Target::OrderBy) + } + + /// Append an orderby operator to the logical plan + #[no_mangle] + pub extern "C" fn append_orderby_operator( + ptr_plan: *const c_void, ptr_orderby: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let orderby = unsafe { Box::from_raw(ptr_orderby as *mut pb::OrderBy) }; + append_operator(ptr_plan, orderby.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_orderby_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod dedup { + use super::*; + + /// To initialize a dedup operator + #[no_mangle] + pub extern "C" fn init_dedup_operator() -> *const c_void { + let dedup = Box::new(pb::Dedup { keys: vec![] }); + Box::into_raw(dedup) as *const c_void + } + + /// Add a key for de-duplicating. + #[no_mangle] + pub extern "C" fn add_dedup_key(ptr_dedup: *const c_void, var: FfiVariable) -> FfiError { + let mut result = FfiError::success(); + let mut dedup = unsafe { Box::from_raw(ptr_dedup as *mut pb::Dedup) }; + let key_result = var.try_into(); + if key_result.is_ok() { + dedup.keys.push(key_result.unwrap()); + } else { + result = key_result.err().unwrap(); + } + std::mem::forget(dedup); + + result + } + + /// Append a dedup operator to the logical plan + #[no_mangle] + pub extern "C" fn append_dedup_operator( + ptr_plan: *const c_void, ptr_dedup: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let dedup = unsafe { Box::from_raw(ptr_dedup as *mut pb::Dedup) }; + append_operator(ptr_plan, dedup.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_dedup_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod unfold { + use super::*; + + /// To initialize an unfold operator + #[no_mangle] + pub extern "C" fn init_unfold_operator() -> *const c_void { + let unfold = Box::new(pb::Unfold { tag: None, alias: None }); + Box::into_raw(unfold) as *const c_void + } + + /// Set the argument pair for unfold, which are: + /// * a tag points to a collection-type data field for unfolding, + /// * an alias for referencing to each element of the collection. + #[no_mangle] + pub extern "C" fn set_unfold_pair( + ptr_unfold: *const c_void, tag: FfiNameOrId, alias: FfiNameOrId, + ) -> FfiError { + let mut result = FfiError::success(); + let mut unfold = unsafe { Box::from_raw(ptr_unfold as *mut pb::Unfold) }; + let tag_result = tag.try_into(); + let alias_result = alias.try_into(); + + if tag_result.is_ok() && alias_result.is_ok() { + unfold.tag = tag_result.unwrap(); + unfold.alias = alias_result.unwrap(); + } else { + result = + if tag_result.is_err() { tag_result.err().unwrap() } else { alias_result.err().unwrap() }; + } + std::mem::forget(unfold); + + result + } + + /// Append an unfold operator to the logical plan + #[no_mangle] + pub extern "C" fn append_unfold_operator( + ptr_plan: *const c_void, ptr_unfold: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let unfold = unsafe { Box::from_raw(ptr_unfold as *mut pb::Unfold) }; + append_operator(ptr_plan, unfold.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_unfold_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod scan { + use std::collections::HashMap; + + use super::*; + + #[allow(dead_code)] + #[derive(Copy, Clone, Debug, PartialEq)] + #[repr(i32)] + pub enum FfiScanOpt { + Entity = 0, + Relation = 1, + } + + /// To initialize a scan operator + #[no_mangle] + pub extern "C" fn init_scan_operator(scan_opt: FfiScanOpt) -> *const c_void { + let scan = Box::new(pb::Scan { + scan_opt: unsafe { std::mem::transmute::(scan_opt) }, + alias: None, + params: Some(pb::QueryParams { + tables: vec![], + columns: vec![], + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + }), + idx_predicate: None, + }); + Box::into_raw(scan) as *const c_void + } + + #[no_mangle] + pub extern "C" fn init_index_predicate() -> *const c_void { + let predicate: Box = Box::new(pb::IndexPredicate { or_predicates: vec![] }); + Box::into_raw(predicate) as *const c_void + } + + fn parse_equiv_predicate( + key: FfiProperty, value: FfiConst, + ) -> Result { + Ok(pb::index_predicate::Triplet { key: key.try_into()?, value: Some(value.try_into()?), cmp: None }) + } + + #[no_mangle] + pub extern "C" fn and_equiv_predicate( + ptr_predicate: *const c_void, key: FfiProperty, value: FfiConst, + ) -> FfiError { + let equiv_pred_result = parse_equiv_predicate(key, value); + if let Ok(equiv_pred) = equiv_pred_result { + let mut predicate = unsafe { Box::from_raw(ptr_predicate as *mut pb::IndexPredicate) }; + if predicate.or_predicates.is_empty() { + predicate + .or_predicates + .push(pb::index_predicate::AndPredicate { predicates: vec![equiv_pred] }); + } else { + predicate + .or_predicates + .last_mut() + .unwrap() + .predicates + .push(equiv_pred) + } + std::mem::forget(predicate); + + FfiError::success() + } else { + equiv_pred_result.err().unwrap() + } + } + + #[no_mangle] + pub extern "C" fn or_equiv_predicate( + ptr_predicate: *const c_void, key: FfiProperty, value: FfiConst, + ) -> FfiError { + let equiv_pred_result = parse_equiv_predicate(key, value); + if let Ok(equiv_pred) = equiv_pred_result { + let mut predicate = unsafe { Box::from_raw(ptr_predicate as *mut pb::IndexPredicate) }; + predicate + .or_predicates + .push(pb::index_predicate::AndPredicate { predicates: vec![equiv_pred] }); + std::mem::forget(predicate); + + FfiError::success() + } else { + equiv_pred_result.err().unwrap() + } + } + + #[no_mangle] + pub extern "C" fn add_scan_index_predicate( + ptr_scan: *const c_void, ptr_predicate: *const c_void, + ) -> FfiError { + let mut scan = unsafe { Box::from_raw(ptr_scan as *mut pb::Scan) }; + let predicate = unsafe { Box::from_raw(ptr_predicate as *mut pb::IndexPredicate) }; + scan.idx_predicate = Some(predicate.as_ref().clone()); + std::mem::forget(scan); + + FfiError::success() + } + + #[no_mangle] + pub extern "C" fn set_scan_params(ptr_scan: *const c_void, ptr_params: *const c_void) -> FfiError { + let mut scan = unsafe { Box::from_raw(ptr_scan as *mut pb::Scan) }; + let mut params = unsafe { Box::from_raw(ptr_params as *mut pb::QueryParams) }; + std::mem::swap(scan.params.as_mut().unwrap(), params.as_mut()); + std::mem::forget(scan); + + FfiError::success() + } + + /// Set an alias for the data if it is a vertex/edge + #[no_mangle] + pub extern "C" fn set_scan_alias(ptr_scan: *const c_void, alias: FfiAlias) -> FfiError { + set_alias(ptr_scan, alias, Target::Scan) + } + + /// Append a scan operator to the logical plan + #[no_mangle] + pub extern "C" fn append_scan_operator( + ptr_plan: *const c_void, ptr_scan: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let scan = unsafe { Box::from_raw(ptr_scan as *mut pb::Scan) }; + append_operator(ptr_plan, scan.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_scan_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod limit { + use super::*; + + #[no_mangle] + pub extern "C" fn init_limit_operator() -> *const c_void { + let limit: Box = Box::new(pb::Limit { range: None }); + Box::into_raw(limit) as *const c_void + } + + #[no_mangle] + pub extern "C" fn set_limit_range(ptr_limit: *const c_void, lower: i32, upper: i32) -> FfiError { + set_range(ptr_limit, lower, upper, Target::Limit) + } + + /// Append an indexed scan operator to the logical plan + #[no_mangle] + pub extern "C" fn append_limit_operator( + ptr_plan: *const c_void, ptr_limit: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let limit = unsafe { Box::from_raw(ptr_limit as *mut pb::Limit) }; + append_operator(ptr_plan, limit.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_limit_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod as_opr { + use super::*; + + /// To initialize an As operator + #[no_mangle] + pub extern "C" fn init_as_operator() -> *const c_void { + let as_opr = Box::new(pb::As { alias: None }); + + Box::into_raw(as_opr) as *const c_void + } + + /// Set the alias of the entity to As + #[no_mangle] + pub extern "C" fn set_as_alias(ptr_as: *const c_void, alias: FfiAlias) -> FfiError { + set_alias(ptr_as, alias, Target::As) + } + + /// Append an As operator to the logical plan + #[no_mangle] + pub extern "C" fn append_as_operator( + ptr_plan: *const c_void, ptr_as: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let as_opr = unsafe { Box::from_raw(ptr_as as *mut pb::As) }; + append_operator(ptr_plan, as_opr.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_as_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod sink { + use super::*; + + /// To initialize an Sink operator + #[no_mangle] + pub extern "C" fn init_sink_operator() -> *const c_void { + let sink_opr = Box::new(pb::Sink { tags: vec![], id_name_mappings: vec![] }); + Box::into_raw(sink_opr) as *const c_void + } + + /// Add the tag of column to output to Sink + #[no_mangle] + pub extern "C" fn add_sink_column(ptr_sink: *const c_void, ffi_tag: FfiNameOrId) -> FfiError { + let mut result = FfiError::success(); + let tag_pb = ffi_tag.try_into(); + if tag_pb.is_ok() { + let mut sink = unsafe { Box::from_raw(ptr_sink as *mut pb::Sink) }; + sink.tags.push(tag_pb.unwrap()); + std::mem::forget(sink); + } else { + result = tag_pb.err().unwrap(); + } + + result + } + + /// Append an Sink operator to the logical plan + #[no_mangle] + pub extern "C" fn append_sink_operator( + ptr_plan: *const c_void, ptr_sink: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let sink_opr = unsafe { Box::from_raw(ptr_sink as *mut pb::Sink) }; + append_operator(ptr_plan, sink_opr.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_sink_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod graph { + use std::collections::HashMap; + + use super::*; + use crate::plan::ffi::join::FfiJoinKind; + + #[allow(dead_code)] + #[derive(Copy, Clone)] + #[repr(i32)] + pub enum FfiDirection { + Out = 0, + In = 1, + Both = 2, + } + + /// To initialize an edge expand operator from an expand base + #[no_mangle] + pub extern "C" fn init_edgexpd_operator(is_edge: bool, dir: FfiDirection) -> *const c_void { + let edgexpd = Box::new(pb::EdgeExpand { + v_tag: None, + direction: unsafe { std::mem::transmute::(dir) }, + params: Some(pb::QueryParams { + tables: vec![], + columns: vec![], + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + }), + is_edge, + alias: None, + }); + + Box::into_raw(edgexpd) as *const c_void + } + + /// Set the start-vertex's tag to conduct this expansion + #[no_mangle] + pub extern "C" fn set_edgexpd_vtag(ptr_edgexpd: *const c_void, v_tag: FfiNameOrId) -> FfiError { + set_tag(ptr_edgexpd, v_tag, Target::EdgeExpand) + } + + #[no_mangle] + pub extern "C" fn set_edgexpd_params( + ptr_edgexpd: *const c_void, ptr_params: *const c_void, + ) -> FfiError { + let mut edgexpd = unsafe { Box::from_raw(ptr_edgexpd as *mut pb::EdgeExpand) }; + let mut params = unsafe { Box::from_raw(ptr_params as *mut pb::QueryParams) }; + std::mem::swap(edgexpd.params.as_mut().unwrap(), params.as_mut()); + std::mem::forget(edgexpd); + + FfiError::success() + } + + /// Set edge alias of this edge expansion + #[no_mangle] + pub extern "C" fn set_edgexpd_alias(ptr_edgexpd: *const c_void, alias: FfiAlias) -> FfiError { + set_alias(ptr_edgexpd, alias, Target::EdgeExpand) + } + + /// Append an edge expand operator to the logical plan + #[no_mangle] + pub extern "C" fn append_edgexpd_operator( + ptr_plan: *const c_void, ptr_edgexpd: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let edgexpd = unsafe { Box::from_raw(ptr_edgexpd as *mut pb::EdgeExpand) }; + append_operator(ptr_plan, edgexpd.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_edgexpd_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } + + #[allow(dead_code)] + #[repr(i32)] + pub enum FfiVOpt { + Start = 0, + End = 1, + Other = 2, + } + + /// To initialize an expansion base + #[no_mangle] + pub extern "C" fn init_getv_operator(opt: FfiVOpt) -> *const c_void { + let getv = Box::new(pb::GetV { + tag: None, + opt: unsafe { std::mem::transmute::(opt) }, + params: Some(pb::QueryParams { + tables: vec![], + columns: vec![], + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + }), + alias: None, + }); + Box::into_raw(getv) as *const c_void + } + + /// Set the tag of edge/path to get the vertex + #[no_mangle] + pub extern "C" fn set_getv_tag(ptr_getv: *const c_void, tag: FfiNameOrId) -> FfiError { + set_tag(ptr_getv, tag, Target::GetV) + } + + #[no_mangle] + pub extern "C" fn set_getv_params(ptr_getv: *const c_void, ptr_params: *const c_void) -> FfiError { + let mut getv = unsafe { Box::from_raw(ptr_getv as *mut pb::GetV) }; + let mut params = unsafe { Box::from_raw(ptr_params as *mut pb::QueryParams) }; + std::mem::swap(getv.params.as_mut().unwrap(), params.as_mut()); + std::mem::forget(getv); + + FfiError::success() + } + + /// Set vertex alias of this getting vertex + #[no_mangle] + pub extern "C" fn set_getv_alias(ptr_getv: *const c_void, alias: FfiAlias) -> FfiError { + set_alias(ptr_getv, alias, Target::GetV) + } + + /// Append the operator to the logical plan + #[no_mangle] + pub extern "C" fn append_getv_operator( + ptr_plan: *const c_void, ptr_getv: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let getv = unsafe { Box::from_raw(ptr_getv as *mut pb::GetV) }; + append_operator(ptr_plan, getv.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_getv_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } + + /// To initialize an path expand operator from an expand base + #[no_mangle] + pub extern "C" fn init_pathxpd_operator( + ptr_expand: *const c_void, is_whole_path: bool, + ) -> *const c_void { + let expand = unsafe { Box::from_raw(ptr_expand as *mut pb::EdgeExpand) }; + let edgexpd = Box::new(pb::PathExpand { + base: Some(expand.as_ref().clone()), + start_tag: None, + is_whole_path, + alias: None, + hop_range: None, + }); + + Box::into_raw(edgexpd) as *const c_void + } + + /// Set path alias of this path expansion + #[no_mangle] + pub extern "C" fn set_pathxpd_tag(ptr_pathxpd: *const c_void, tag: FfiNameOrId) -> FfiError { + set_tag(ptr_pathxpd, tag, Target::PathExpand) + } + + /// Set path alias of this path expansion + #[no_mangle] + pub extern "C" fn set_pathxpd_alias(ptr_pathxpd: *const c_void, alias: FfiAlias) -> FfiError { + set_alias(ptr_pathxpd, alias, Target::PathExpand) + } + + /// Set the hop-range limitation of expanding path + #[no_mangle] + pub extern "C" fn set_pathxpd_hops(ptr_pathxpd: *const c_void, lower: i32, upper: i32) -> FfiError { + set_range(ptr_pathxpd, lower, upper, Target::PathExpand) + } + + /// Append an path-expand operator to the logical plan + #[no_mangle] + pub extern "C" fn append_pathxpd_operator( + ptr_plan: *const c_void, ptr_pathxpd: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let pathxpd = unsafe { Box::from_raw(ptr_pathxpd as *mut pb::PathExpand) }; + append_operator(ptr_plan, pathxpd.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_pathxpd_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } + + #[no_mangle] + pub extern "C" fn init_pattern_operator() -> *const c_void { + let pattern = Box::new(pb::Pattern { sentences: vec![] }); + + Box::into_raw(pattern) as *const c_void + } + + #[no_mangle] + pub extern "C" fn add_pattern_sentence( + ptr_pattern: *const c_void, ptr_sentence: *const c_void, + ) -> FfiError { + let mut pattern = unsafe { Box::from_raw(ptr_pattern as *mut pb::Pattern) }; + let sentence = unsafe { Box::from_raw(ptr_sentence as *mut pb::pattern::Sentence) }; + pattern + .sentences + .push(sentence.as_ref().clone()); + std::mem::forget(pattern); + + FfiError::success() + } + + #[no_mangle] + pub extern "C" fn init_pattern_sentence(join_kind: FfiJoinKind) -> *const c_void { + let sentence = Box::new(pb::pattern::Sentence { + start: None, + binders: vec![], + end: None, + join_kind: unsafe { std::mem::transmute(join_kind) }, + }); + + Box::into_raw(sentence) as *const c_void + } + + fn set_sentence_tag(ptr_sentence: *const c_void, tag: FfiNameOrId, is_start: bool) -> FfiError { + let mut sentence = unsafe { Box::from_raw(ptr_sentence as *mut pb::pattern::Sentence) }; + let pb = tag.try_into(); + let result_code = if pb.is_ok() { + if is_start { + sentence.start = pb.unwrap(); + } else { + sentence.end = pb.unwrap(); + } + FfiError::success() + } else { + pb.err().unwrap() + }; + std::mem::forget(sentence); + + result_code + } + + #[no_mangle] + pub extern "C" fn set_sentence_start(ptr_sentence: *const c_void, tag: FfiNameOrId) -> FfiError { + set_sentence_tag(ptr_sentence, tag, true) + } + + #[no_mangle] + pub extern "C" fn set_sentence_end(ptr_sentence: *const c_void, tag: FfiNameOrId) -> FfiError { + set_sentence_tag(ptr_sentence, tag, false) + } + + #[derive(Copy, Clone, Debug)] + #[repr(i32)] + #[allow(dead_code)] + pub enum FfiBinderOpt { + Edge = 0, + Path = 1, + Vertex = 2, + } + + #[no_mangle] + pub extern "C" fn add_sentence_binder( + ptr_sentence: *const c_void, ptr: *const c_void, binder: FfiBinderOpt, + ) -> FfiError { + let mut sentence = unsafe { Box::from_raw(ptr_sentence as *mut pb::pattern::Sentence) }; + match binder { + FfiBinderOpt::Edge => { + let edgexpd = unsafe { Box::from_raw(ptr as *mut pb::EdgeExpand) }; + sentence.binders.push(pb::pattern::Binder { + item: Some(pb::pattern::binder::Item::Edge(edgexpd.as_ref().clone())), + }); + } + FfiBinderOpt::Path => { + let pathxpd = unsafe { Box::from_raw(ptr as *mut pb::PathExpand) }; + sentence.binders.push(pb::pattern::Binder { + item: Some(pb::pattern::binder::Item::Path(pathxpd.as_ref().clone())), + }); + } + FfiBinderOpt::Vertex => { + let getv = unsafe { Box::from_raw(ptr as *mut pb::GetV) }; + sentence.binders.push(pb::pattern::Binder { + item: Some(pb::pattern::binder::Item::Vertex(getv.as_ref().clone())), + }); + } + } + std::mem::forget(sentence); + + FfiError::success() + } + + /// Append a pattern operator to the logical plan + #[no_mangle] + pub extern "C" fn append_pattern_operator( + ptr_plan: *const c_void, ptr_pattern: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let pattern = unsafe { Box::from_raw(ptr_pattern as *mut pb::Pattern) }; + append_operator(ptr_plan, pattern.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_pattern_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} + +mod subtask { + use super::*; + use crate::plan::ffi::join::FfiJoinKind; + + /// To initialize an apply operator from a root node (id) of the subtask. Before initializing + /// the subtask, one need to first prepare the subtask and append the operators within to the + /// logical plan. + #[no_mangle] + pub extern "C" fn init_apply_operator(subtask_root: i32, join_kind: FfiJoinKind) -> *const c_void { + let apply = Box::new(pb::Apply { + join_kind: unsafe { std::mem::transmute::(join_kind) }, + tags: vec![], + subtask: subtask_root, + alias: None, + }); + + Box::into_raw(apply) as *const c_void + } + + #[no_mangle] + pub extern "C" fn add_apply_tag(ptr_apply: *const c_void, ffi_tag: FfiNameOrId) -> FfiError { + let mut result = FfiError::success(); + let tag_pb: Result, FfiError> = ffi_tag.try_into(); + if tag_pb.is_ok() { + if let Some(tag) = tag_pb.unwrap() { + let mut apply = unsafe { Box::from_raw(ptr_apply as *mut pb::Apply) }; + apply.tags.push(tag); + std::mem::forget(apply); + } + } else { + result = tag_pb.err().unwrap(); + } + + result + } + + #[no_mangle] + pub extern "C" fn set_apply_alias(ptr_apply: *const c_void, alias: FfiAlias) -> FfiError { + set_alias(ptr_apply, alias, Target::Apply) + } + + /// Append an apply operator to the logical plan. + /// If the apply is used alone (other than segment apply), the parent node must set and present + /// in the logical plan. + #[no_mangle] + pub extern "C" fn append_apply_operator( + ptr_plan: *const c_void, ptr_apply: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + let apply = unsafe { Box::from_raw(ptr_apply as *mut pb::Apply) }; + append_operator(ptr_plan, apply.as_ref().clone().into(), vec![parent], id) + } + + #[no_mangle] + pub extern "C" fn destroy_apply_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } + + /// To initialize a segment apply operator from an apply operator. + #[no_mangle] + pub extern "C" fn init_segapply_operator(ptr_apply: *const c_void) -> *const c_void { + let apply = unsafe { Box::from_raw(ptr_apply as *mut pb::Apply) }; + let segapply = + Box::new(pb::SegmentApply { keys: vec![], apply_subtask: Some(apply.as_ref().clone()) }); + + Box::into_raw(segapply) as *const c_void + } + + /// To add the key for grouping on which the segment apply can be conducted. + #[no_mangle] + pub extern "C" fn add_segapply_key(ptr_segapply: *const c_void, ffi_key: FfiNameOrId) -> FfiError { + let mut result = FfiError::success(); + let key_pb: Result, FfiError> = ffi_key.try_into(); + if key_pb.is_ok() { + if let Some(key) = key_pb.unwrap() { + let mut segapply = unsafe { Box::from_raw(ptr_segapply as *mut pb::SegmentApply) }; + segapply.keys.push(key); + std::mem::forget(segapply); + } + } else { + result = key_pb.err().unwrap(); + } + + result + } + + /// Append an apply operator to the logical plan. The parent node id for appending a segment apply operator + /// must not be negative and must present in the logical plan. + #[no_mangle] + pub extern "C" fn append_segapply_operator( + ptr_plan: *const c_void, ptr_segapply: *const c_void, parent: i32, id: *mut i32, + ) -> FfiError { + if parent < 0 { + FfiError::new(ResultCode::NegativeIndexError, format!("invalid parent id {:?}", parent)) + } else { + let segapply = unsafe { Box::from_raw(ptr_segapply as *mut pb::SegmentApply) }; + append_operator(ptr_plan, segapply.as_ref().clone().into(), vec![parent], id) + } + } + + #[no_mangle] + pub extern "C" fn destroy_segapply_operator(ptr: *const c_void) { + destroy_ptr::(ptr) + } +} diff --git a/research/query_service/ir/core/src/plan/logical.rs b/research/query_service/ir/core/src/plan/logical.rs new file mode 100644 index 000000000000..aa39e2ceb2b5 --- /dev/null +++ b/research/query_service/ir/core/src/plan/logical.rs @@ -0,0 +1,2574 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::cell::RefCell; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::convert::{TryFrom, TryInto}; +use std::fmt; +use std::io; +use std::rc::Rc; + +use ir_common::error::ParsePbError; +use ir_common::generated::algebra as pb; +use ir_common::generated::common as common_pb; +use ir_common::NameOrId; +use vec_map::VecMap; + +use crate::error::{IrError, IrResult}; +use crate::plan::meta::{PlanMeta, StoreMeta, INVALID_META_ID, STORE_META}; +use crate::plan::patmat::{MatchingStrategy, NaiveStrategy}; +use crate::JsonIO; + +/// An internal representation of the pb-[`Node`]. +/// +/// [`Node`]: crate::generated::algebra::logical_plan::Node +#[derive(Clone, Debug, PartialEq)] +pub struct Node { + pub(crate) id: u32, + pub(crate) opr: pb::logical_plan::Operator, + pub(crate) parents: BTreeSet, + pub(crate) children: BTreeSet, +} + +#[allow(dead_code)] +impl Node { + pub fn new(id: u32, opr: pb::logical_plan::Operator) -> Node { + Node { id, opr, parents: BTreeSet::new(), children: BTreeSet::new() } + } + + pub fn add_child(&mut self, child_id: u32) { + self.children.insert(child_id); + } + + pub fn get_first_child(&self) -> Option { + self.children.iter().next().cloned() + } + + pub fn add_parent(&mut self, parent_id: u32) { + self.parents.insert(parent_id); + } +} + +pub(crate) type NodeType = Rc>; + +/// An internal representation of the pb-[`LogicalPlan`]. +/// +/// [`LogicalPlan`]: crate::generated::algebra::LogicalPlan +#[derive(Default, Clone)] +pub struct LogicalPlan { + pub(crate) nodes: VecMap, + /// To record the nodes' maximum id in the logical plan. Note that the nodes + /// **include the removed ones** + pub(crate) max_node_id: u32, + /// The metadata of the logical plan + pub(crate) meta: PlanMeta, +} + +impl PartialEq for LogicalPlan { + fn eq(&self, other: &Self) -> bool { + if self.nodes.len() != other.nodes.len() { + return false; + } + for (this_node, other_node) in self + .nodes + .iter() + .map(|(_, node)| node) + .zip(other.nodes.iter().map(|(_, node)| node)) + { + if this_node != other_node { + return false; + } + } + + true + } +} + +impl fmt::Debug for LogicalPlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries(self.nodes.iter().map(|(_, node)| node.borrow())) + .finish() + } +} + +impl TryFrom for LogicalPlan { + type Error = ParsePbError; + + fn try_from(pb: pb::LogicalPlan) -> Result { + let nodes_pb = pb.nodes; + let mut plan = LogicalPlan::default(); + let mut id_map = HashMap::::new(); + let mut parents = HashMap::>::new(); + for (id, node) in nodes_pb.iter().enumerate() { + for &child in &node.children { + if child <= id as i32 { + return Err(ParsePbError::ParseError(format!( + "the child node's id {:?} is not larger than parent node's id {:?}", + child, id + ))); + } + parents + .entry(child as u32) + .or_insert_with(BTreeSet::new) + .insert(id as u32); + } + } + + for (id, node) in nodes_pb.into_iter().enumerate() { + if let Some(mut opr) = node.opr { + match opr.opr.as_mut() { + Some(pb::logical_plan::operator::Opr::Apply(apply)) => { + apply.subtask = id_map[&(apply.subtask as u32)] as i32; + } + _ => {} + } + let parent_ids = parents + .get(&(id as u32)) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|old| id_map[&old]) + .collect::>(); + let new_id = plan + .append_operator_as_node(opr, parent_ids) + .map_err(|err| ParsePbError::ParseError(format!("{:?}", err)))?; + id_map.insert(id as u32, new_id); + } else { + return Err(ParsePbError::EmptyFieldError("Node::opr".to_string())); + } + } + + Ok(plan) + } +} + +impl From for pb::LogicalPlan { + fn from(plan: LogicalPlan) -> Self { + let mut id_map: HashMap = HashMap::with_capacity(plan.len()); + // As there might be some nodes being removed, we gonna remap the nodes' ids + for (id, node) in plan.nodes.iter().enumerate() { + id_map.insert(node.0 as u32, id as i32); + } + let mut plan_pb = pb::LogicalPlan { nodes: vec![] }; + for (_, node) in &plan.nodes { + let mut node_pb = pb::logical_plan::Node { opr: None, children: vec![] }; + let mut operator = node.borrow().opr.clone(); + match operator.opr.as_mut() { + Some(pb::logical_plan::operator::Opr::Apply(apply)) => { + apply.subtask = id_map[&(apply.subtask as u32)] as i32; + } + _ => {} + } + node_pb.opr = Some(operator); + node_pb.children = node + .borrow() + .children + .iter() + .map(|old_id| id_map[old_id]) + .collect(); + plan_pb.nodes.push(node_pb); + } + + plan_pb + } +} + +fn clone_node(node: NodeType) -> Node { + let mut clone_node = (*node.borrow()).clone(); + clone_node.children.clear(); + clone_node.parents.clear(); + + clone_node +} + +// Implement some private functions +#[allow(dead_code)] +impl LogicalPlan { + /// Get the corresponding merge node of the given branch node. + fn get_merge_node(&self, branch_node: NodeType) -> Option { + if branch_node.borrow().children.len() > 1 { + let mut layer = 0; + let mut curr_node_opt = Some(branch_node.clone()); + while curr_node_opt.is_some() { + let next_node_id = curr_node_opt + .as_ref() + .unwrap() + .borrow() + .get_first_child(); + curr_node_opt = next_node_id.and_then(|id| self.get_node(id)); + if let Some(curr_node) = &curr_node_opt { + let curr_ref = curr_node.borrow(); + if curr_ref.children.len() > 1 { + layer += curr_ref.children.len(); + } + if curr_ref.parents.len() > 1 { + // Every branch node must have a corresponding merge node in a valid plan + if layer > 0 { + layer -= curr_ref.parents.len(); + } else { + break; + } + } + } + } + if layer == 0 { + curr_node_opt + } else { + None + } + } else { + None + } + } +} + +#[allow(dead_code)] +impl LogicalPlan { + /// Create a new logical plan from some root. + pub fn with_root(node: Node) -> Self { + let mut nodes = VecMap::new(); + nodes.insert(node.id as usize, Rc::new(RefCell::new(node))); + Self { nodes, max_node_id: 1, meta: PlanMeta::default() } + } + + /// Get a node reference from the logical plan + pub fn get_node(&self, id: u32) -> Option { + self.nodes.get(id as usize).cloned() + } + + pub fn get_meta(&self) -> &PlanMeta { + &self.meta + } + + /// Append a new node into the logical plan, with specified `parent_ids` + /// as its parent nodes. In order to do so, all specified parents must present in the + /// logical plan. + /// + /// # Return + /// * If succeed, the id of the newly added node + /// * Otherwise, `IrError::ParentNodeNotExist` + pub fn append_node(&mut self, mut node: Node, parent_ids: Vec) -> IrResult { + let id = node.id; + if !self.is_empty() && !parent_ids.is_empty() { + let mut parent_nodes = vec![]; + for parent_id in parent_ids { + if let Some(parent_node) = self.get_node(parent_id) { + parent_nodes.push(parent_node); + } else { + return Err(IrError::ParentNodeNotExist(parent_id)); + } + } + for parent_node in parent_nodes { + node.add_parent(parent_node.borrow().id); + parent_node.borrow_mut().add_child(id); + } + } + let node_rc = Rc::new(RefCell::new(node)); + self.nodes.insert(id as usize, node_rc.clone()); + self.max_node_id = std::cmp::max(self.max_node_id, id) + 1; + + Ok(id) + } + + /// Append an existing logical plan to the logical plan, with the specified + /// parent node's id. Note that we currently only allow appending a logical + /// plan to one single parent node. + pub fn append_plan(&mut self, plan: pb::LogicalPlan, parent_ids: Vec) -> IrResult { + if parent_ids.len() != 1 { + return Err(IrError::Unsupported( + "only support appending plan for one single parent!".to_string(), + )); + } + let mut id_map: HashMap = HashMap::new(); + let mut parents: HashMap> = HashMap::new(); + let mut result_id = 0_u32; + for (id, node) in plan.nodes.iter().enumerate() { + for child in &node.children { + parents + .entry(*child as u32) + .or_insert_with(BTreeSet::new) + .insert(id as u32); + } + } + for (id, node) in plan.nodes.into_iter().enumerate() { + if let Some(opr) = node.opr { + let new_parents = if !parents.contains_key(&(id as u32)) { + parent_ids.clone() + } else { + parents + .get(&(id as u32)) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|old| { + id_map + .get(&old) + .cloned() + .ok_or(IrError::ParentNodeNotExist(old)) + }) + .collect::>>()? + }; + let new_id = self.append_operator_as_node(opr, new_parents)?; + id_map.insert(id as u32, new_id); + result_id = new_id; + } else { + return Err(IrError::MissingData("Node::opr".to_string())); + } + } + + Ok(result_id) + } + + /// Append an operator into the logical plan, as a new node with `self.max_node_id` as its id. + pub fn append_operator_as_node( + &mut self, mut opr: pb::logical_plan::Operator, parent_ids: Vec, + ) -> IrResult { + use common_pb::expr_opr::Item; + use pb::logical_plan::operator::Opr; + + let old_curr_nodes = self.meta.get_curr_nodes().to_vec(); + let mut is_update_curr = false; + if opr.opr.is_none() { + return Err(IrError::MissingData("Operator::opr".to_string())); + } + if let Ok(meta) = STORE_META.read() { + match opr.opr.as_ref().unwrap() { + Opr::Scan(_) | Opr::Edge(_) | Opr::Vertex(_) | Opr::Path(_) => { + self.meta.set_curr_node(self.max_node_id); + } + Opr::Project(ref proj) => { + is_update_curr = true; + if proj.mappings.len() == 1 { + // change current node as the tagged node + if let Some(expr) = &proj.mappings[0].expr { + if expr.operators.len() == 1 { + if let common_pb::ExprOpr { item: Some(Item::Var(var)) } = + expr.operators.get(0).unwrap() + { + if var.tag.is_some() && var.property.is_none() + // tag as Head + { + is_update_curr = false; + } + } + } + } + } + } + Opr::Union(_) => { + self.meta + .set_union_curr_nodes(parent_ids.clone()); + } + Opr::As(_) + | Opr::Select(_) + | Opr::OrderBy(_) + | Opr::Dedup(_) + | Opr::Limit(_) + | Opr::Sink(_) + | Opr::Pattern(_) => {} // do not change current node + _ => is_update_curr = true, + } + + opr.preprocess(&meta, &mut self.meta)?; + } + + let new_curr_node_rst = match opr.opr.as_ref().unwrap() { + Opr::Pattern(pattern) => { + if parent_ids.len() == 1 { + let strategy = NaiveStrategy::try_from(pattern.clone())?; + let plan = strategy.build_logical_plan()?; + self.append_plan(plan, parent_ids) + } else { + Err(IrError::Unsupported( + "only one single parent is supported for the `Pattern` operator".to_string(), + )) + } + } + _ => self.append_node(Node::new(self.max_node_id, opr), parent_ids), + }; + + if let Ok(new_curr_node) = &new_curr_node_rst { + if is_update_curr { + self.meta.set_curr_node(*new_curr_node); + } + } else { + if old_curr_nodes.len() == 1 { + self.meta.set_curr_node(old_curr_nodes[0]); + } else { + self.meta.set_union_curr_nodes(old_curr_nodes); + } + } + + new_curr_node_rst + } + + /// Remove a node from the logical plan, and do the following: + /// * For each of its parent, if present, remove this node's id reference from its `children`. + /// * For each of its children, remove this node's id reference from its `parent`, and if + /// the child's parent becomes empty, the child must be removed recursively. + /// + /// Note that this does not decrease `self.node_max_id`, which serves as the indication + /// of new id of the plan. + pub fn remove_node(&mut self, id: u32) -> Option { + let node = self.nodes.remove(id as usize); + if let Some(n) = &node { + for p in &n.borrow().parents { + if let Some(parent_node) = self.get_node(*p) { + parent_node.borrow_mut().children.remove(&id); + } + } + + for c in &n.borrow().children { + if let Some(child) = self.get_node(*c) { + child.borrow_mut().parents.remove(&id); + if child.borrow().parents.is_empty() { + // Recursively remove the child + let _ = self.remove_node(*c); + } + } + } + } + node + } + + pub fn len(&self) -> usize { + self.nodes.len() + } + + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + pub fn root(&self) -> Option { + self.nodes + .iter() + .next() + .map(|(_, node)| node.clone()) + } + + /// Append branch plans to a certain node which has **no** children in this logical plan. + pub fn append_branch_plans(&mut self, node: NodeType, subplans: Vec) { + if !node.borrow().children.is_empty() { + return; + } else { + for subplan in subplans { + if let Some((_, root)) = subplan.nodes.iter().next() { + node.borrow_mut() + .children + .insert(root.borrow().id); + root.borrow_mut() + .parents + .insert(node.borrow().id); + } + self.nodes.extend(subplan.nodes.into_iter()); + } + } + } + + /// From a branch node `root`, obtain the sub-plans (branch node excluded), each representing + /// a branch of operators, till the merge node (merge node excluded). + /// + /// # Return + /// * the merge node and sub-plans if the `brach_node` is indeed a branch node (has more + /// than one child), and its corresponding merge_node present. + /// * `None` and empty sub-plans if otherwise. + pub fn get_branch_plans(&self, branch_node: NodeType) -> (Option, Vec) { + let mut plans = vec![]; + let mut merge_node_opt = None; + if branch_node.borrow().children.len() > 1 { + merge_node_opt = self.get_merge_node(branch_node.clone()); + if let Some(merge_node) = &merge_node_opt { + for &child_node_id in &branch_node.borrow().children { + if let Some(child_node) = self.get_node(child_node_id) { + if let Some(subplan) = self.subplan(child_node, merge_node.clone()) { + plans.push(subplan) + } + } + } + } + } + + (merge_node_opt, plans) + } + + /// To construct a subplan from every node lying between `from_node` (included) and `to_node` (excluded) + /// in the logical plan. Thus, for the subplan to be valid, `to_node` must refer to a + /// downstream node against `from_node` in the plan. + /// + /// If there are some branches between `from_node` and `to_node`, there are two cases: + /// * 1. `to_node` lies within a sub-branch. In this case, **NO** subplan can be produced; + /// * 2. `to_node` is a downstream node of the merge node of the branch, then all branches + /// of nodes must be included in the subplan. For example, F is a `from_node` which has two + /// branches, namely F -> A1 -> B1 -> M, and F -> A2 -> B2 -> M. we have `to_node` T connected + /// to M in the logical plan, as M -> T0 -> T, the subplan from F to T, must include the operators + /// of F, A1, B1, A2, B2, M and T0. + /// + /// # Return + /// * The subplan in case of success, + /// * `None` if `from_node` is `to_node`, or could not arrive at `to_node` following the + /// plan, or there is a branch node in between, but fail to locate the corresponding merge node. + pub fn subplan(&self, from_node: NodeType, to_node: NodeType) -> Option { + if from_node == to_node { + return None; + } + let mut plan = LogicalPlan::with_root(clone_node(from_node.clone())); + plan.meta = self.meta.clone(); + let mut curr_node = from_node; + while curr_node != to_node { + if curr_node.borrow().children.is_empty() { + // While still not locating to_node + return None; + } else if curr_node.borrow().children.len() == 1 { + let next_node_id = curr_node.borrow().get_first_child().unwrap(); + if let Some(next_node) = self.get_node(next_node_id) { + if next_node.borrow().id != to_node.borrow().id { + plan.append_node(clone_node(next_node.clone()), vec![curr_node.borrow().id]) + .expect("append node to subplan error"); + } + curr_node = next_node; + } else { + return None; + } + } else { + let (merge_node_opt, subplans) = self.get_branch_plans(curr_node.clone()); + if let Some(merge_node) = merge_node_opt { + plan.append_branch_plans(plan.get_node(curr_node.borrow().id).unwrap(), subplans); + if merge_node.borrow().id != to_node.borrow().id { + let merge_node_parent = merge_node + .borrow() + .parents + .iter() + .map(|x| *x) + .collect(); + let merge_node_clone = clone_node(merge_node.clone()); + plan.append_node(merge_node_clone, merge_node_parent) + .expect("append node to subplan error"); + } + curr_node = merge_node; + } else { + return None; + } + } + } + Some(plan) + } + + /// Given a node that contains a subtask, which is typically an `Apply` operator, + /// try to extract the subtask as a logical plan. + /// + /// TODO(longbin): There may be an issue when the last node of a subtask is a merge node. + pub fn extract_subplan(&self, node: NodeType) -> Option { + let node_borrow = node.borrow(); + match &node_borrow.opr.opr { + Some(pb::logical_plan::operator::Opr::Apply(apply_opr)) => { + if let Some(from_node) = self.get_node(apply_opr.subtask as u32) { + let mut curr_node = from_node.clone(); + while let Some(to_node) = curr_node + .clone() + .borrow() + .get_first_child() + .and_then(|node_id| self.get_node(node_id)) + { + curr_node = to_node.clone(); + } + let parent_ids = curr_node + .borrow() + .parents + .iter() + .cloned() + .collect(); + let mut subplan = if from_node == curr_node { + let mut p = LogicalPlan::default(); + p.meta = self.meta.clone(); + Some(p) + } else { + self.subplan(from_node, curr_node.clone()) + }; + if let Some(plan) = subplan.as_mut() { + plan.append_node(curr_node.borrow().clone(), parent_ids) + .expect("append node to subplan error!"); + } + subplan + } else { + None + } + } + _ => None, + } + } +} + +impl JsonIO for LogicalPlan { + fn into_json(self, writer: W) -> io::Result<()> { + let plan_pb: pb::LogicalPlan = self.into(); + serde_json::to_writer_pretty(writer, &plan_pb)?; + + Ok(()) + } + + fn from_json(reader: R) -> io::Result { + let plan_pb = serde_json::from_reader::<_, pb::LogicalPlan>(reader)?; + Self::try_from(plan_pb).map_err(|_| io::Error::from(io::ErrorKind::InvalidData)) + } +} + +pub trait AsLogical { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()>; +} + +fn preprocess_var( + var: &mut common_pb::Variable, meta: &StoreMeta, plan_meta: &mut PlanMeta, +) -> IrResult<()> { + var.tag + .as_mut() + .map(|tag| get_or_set_tag_id(tag, true, plan_meta)) + .transpose()?; + let tag = var + .tag + .clone() + .map(|tag| tag.try_into()) + .transpose()?; + let mut node_meta = plan_meta.tag_node_metas_mut(tag.as_ref())?; + if let Some(property) = var.property.as_mut() { + if let Some(key) = property.item.as_mut() { + match key { + common_pb::property::Item::Key(key) => { + if let Some(schema) = &meta.schema { + if plan_meta.is_column_id() && schema.is_column_id() { + let new_key = schema + .get_column_id_from_pb(key) + .unwrap_or(INVALID_META_ID) + .into(); + debug!("column: {:?} -> {:?}", key, new_key); + *key = new_key; + } + } + debug!("add column ({:?}) to {:?}", key, var.tag); + node_meta.insert_column(key.clone().try_into()?); + } + common_pb::property::Item::All(_) => node_meta.set_is_all_columns(true), + _ => {} + } + } + } + + Ok(()) +} + +fn preprocess_const( + val: &mut common_pb::Value, meta: &StoreMeta, plan_meta: &mut PlanMeta, +) -> IrResult<()> { + if let Some(schema) = &meta.schema { + // A Const needs to be preprocessed only if it is while comparing a label (table) + if plan_meta.is_table_id() && schema.is_table_id() { + if let Some(item) = val.item.as_mut() { + match item { + common_pb::value::Item::Str(name) => { + let new_item = common_pb::value::Item::I32( + schema + .get_table_id(name) + .ok_or(IrError::TableNotExist(NameOrId::Str(name.to_string())))?, + ); + debug!("table: {:?} -> {:?}", item, new_item); + *item = new_item; + } + common_pb::value::Item::StrArray(names) => { + let new_item = common_pb::value::Item::I32Array(common_pb::I32Array { + item: names + .item + .iter() + .map(|name| { + schema + .get_table_id(name) + .ok_or(IrError::TableNotExist(NameOrId::Str(name.to_string()))) + }) + .collect::>>()?, + }); + debug!("table: {:?} -> {:?}", item, new_item); + *item = new_item; + } + _ => {} + } + } + } + } + Ok(()) +} + +fn preprocess_expression( + expr: &mut common_pb::Expression, meta: &StoreMeta, plan_meta: &mut PlanMeta, +) -> IrResult<()> { + let mut count = 0; + for opr in expr.operators.iter_mut() { + if let Some(item) = opr.item.as_mut() { + match item { + common_pb::expr_opr::Item::Var(var) => { + if let Some(property) = var.property.as_mut() { + if let Some(key) = property.item.as_mut() { + match key { + common_pb::property::Item::Label(_) => count = 1, + _ => count = 0, + } + } + } + preprocess_var(var, meta, plan_meta)?; + } + common_pb::expr_opr::Item::Logical(l) => { + if count == 1 { + // means previous one is LabelKey + // The logical operator of Eq, Ne, Lt, Le, Gt, Ge, Within, Without + if *l >= 0 && *l <= 7 { + count = 2; // indicates LabelKey + } + } else { + count = 0; + } + } + common_pb::expr_opr::Item::Const(c) => { + if count == 2 { + // indicates LabelKey labelValue + preprocess_const(c, meta, plan_meta)?; + } + count = 0; + } + common_pb::expr_opr::Item::Vars(vars) | common_pb::expr_opr::Item::VarMap(vars) => { + for var in &mut vars.keys { + preprocess_var(var, meta, plan_meta)?; + } + count = 0; + } + _ => count = 0, + } + } + } + + Ok(()) +} + +fn preprocess_params( + params: &mut pb::QueryParams, meta: &StoreMeta, plan_meta: &mut PlanMeta, +) -> IrResult<()> { + if let Some(pred) = &mut params.predicate { + preprocess_expression(pred, meta, plan_meta)?; + } + if let Some(schema) = &meta.schema { + if plan_meta.is_table_id() && schema.is_table_id() { + for table in params.tables.iter_mut() { + let new_table = schema + .get_table_id_from_pb(table) + .ok_or(IrError::TableNotExist(table.clone().try_into()?))? + .into(); + debug!("table: {:?} -> {:?}", table, new_table); + *table = new_table; + } + } + } + for column in params.columns.iter_mut() { + if let Some(schema) = &meta.schema { + if plan_meta.is_column_id() && schema.is_column_id() { + let new_column = schema + .get_column_id_from_pb(column) + .unwrap_or(INVALID_META_ID) + .into(); + debug!("column: {:?} -> {:?}", column, new_column); + *column = new_column; + } + } + debug!("add column ({:?}) to HEAD", column); + plan_meta + .curr_node_metas_mut() + .insert_column(column.clone().try_into()?); + } + Ok(()) +} + +fn get_or_set_tag_id( + tag_pb: &mut common_pb::NameOrId, should_exist: bool, plan_meta: &mut PlanMeta, +) -> IrResult { + let tag = tag_pb.clone().try_into()?; + let (is_exist, tag_id) = plan_meta.get_or_set_tag_id(&tag); + if should_exist && !is_exist { + return Err(IrError::TagNotExist(tag)); + } + if plan_meta.is_tag_id() { + *tag_pb = common_pb::NameOrId { item: Some(common_pb::name_or_id::Item::Id(tag_id as i32)) } + } + Ok(is_exist) +} + +impl AsLogical for pb::Project { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + use common_pb::expr_opr::Item; + let len = self.mappings.len(); + for (idx, mapping) in self.mappings.iter_mut().enumerate() { + if let Some(alias) = mapping.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into()?, plan_meta.get_dummy_nodes()); + } + if let Some(expr) = &mut mapping.expr { + preprocess_expression(expr, meta, plan_meta)?; + // Verify if it is the case of project("@a"), which changes the current nodes + // to where the tag "a" points to. + if len == 1 && idx == 0 && expr.operators.len() == 1 { + if let Some(common_pb::ExprOpr { item: Some(Item::Var(var)) }) = expr.operators.first() + { + if var.tag.is_some() && var.property.is_none() { + if let Some(nodes) = + plan_meta.get_tag_nodes(&var.tag.clone().unwrap().try_into()?) + { + if nodes.len() == 1 { + plan_meta.set_curr_node(nodes[0]); + } else { + plan_meta.set_union_curr_nodes(nodes); + } + } + } + } + } + } + } + Ok(()) + } +} + +impl AsLogical for pb::Select { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(pred) = self.predicate.as_mut() { + preprocess_expression(pred, meta, plan_meta)?; + } + Ok(()) + } +} + +impl AsLogical for pb::Scan { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + plan_meta + .curr_node_metas_mut() + .set_is_add_column(true); + if let Some(alias) = self.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into()?, plan_meta.get_curr_nodes()); + } + if let Some(params) = self.params.as_mut() { + preprocess_params(params, meta, plan_meta)?; + } + if let Some(index_pred) = self.idx_predicate.as_mut() { + index_pred.preprocess(meta, plan_meta)?; + } + Ok(()) + } +} + +impl AsLogical for pb::EdgeExpand { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + if self.is_edge { + // as edge is always local, do not need to add column for remote fetching + plan_meta + .curr_node_metas_mut() + .set_is_add_column(false); + } else { + plan_meta + .curr_node_metas_mut() + .set_is_add_column(true); + } + if let Some(params) = self.params.as_mut() { + preprocess_params(params, meta, plan_meta)?; + } + if let Some(alias) = self.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into()?, plan_meta.get_curr_nodes()); + } + plan_meta + .curr_node_metas_mut() + .set_is_add_column(true); + + Ok(()) + } +} + +impl AsLogical for pb::PathExpand { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(base) = self.base.as_mut() { + base.preprocess(meta, plan_meta)?; + } + if let Some(alias) = self.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into()?, plan_meta.get_curr_nodes()); + } + // PathExpand would never require adding columns + plan_meta + .curr_node_metas_mut() + .set_is_add_column(false); + + Ok(()) + } +} + +impl AsLogical for pb::GetV { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + plan_meta + .curr_node_metas_mut() + .set_is_add_column(true); + if let Some(params) = self.params.as_mut() { + preprocess_params(params, meta, plan_meta)?; + } + if let Some(alias) = self.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into()?, plan_meta.get_curr_nodes()); + } + Ok(()) + } +} + +impl AsLogical for pb::Dedup { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + for var in self.keys.iter_mut() { + preprocess_var(var, meta, plan_meta)?; + } + Ok(()) + } +} + +impl AsLogical for pb::GroupBy { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + for mapping in self.mappings.iter_mut() { + if let Some(key) = &mut mapping.key { + preprocess_var(key, meta, plan_meta)?; + if let Some(alias) = mapping.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + } + if key.property.is_none() { + let node_ids = if let Some(tag_pb) = key.tag.clone() { + let tag = tag_pb.try_into()?; + plan_meta + .get_tag_nodes(&tag) + .ok_or(IrError::TagNotExist(tag))? + } else { + plan_meta.get_curr_nodes() + }; + if let Some(alias) = mapping.alias.as_mut() { + plan_meta.insert_tag_nodes(alias.clone().try_into()?, node_ids); + } + } + } + } + for agg_fn in self.functions.iter_mut() { + for var in agg_fn.vars.iter_mut() { + preprocess_var(var, meta, plan_meta)?; + } + if let Some(alias) = agg_fn.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into()?, plan_meta.get_dummy_nodes()); + } + } + + Ok(()) + } +} + +impl AsLogical for pb::IndexPredicate { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + for and_pred in self.or_predicates.iter_mut() { + for pred in and_pred.predicates.iter_mut() { + if let Some(pred_key) = &mut pred.key { + if let Some(key_item) = pred_key.item.as_mut() { + match key_item { + common_pb::property::Item::Key(key) => { + if let Some(schema) = &meta.schema { + if plan_meta.is_column_id() && schema.is_column_id() { + let new_key = schema + .get_column_id_from_pb(key) + .unwrap_or(INVALID_META_ID) + .into(); + debug!("column: {:?} -> {:?}", key, new_key); + *key = new_key; + } + } + debug!("add column ({:?}) to HEAD", key); + plan_meta + .curr_node_metas_mut() + .insert_column(key.clone().try_into()?); + } + common_pb::property::Item::Label(_) => { + if let Some(val) = pred.value.as_mut() { + preprocess_const(val, meta, plan_meta)?; + } + } + _ => {} + } + } + } + } + } + + Ok(()) + } +} + +impl AsLogical for pb::OrderBy { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + for pair in self.pairs.iter_mut() { + if let Some(key) = &mut pair.key { + preprocess_var(key, meta, plan_meta)?; + } + } + + Ok(()) + } +} + +impl AsLogical for pb::Limit { + fn preprocess(&mut self, _meta: &StoreMeta, _plan_meta: &mut PlanMeta) -> IrResult<()> { + Ok(()) + } +} + +impl AsLogical for pb::As { + fn preprocess(&mut self, _meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(alias) = self.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into().unwrap(), plan_meta.get_curr_nodes()); + } + Ok(()) + } +} + +impl AsLogical for pb::Join { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + for left_key in self.left_keys.iter_mut() { + preprocess_var(left_key, meta, plan_meta)? + } + for right_key in self.right_keys.iter_mut() { + preprocess_var(right_key, meta, plan_meta)? + } + Ok(()) + } +} + +impl AsLogical for pb::Sink { + fn preprocess(&mut self, _meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + for tag_key in self.tags.iter_mut() { + if let Some(tag) = tag_key.key.as_mut() { + get_or_set_tag_id(tag, true, plan_meta)?; + } + } + Ok(()) + } +} + +impl AsLogical for pb::Apply { + fn preprocess(&mut self, _meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(alias) = self.alias.as_mut() { + get_or_set_tag_id(alias, false, plan_meta)?; + plan_meta.insert_tag_nodes(alias.clone().try_into().unwrap(), plan_meta.get_dummy_nodes()); + } + Ok(()) + } +} + +fn check_refer_existing_tag( + mut tag_pb: common_pb::NameOrId, plan_meta: &mut PlanMeta, pattern_tags: &mut HashSet, +) -> IrResult<()> { + let mut is_existing_tag = get_or_set_tag_id(&mut tag_pb, false, plan_meta)?; + let tag: NameOrId = tag_pb.try_into()?; + + if is_existing_tag { + is_existing_tag = !pattern_tags.contains(&tag); + } + if is_existing_tag { + Err(IrError::InvalidPattern(format!("`pb::Pattern` cannot reference existing tag: {:?}", tag))) + } else { + pattern_tags.insert(tag); + + Ok(()) + } +} + +impl AsLogical for pb::Pattern { + fn preprocess(&mut self, _meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + let mut pattern_tags = HashSet::new(); + for sentence in &self.sentences { + if let Some(alias) = &sentence.start { + check_refer_existing_tag(alias.clone(), plan_meta, &mut pattern_tags)?; + } else { + return Err(IrError::InvalidPattern( + "the start tag in `pb::Pattern` does not exist".to_string(), + )); + } + if let Some(alias) = &sentence.end { + check_refer_existing_tag(alias.clone(), plan_meta, &mut pattern_tags)?; + } + } + + Ok(()) + } +} + +impl AsLogical for pb::logical_plan::Operator { + fn preprocess(&mut self, meta: &StoreMeta, plan_meta: &mut PlanMeta) -> IrResult<()> { + use pb::logical_plan::operator::Opr; + if let Some(opr) = self.opr.as_mut() { + match opr { + Opr::Project(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Select(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Scan(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Edge(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Path(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Vertex(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Dedup(opr) => opr.preprocess(meta, plan_meta)?, + Opr::GroupBy(opr) => opr.preprocess(meta, plan_meta)?, + Opr::OrderBy(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Limit(opr) => opr.preprocess(meta, plan_meta)?, + Opr::As(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Join(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Sink(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Apply(opr) => opr.preprocess(meta, plan_meta)?, + Opr::Pattern(opr) => opr.preprocess(meta, plan_meta)?, + _ => {} + } + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use ir_common::expr_parse::str_to_expr_pb; + use ir_common::generated::algebra::logical_plan::operator::Opr; + use ir_common::generated::common::property::Item; + use ir_common::NameOrId; + + use super::*; + use crate::plan::meta::set_schema_simple; + + #[allow(dead_code)] + fn query_params( + tables: Vec, columns: Vec, + ) -> pb::QueryParams { + pb::QueryParams { + tables, + columns, + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + } + } + + #[test] + fn logical_plan_construct() { + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + let mut plan = LogicalPlan::default(); + + let id = plan + .append_operator_as_node(opr.clone(), vec![]) + .unwrap(); + assert_eq!(id, 0); + assert_eq!(plan.len(), 1); + assert_eq!(plan.max_node_id, 1); + let node0 = plan.get_node(0).unwrap().clone(); + + let id = plan + .append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); + assert_eq!(id, 1); + assert_eq!(plan.len(), 2); + assert_eq!(plan.max_node_id, 2); + let node1 = plan.get_node(1).unwrap().clone(); + + let parents = node1 + .borrow() + .parents + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(parents, vec![0]); + + let children = node0 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![1]); + + let id = plan + .append_operator_as_node(opr.clone(), vec![0, 1]) + .unwrap(); + assert_eq!(id, 2); + assert_eq!(plan.len(), 3); + assert_eq!(plan.max_node_id, 3); + let node2 = plan.get_node(2).unwrap().clone(); + + let parents = node2 + .borrow() + .parents + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(parents, vec![0, 1]); + + let children = node0 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![1, 2]); + + let children = node1 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![2]); + + let node2 = plan.remove_node(2); + assert_eq!(node2.unwrap().borrow().id, 2); + assert_eq!(plan.len(), 2); + assert_eq!(plan.max_node_id, 3); + let children = node0 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![1]); + + let children = node1 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert!(children.is_empty()); + + let _id = plan.append_operator_as_node(opr.clone(), vec![0, 2]); + match _id.err().unwrap() { + IrError::ParentNodeNotExist(node) => assert_eq!(node, 2), + _ => panic!("wrong error type"), + } + assert_eq!(plan.len(), 2); + assert_eq!(plan.max_node_id, 3); + let children = node0 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![1]); + + // add node2 back again for further testing recursive removal + let _ = plan + .append_operator_as_node(opr.clone(), vec![0, 1]) + .unwrap(); + let node3 = plan.get_node(3).unwrap(); + let _ = plan.remove_node(1); + assert_eq!(plan.len(), 2); + assert_eq!(plan.max_node_id, 4); + let children = node0 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![3]); + + let parents = node3 + .borrow() + .parents + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(parents, vec![0]); + } + + #[test] + fn logical_plan_from_pb() { + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + let root_pb = pb::logical_plan::Node { opr: Some(opr.clone()), children: vec![1, 2] }; + let node1_pb = pb::logical_plan::Node { opr: Some(opr.clone()), children: vec![2] }; + let node2_pb = pb::logical_plan::Node { opr: Some(opr.clone()), children: vec![] }; + let plan_pb = pb::LogicalPlan { nodes: vec![root_pb, node1_pb, node2_pb] }; + + let plan = LogicalPlan::try_from(plan_pb).unwrap(); + assert_eq!(plan.len(), 3); + let node0 = plan.get_node(0).unwrap(); + let node1 = plan.get_node(1).unwrap(); + let node2 = plan.get_node(2).unwrap(); + + let children = node0 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![1, 2]); + + let children = node1 + .borrow() + .children + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(children, vec![2]); + + let parents = node1 + .borrow() + .parents + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(parents, vec![0]); + + let parents = node2 + .borrow() + .parents + .iter() + .map(|x| *x) + .collect::>(); + assert_eq!(parents, vec![0, 1]); + } + + #[test] + fn logical_plan_into_pb() { + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + let mut plan = LogicalPlan::default(); + + let _ = plan + .append_operator_as_node(opr.clone(), vec![]) + .unwrap(); + let _ = plan + .append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); + let _ = plan + .append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); + + let _ = plan.remove_node(1); + + let plan_pb = pb::LogicalPlan::from(plan); + assert_eq!(plan_pb.nodes.len(), 2); + + let node0 = &plan_pb.nodes[0]; + let node1 = &plan_pb.nodes[1]; + assert_eq!(node0.opr, Some(opr.clone())); + assert_eq!(node0.children, vec![1]); + assert_eq!(node1.opr, Some(opr.clone())); + assert!(node1.children.is_empty()); + } + + #[test] + fn preprocess_expr() { + let mut plan_meta = PlanMeta::default().with_store_conf(true, true); + plan_meta.insert_tag_nodes("a".into(), vec![1]); + plan_meta.insert_tag_nodes("b".into(), vec![2]); + plan_meta.get_or_set_tag_id(&"a".into()); + plan_meta.get_or_set_tag_id(&"b".into()); + plan_meta + .curr_node_metas_mut() + .set_is_add_column(true); + plan_meta + .tag_node_metas_mut(Some(&"a".into())) + .unwrap() + .set_is_add_column(true); + plan_meta + .tag_node_metas_mut(Some(&"b".into())) + .unwrap() + .set_is_add_column(true); + + set_schema_simple( + vec![("person".to_string(), 0), ("software".to_string(), 1)], + vec![("knows".to_string(), 0), ("creates".to_string(), 1)], + vec![("id".to_string(), 0), ("name".to_string(), 1), ("age".to_string(), 2)], + ); + + let mut expression = str_to_expr_pb("@.~label == \"person\"".to_string()).unwrap(); + preprocess_expression(&mut expression, &STORE_META.read().unwrap(), &mut plan_meta).unwrap(); + let opr = expression.operators.get(2).unwrap().clone(); + match opr.item.unwrap() { + common_pb::expr_opr::Item::Const(val) => match val.item.unwrap() { + common_pb::value::Item::I32(i) => assert_eq!(i, 0), + _ => panic!(), + }, + _ => panic!(), + } + + let mut expression = + str_to_expr_pb("@.~label within [\"person\", \"software\"]".to_string()).unwrap(); + preprocess_expression(&mut expression, &STORE_META.read().unwrap(), &mut plan_meta).unwrap(); + let opr = expression.operators.get(2).unwrap().clone(); + match opr.item.unwrap() { + common_pb::expr_opr::Item::Const(val) => match val.item.unwrap() { + common_pb::value::Item::I32Array(arr) => { + assert_eq!(arr.item, vec![0, 1]); + } + _ => panic!(), + }, + _ => panic!(), + } + + let mut expression = + str_to_expr_pb("(@.name == \"person\") && @a.~label == \"knows\"".to_string()).unwrap(); + preprocess_expression(&mut expression, &STORE_META.read().unwrap(), &mut plan_meta).unwrap(); + + // person should not be mapped, as name is not a label key + let opr = expression.operators.get(3).unwrap().clone(); + match opr.item.unwrap() { + common_pb::expr_opr::Item::Const(val) => match val.item.unwrap() { + common_pb::value::Item::Str(str) => assert_eq!(str, "person".to_string()), + _ => panic!(), + }, + _ => panic!(), + } + // "knows maps to 0" + let opr = expression.operators.get(8).unwrap().clone(); + match opr.item.unwrap() { + common_pb::expr_opr::Item::Const(val) => match val.item.unwrap() { + common_pb::value::Item::I32(i) => assert_eq!(i, 0), + _ => panic!(), + }, + _ => panic!(), + } + + // Assert whether the columns have been updated in PlanMeta + assert_eq!( + plan_meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns(), + // has a new column "name", which is mapped to 1 + &vec![1.into()] + .into_iter() + .collect::>() + ); + + // name maps to 1 + let mut expression = str_to_expr_pb("@a.name == \"John\"".to_string()).unwrap(); + preprocess_expression(&mut expression, &STORE_META.read().unwrap(), &mut plan_meta).unwrap(); + let opr = expression.operators.get(0).unwrap().clone(); + match opr.item.unwrap() { + common_pb::expr_opr::Item::Var(var) => { + match var.clone().property.unwrap().item.unwrap() { + Item::Key(key) => assert_eq!(key, 1.into()), + _ => panic!(), + } + assert_eq!(var.tag.unwrap(), "a".into()); + } + _ => panic!(), + } + + // Assert whether the columns have been updated in PlanMeta + assert_eq!( + plan_meta + .get_node_meta(1) + .unwrap() + .borrow() + .get_columns(), + // node1 with tag a has a new column "name", which is mapped to 1 + &vec![1.into()] + .into_iter() + .collect::>() + ); + + let mut expression = str_to_expr_pb("{@a.name, @b.id}".to_string()).unwrap(); + preprocess_expression(&mut expression, &STORE_META.read().unwrap(), &mut plan_meta).unwrap(); + let opr = expression.operators.get(0).unwrap().clone(); + match opr.item.unwrap() { + common_pb::expr_opr::Item::VarMap(vars) => { + let var1 = vars.keys[0].clone(); + match var1.property.unwrap().item.unwrap() { + Item::Key(key) => assert_eq!(key, 1.into()), + _ => panic!(), + } + assert_eq!(var1.tag.unwrap(), "a".into()); + let var2 = vars.keys[1].clone(); + match var2.property.unwrap().item.unwrap() { + Item::Key(key) => assert_eq!(key, 0.into()), + _ => panic!(), + } + assert_eq!(var2.tag.unwrap(), "b".into()); + } + _ => panic!(), + } + + // Assert whether the columns have been updated in PlanMeta + assert_eq!( + plan_meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns(), + // node1 with tag a has a new column "name", which is mapped to 1 + &vec![1.into()] + .into_iter() + .collect::>() + ); + assert_eq!( + plan_meta + .get_node_meta(2) + .unwrap() + .borrow() + .get_columns(), + // node2 with tag b has a new column "id", which is mapped to 0 + &vec![0.into()] + .into_iter() + .collect::>() + ); + } + + #[test] + fn preprocess_scan() { + let mut plan_meta = PlanMeta::default().with_store_conf(true, true); + plan_meta.insert_tag_nodes("a".into(), vec![1]); + plan_meta.get_or_set_tag_id(&"a".into()); + plan_meta + .curr_node_metas_mut() + .set_is_add_column(true); + plan_meta + .tag_node_metas_mut(Some(&"a".into())) + .unwrap() + .set_is_add_column(true); + set_schema_simple( + vec![("person".to_string(), 0), ("software".to_string(), 1)], + vec![("knows".to_string(), 0), ("creates".to_string(), 1)], + vec![("id".to_string(), 0), ("name".to_string(), 1), ("age".to_string(), 2)], + ); + let mut scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(pb::QueryParams { + tables: vec!["person".into()], + columns: vec!["age".into(), "name".into()], + is_all_columns: false, + limit: None, + predicate: Some( + str_to_expr_pb("@a.~label > \"person\" && @a.age == 10".to_string()).unwrap(), + ), + extra: HashMap::new(), + }), + idx_predicate: Some(vec!["software".to_string()].into()), + }; + scan.preprocess(&STORE_META.read().unwrap(), &mut plan_meta) + .unwrap(); + assert_eq!(scan.clone().params.unwrap().tables[0], 0.into()); + assert_eq!( + scan.idx_predicate.unwrap().or_predicates[0].predicates[0] + .value + .clone() + .unwrap(), + 1.into() + ); + let operators = scan + .params + .clone() + .unwrap() + .predicate + .unwrap() + .operators; + match operators.get(2).unwrap().item.as_ref().unwrap() { + common_pb::expr_opr::Item::Const(val) => assert_eq!(val.clone(), 0.into()), + _ => panic!(), + } + match operators.get(4).unwrap().item.as_ref().unwrap() { + common_pb::expr_opr::Item::Var(var) => { + match var + .property + .as_ref() + .unwrap() + .item + .clone() + .unwrap() + { + Item::Key(key) => assert_eq!(key, 2.into()), + _ => panic!(), + } + } + _ => panic!(), + } + // Assert whether the columns have been updated in PlanMeta + assert_eq!( + plan_meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns(), + &vec![1.into(), 2.into()] + .into_iter() + .collect::>() + ); + assert_eq!( + plan_meta + .get_node_meta(1) + .unwrap() + .borrow() + .get_columns(), + &vec![2.into()] + .into_iter() + .collect::>() + ); + } + + #[test] + fn tag_maintain_case1() { + let mut plan = LogicalPlan::default(); + // g.V().hasLabel("person").has("age", 27).valueMap("age", "name", "id") + + // g.V() + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + // .hasLabel("person") + let select = pb::Select { predicate: str_to_expr_pb("@.~label == \"person\"".to_string()).ok() }; + plan.append_operator_as_node(select.into(), vec![0]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + // .has("age", 27) + let select = pb::Select { predicate: str_to_expr_pb("@.age == 27".to_string()).ok() }; + plan.append_operator_as_node(select.into(), vec![1]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + assert_eq!( + plan.meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns(), + &vec!["age".into()] + .into_iter() + .collect::>() + ); + + // .valueMap("age", "name", "id") + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("{@.name, @.age, @.id}".to_string()).ok(), + alias: None, + }], + is_append: false, + }; + plan.append_operator_as_node(project.into(), vec![2]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[3]); + assert_eq!( + plan.meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns(), + &vec!["age".into(), "id".into(), "name".into()] + .into_iter() + .collect::>() + ); + } + + #[test] + fn tag_maintain_case2() { + let mut plan = LogicalPlan::default(); + // g.V().out().as("here").has("lang", "java").select("here").values("name") + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + // .out().as("here") + let expand = pb::EdgeExpand { + v_tag: Some("a".into()), + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: false, + alias: Some("here".into()), + }; + plan.append_operator_as_node(expand.into(), vec![0]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + + // .has("lang", "Java") + let select = pb::Select { predicate: str_to_expr_pb("@.lang == \"Java\"".to_string()).ok() }; + plan.append_operator_as_node(select.into(), vec![1]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + assert_eq!( + plan.meta + .get_node_meta(1) + .unwrap() + .borrow() + .get_columns(), + &vec!["lang".into()] + .into_iter() + .collect::>() + ); + + // .select("here") + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@here".to_string()).ok(), + alias: None, + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![2]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + + // .values("name") + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@.name".to_string()).ok(), + alias: None, + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![3]) + .unwrap(); + assert_eq!( + plan.meta + .get_node_meta(1) + .unwrap() + .borrow() + .get_columns(), + &vec!["lang".into(), "name".into()] + .into_iter() + .collect::>() + ); + assert_eq!(plan.meta.get_curr_nodes(), &[4]); + } + + #[test] + fn tag_maintain_case3() { + let mut plan = LogicalPlan::default(); + // g.V().outE().as("e").inV().as("v").select("e").order().by("weight").select("v").values("name").dedup() + + // g.V() + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + // .outE().as("e") + let expand = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: true, + alias: Some("e".into()), + }; + plan.append_operator_as_node(expand.into(), vec![0]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + assert_eq!(plan.meta.get_tag_nodes(&"e".into()).unwrap(), vec![1]); + + // .inV().as("v") + let getv = pb::GetV { + tag: None, + opt: 1, + params: Some(query_params(vec![], vec![])), + alias: Some("v".into()), + }; + plan.append_operator_as_node(getv.into(), vec![1]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[2]); + assert_eq!(plan.meta.get_tag_nodes(&"v".into()).unwrap(), vec![2]); + + // .select("e") + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@e".to_string()).ok(), + alias: Some("project_e".into()), + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![2]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + + // .order().by("weight") + let orderby = pb::OrderBy { + pairs: vec![pb::order_by::OrderingPair { + key: Some(common_pb::Variable { + tag: None, + property: Some(common_pb::Property { + item: Some(common_pb::property::Item::Key("weight".into())), + }), + }), + order: 1, + }], + limit: None, + }; + plan.append_operator_as_node(orderby.into(), vec![3]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + assert_eq!( + plan.meta + .curr_node_metas() + .unwrap() + .get_columns() + .clone(), + vec!["weight".into()].into_iter().collect() + ); + + // select("v") + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@v".to_string()).ok(), + alias: Some("project_v".into()), + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![4]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[2]); + + // .values("name") + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@.name".to_string()).ok(), + alias: Some("name".into()), + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![5]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[6]); + assert_eq!( + plan.meta + .get_node_meta(2) + .unwrap() + .borrow() + .get_columns() + .clone(), + vec!["name".into()].into_iter().collect() + ); + } + + #[test] + fn tag_maintain_case4() { + let mut plan = LogicalPlan::default(); + // g.V("person").has("name", "John").as('a').outE("knows").as('b') + // .has("date", 20200101).inV().as('c').has('id', 10) + // .select('a').by(valueMap('age', "name")) + // .select('c').by(valueMap('id', "name")) + + // g.V("person") + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec!["person".into()], vec![])), + idx_predicate: None, + }; + let mut opr_id = plan + .append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + // .has("name", "John") + let select = pb::Select { predicate: str_to_expr_pb("@.name == \"John\"".to_string()).ok() }; + opr_id = plan + .append_operator_as_node(select.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + assert_eq!( + plan.meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns(), + &vec!["name".into()] + .into_iter() + .collect::>() + ); + + // .as('a') + let as_opr = pb::As { alias: Some("a".into()) }; + opr_id = plan + .append_operator_as_node(as_opr.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + assert_eq!(plan.meta.get_tag_nodes(&"a".into()).unwrap(), vec![0]); + + // outE("knows").as('b').has("date", 20200101) + let expand = pb::EdgeExpand { + v_tag: Some("a".into()), + direction: 0, + params: Some(query_params(vec!["knows".into()], vec![])), + is_edge: true, + alias: Some("b".into()), + }; + opr_id = plan + .append_operator_as_node(expand.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[opr_id as u32]); + assert_eq!(plan.meta.get_tag_nodes(&"b".into()).unwrap(), vec![opr_id as u32]); + + //.inV().as('c') + let getv = pb::GetV { + tag: None, + opt: 2, + params: Some(query_params(vec![], vec![])), + alias: Some("c".into()), + }; + opr_id = plan + .append_operator_as_node(getv.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[opr_id as u32]); + assert_eq!(plan.meta.get_tag_nodes(&"c".into()).unwrap(), vec![opr_id as u32]); + + // .has("id", 10) + let select = pb::Select { predicate: str_to_expr_pb("@.id == 10".to_string()).ok() }; + opr_id = plan + .append_operator_as_node(select.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[opr_id as u32 - 1]); + assert_eq!( + plan.meta + .get_node_meta(opr_id as u32 - 1) + .unwrap() + .borrow() + .get_columns(), + &vec!["id".into()] + .into_iter() + .collect::>() + ); + + // .select('a').by(valueMap('age', "name")) + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("{@a.age, @a.name}".to_string()).ok(), + alias: None, + }], + is_append: true, + }; + opr_id = plan + .append_operator_as_node(project.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[opr_id as u32]); + assert_eq!( + plan.meta + .get_node_meta(plan.meta.get_tag_nodes(&"a".into()).unwrap()[0]) + .unwrap() + .borrow() + .get_columns(), + &vec!["age".into(), "name".into()] + .into_iter() + .collect::>() + ); + + // .select('c').by(valueMap('age', "name")) + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("{@c.age, @c.name}".to_string()).ok(), + alias: None, + }], + is_append: true, + }; + opr_id = plan + .append_operator_as_node(project.into(), vec![opr_id as u32]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[opr_id as u32]); + assert_eq!( + plan.meta + .get_node_meta(plan.meta.get_tag_nodes(&"c".into()).unwrap()[0]) + .unwrap() + .borrow() + .get_columns(), + &vec!["age".into(), "id".into(), "name".into()] + .into_iter() + .collect::>() + ); + } + + #[test] + fn tag_maintain_groupby_case1() { + // groupBy contains tagging a keys that is further a vertex + let mut plan = LogicalPlan::default(); + // g.V().groupCount().order().by(select(keys).by('name')) + + // g.V() + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + let group = pb::GroupBy { + mappings: vec![pb::group_by::KeyAlias { + key: Some(common_pb::Variable { tag: None, property: None }), + alias: Some("~keys_2_0".into()), + }], + functions: vec![pb::group_by::AggFunc { + vars: vec![], + aggregate: 3, + alias: Some("~values_2_0".into()), + }], + }; + plan.append_operator_as_node(group.into(), vec![0]) + .unwrap(); + assert_eq!( + plan.meta + .get_tag_nodes(&"~keys_2_0".into()) + .unwrap(), + vec![0] + ); + + let order = pb::OrderBy { + pairs: vec![pb::order_by::OrderingPair { + key: Some(common_pb::Variable { + tag: Some("~keys_2_0".into()), + property: Some(common_pb::Property { + item: Some(common_pb::property::Item::Key("name".into())), + }), + }), + order: 0, + }], + limit: None, + }; + plan.append_operator_as_node(order.into(), vec![1]) + .unwrap(); + assert!(plan + .meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns() + .contains(&"name".into())); + } + + #[test] + fn tag_maintain_groupby_case2() { + // groupBy contains tagging a keys that is further a vertex + let mut plan = LogicalPlan::default(); + // g.V().groupCount().select(values) + + // g.V() + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + let group = pb::GroupBy { + mappings: vec![pb::group_by::KeyAlias { + key: Some(common_pb::Variable { tag: None, property: None }), + alias: Some("~keys_2_0".into()), + }], + functions: vec![pb::group_by::AggFunc { + vars: vec![], + aggregate: 3, + alias: Some("~values_2_0".into()), + }], + }; + plan.append_operator_as_node(group.into(), vec![0]) + .unwrap(); + assert_eq!( + plan.meta + .get_tag_nodes(&"~keys_2_0".into()) + .unwrap(), + vec![0] + ); + + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@~values_2_0".to_string()).ok(), + alias: None, + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![1]) + .unwrap(); + } + + #[test] + fn tag_maintain_orderby() { + let mut plan = LogicalPlan::default(); + // g.E(xx).values("workFrom").as("a").order().by(select("a")) + + let scan = pb::Scan { + scan_opt: 1, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[0]); + + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@.workFrom".to_string()).ok(), + alias: Some("a".into()), + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![0]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + assert!(plan + .meta + .get_node_meta(0) + .unwrap() + .borrow() + .get_columns() + .contains(&"workFrom".into())); + + let order = pb::OrderBy { + pairs: vec![pb::order_by::OrderingPair { + key: Some(common_pb::Variable { tag: Some("a".into()), property: None }), + order: 0, + }], + limit: None, + }; + plan.append_operator_as_node(order.into(), vec![1]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), &[1]); + } + + #[test] + fn tag_maintain_union() { + let mut plan = LogicalPlan::default(); + // g.V().union(out(), out().out()).as('a').select('a').by(valueMap('name', 'age')) + // g.V() + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + + let expand1 = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: false, + alias: None, + }; + + let expand2 = expand1.clone(); + let expand3 = expand1.clone(); + let id1 = plan + .append_operator_as_node(expand1.into(), vec![0]) + .unwrap(); + let opr_id = plan + .append_operator_as_node(expand2.into(), vec![0]) + .unwrap(); + let id2 = plan + .append_operator_as_node(expand3.into(), vec![opr_id]) + .unwrap(); + let union = pb::Union { parents: vec![id1 as i32, id2 as i32] }; + plan.append_operator_as_node(union.into(), vec![id1, id2]) + .unwrap(); + assert_eq!(plan.meta.get_curr_nodes(), vec![id1, id2]); + + let as_opr = pb::As { alias: Some("a".into()) }; + let opr_id = plan + .append_operator_as_node(as_opr.into(), vec![id1, id2]) + .unwrap(); + assert_eq!(plan.meta.get_tag_nodes(&"a".into()).unwrap(), vec![id1, id2]); + + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("{@a.name, @a.age}".to_string()).ok(), + alias: None, + }], + is_append: false, + }; + plan.append_operator_as_node(project.into(), vec![opr_id]) + .unwrap(); + assert_eq!( + plan.meta + .get_node_meta(id1) + .unwrap() + .borrow() + .get_columns(), + &vec!["name".into(), "age".into()] + .into_iter() + .collect::>() + ); + assert_eq!( + plan.meta + .get_node_meta(id2) + .unwrap() + .borrow() + .get_columns(), + &vec!["name".into(), "age".into()] + .into_iter() + .collect::>() + ); + } + + #[test] + fn tag_projection_not_exist() { + let mut plan = LogicalPlan::default(); + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@keys.name".to_string()).ok(), + alias: None, + }], + is_append: false, + }; + let result = plan.append_operator_as_node(project.into(), vec![0]); + match result.err() { + Some(IrError::TagNotExist(_)) => {} + _ => panic!("should produce `IrError::TagNotExist`"), + } + } + + #[test] + fn extract_subplan_from_apply_case1() { + let mut plan = LogicalPlan::default(); + // g.V().as("v").where(out().as("o").has("lang", "java")).select("v").values("name") + + // g.V("person") + let scan = pb::Scan { + scan_opt: 0, + alias: Some("v".into()), + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + + let opr_id = plan + .append_operator_as_node(scan.into(), vec![]) + .unwrap(); + + // .out().as("o") + let expand = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: false, + alias: Some("o".into()), + }; + + let root_id = plan + .append_operator_as_node(expand.into(), vec![]) + .unwrap(); + + // .has("lang", "Java") + let select = pb::Select { predicate: str_to_expr_pb("@.lang == \"Java\"".to_string()).ok() }; + plan.append_operator_as_node(select.into(), vec![root_id]) + .unwrap(); + + let apply = pb::Apply { join_kind: 4, tags: vec![], subtask: root_id as i32, alias: None }; + let opr_id = plan + .append_operator_as_node(apply.into(), vec![opr_id]) + .unwrap(); + + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@v.name".to_string()).ok(), + alias: None, + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![opr_id]) + .unwrap(); + + let subplan = plan + .extract_subplan(plan.get_node(opr_id).unwrap()) + .unwrap(); + assert_eq!(subplan.len(), 2); + for (id, node) in &subplan.nodes { + let node_ref = node.borrow(); + if id == root_id as usize { + match node_ref.opr.opr.as_ref().unwrap() { + Opr::Edge(_) => {} + _ => panic!("should be edge expand"), + } + assert_eq!(subplan.meta.get_tag_nodes(&"o".into()).unwrap(), vec![root_id]); + assert_eq!( + subplan + .meta + .get_node_meta(root_id) + .unwrap() + .borrow() + .get_columns() + .iter() + .cloned() + .collect::>(), + vec!["lang".into()] + ) + } else { + match node_ref.opr.opr.as_ref().unwrap() { + Opr::Select(_) => {} + _ => panic!("should be select"), + } + } + } + } + + #[test] + fn extract_subplan_from_apply_case2() { + let mut plan = LogicalPlan::default(); + // g.V().where(not(out("created"))).values("name") + let scan = pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }; + + plan.append_operator_as_node(scan.into(), vec![]) + .unwrap(); + + let expand = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: false, + alias: None, + }; + let root_id = plan + .append_operator_as_node(expand.into(), vec![]) + .unwrap(); + + let apply = pb::Apply { join_kind: 5, tags: vec![], subtask: root_id as i32, alias: None }; + plan.append_operator_as_node(apply.into(), vec![0]) + .unwrap(); + + let project = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@.name".to_string()).ok(), + alias: Some("name".into()), + }], + is_append: true, + }; + plan.append_operator_as_node(project.into(), vec![2]) + .unwrap(); + + let sink = pb::Sink { + tags: vec![common_pb::NameOrIdKey { key: Some("name".into()) }], + id_name_mappings: vec![], + }; + plan.append_operator_as_node(sink.into(), vec![3]) + .unwrap(); + + let subplan = plan + .extract_subplan(plan.get_node(2).unwrap()) + .unwrap(); + assert_eq!(subplan.len(), 1); + match subplan + .get_node(1) + .unwrap() + .borrow() + .opr + .opr + .as_ref() + .unwrap() + { + Opr::Edge(_) => {} + _ => panic!("wrong operator: should be `EdgeExpand`"), + } + } + + // The plan looks like: + // root + // / \ + // 1 2 + // / \ | + // 3 4 | + // \ / | + // 5 | + // \ / + // 6 + // | + // 7 + fn create_logical_plan() -> LogicalPlan { + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(opr.clone(), vec![]) + .unwrap(); // root + plan.append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); // node 1 + plan.append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); // node 2 + plan.append_operator_as_node(opr.clone(), vec![1]) + .unwrap(); // node 3 + plan.append_operator_as_node(opr.clone(), vec![1]) + .unwrap(); // node 4 + plan.append_operator_as_node(opr.clone(), vec![3, 4]) + .unwrap(); // node 5 + plan.append_operator_as_node(opr.clone(), vec![2, 5]) + .unwrap(); // node 6 + plan.append_operator_as_node(opr.clone(), vec![6]) + .unwrap(); // node 7 + + plan + } + + #[test] + fn get_merge_node() { + let plan = create_logical_plan(); + let merge_node = plan.get_merge_node(plan.get_node(1).unwrap()); + assert_eq!(merge_node, plan.get_node(5)); + let merge_node = plan.get_merge_node(plan.get_node(0).unwrap()); + assert_eq!(merge_node, plan.get_node(6)); + // Not a branch node + let merge_node = plan.get_merge_node(plan.get_node(2).unwrap()); + assert!(merge_node.is_none()); + } + + #[test] + fn merge_branch_plans() { + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + let mut plan = LogicalPlan::with_root(Node::new(0, opr.clone())); + + let mut subplan1 = LogicalPlan::with_root(Node::new(1, opr.clone())); + subplan1 + .append_node(Node::new(3, opr.clone()), vec![1]) + .unwrap(); + subplan1 + .append_node(Node::new(4, opr.clone()), vec![1]) + .unwrap(); + subplan1 + .append_node(Node::new(5, opr.clone()), vec![3, 4]) + .unwrap(); + + let subplan2 = LogicalPlan::with_root(Node::new(2, opr.clone())); + + plan.append_branch_plans(plan.get_node(0).unwrap(), vec![subplan1, subplan2]); + let mut expected_plan = create_logical_plan(); + expected_plan.remove_node(6); + + plan.append_node(Node::new(6, opr.clone()), vec![2, 5]) + .unwrap(); + plan.append_node(Node::new(7, opr.clone()), vec![6]) + .unwrap(); + + assert_eq!(plan, create_logical_plan()); + } + + #[test] + fn subplan() { + let plan = create_logical_plan(); + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + let subplan = plan.subplan(plan.get_node(2).unwrap(), plan.get_node(7).unwrap()); + let mut expected_plan = LogicalPlan::with_root(Node::new(2, opr.clone())); + expected_plan + .append_node(Node::new(6, opr.clone()), vec![2]) + .unwrap(); + assert_eq!(subplan.unwrap(), expected_plan); + + // The node 3 is at one of the branches, which is incomplete and hence invalid subplan + let subplan = plan.subplan(plan.get_node(1).unwrap(), plan.get_node(3).unwrap()); + assert!(subplan.is_none()); + + let subplan = plan.subplan(plan.get_node(1).unwrap(), plan.get_node(6).unwrap()); + let mut expected_plan = LogicalPlan::with_root(Node::new(1, opr.clone())); + expected_plan + .append_node(Node::new(3, opr.clone()), vec![1]) + .unwrap(); + expected_plan + .append_node(Node::new(4, opr.clone()), vec![1]) + .unwrap(); + expected_plan + .append_node(Node::new(5, opr.clone()), vec![3, 4]) + .unwrap(); + + assert_eq!(subplan.unwrap(), expected_plan); + } + + #[test] + fn get_branch_plans() { + let plan = create_logical_plan(); + let (merge_node, subplans) = plan.get_branch_plans(plan.get_node(1).unwrap()); + let opr = pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::As(pb::As { alias: None })), + }; + + let plan1 = LogicalPlan::with_root(Node::new(3, opr.clone())); + let plan2 = LogicalPlan::with_root(Node::new(4, opr.clone())); + + assert_eq!(merge_node, plan.get_node(5)); + assert_eq!(subplans, vec![plan1, plan2]); + + let (merge_node, subplans) = plan.get_branch_plans(plan.get_node(0).unwrap()); + let mut plan1 = LogicalPlan::with_root(Node::new(1, opr.clone())); + plan1 + .append_node(Node::new(3, opr.clone()), vec![1]) + .unwrap(); + plan1 + .append_node(Node::new(4, opr.clone()), vec![1]) + .unwrap(); + plan1 + .append_node(Node::new(5, opr.clone()), vec![3, 4]) + .unwrap(); + let plan2 = LogicalPlan::with_root(Node::new(2, opr.clone())); + + assert_eq!(merge_node, plan.get_node(6)); + assert_eq!(subplans, vec![plan1, plan2]); + + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(opr.clone(), vec![]) + .unwrap(); // root + plan.append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); // node 1 + plan.append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); // node 2 + plan.append_operator_as_node(opr.clone(), vec![0]) + .unwrap(); // node 3 + plan.append_operator_as_node(opr.clone(), vec![1, 2, 3]) + .unwrap(); // node 4 + + let (merge_node, subplans) = plan.get_branch_plans(plan.get_node(0).unwrap()); + let plan1 = LogicalPlan::with_root(Node::new(1, opr.clone())); + let plan2 = LogicalPlan::with_root(Node::new(2, opr.clone())); + let plan3 = LogicalPlan::with_root(Node::new(3, opr.clone())); + + assert_eq!(merge_node, plan.get_node(4)); + assert_eq!(subplans, vec![plan1, plan2, plan3]); + } +} diff --git a/research/query_service/ir/core/src/plan/meta.rs b/research/query_service/ir/core/src/plan/meta.rs new file mode 100644 index 000000000000..1e1b2853f2c5 --- /dev/null +++ b/research/query_service/ir/core/src/plan/meta.rs @@ -0,0 +1,666 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::cell::RefCell; +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet}; +use std::io; +use std::rc::Rc; +use std::sync::RwLock; + +use ir_common::generated::common as common_pb; +use ir_common::generated::schema as schema_pb; +use ir_common::NameOrId; + +use crate::error::{IrError, IrResult}; +use crate::JsonIO; + +pub static INVALID_META_ID: i32 = -1; + +lazy_static! { + pub static ref STORE_META: RwLock = RwLock::new(StoreMeta::default()); +} + +pub fn set_schema_from_json(read: R) { + if let Ok(mut meta) = STORE_META.write() { + if let Ok(schema) = Schema::from_json(read) { + meta.schema = Some(schema); + } + } +} + +/// The simple schema, mapping either label or property name into id. +pub fn set_schema_simple( + entities: Vec<(String, i32)>, relations: Vec<(String, i32)>, columns: Vec<(String, i32)>, +) { + if let Ok(mut meta) = STORE_META.write() { + let schema: Schema = (entities, relations, columns).into(); + meta.schema = Some(schema) + } +} + +pub fn reset_schema() { + if let Ok(mut meta) = STORE_META.write() { + meta.schema = None; + } +} + +#[derive(Clone, Debug, Default)] +pub struct StoreMeta { + pub schema: Option, +} + +#[derive(Clone, Debug)] +pub struct LabelMeta { + name: String, + id: i32, +} + +impl Default for LabelMeta { + fn default() -> Self { + Self { name: "INVALID".into(), id: INVALID_META_ID } + } +} + +impl From<(String, i32)> for LabelMeta { + fn from(tuple: (String, i32)) -> Self { + Self { name: tuple.0, id: tuple.1 } + } +} + +impl From for LabelMeta { + fn from(label: schema_pb::LabelMeta) -> Self { + Self { name: label.name, id: label.id } + } +} + +impl From for schema_pb::LabelMeta { + fn from(label: LabelMeta) -> Self { + Self { name: label.name, id: label.id } + } +} + +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub enum KeyType { + Entity = 0, + Relation = 1, + Column = 2, +} + +impl Default for KeyType { + fn default() -> Self { + Self::Entity + } +} + +#[derive(Clone, Debug, Default)] +pub struct Schema { + /// A map from table name to its internally encoded id + /// In the concept of graph database, this is also known as label + table_map: BTreeMap, + /// A map from column name to its internally encoded id + /// In the concept of graph database, this is also known as property + column_map: BTreeMap, + /// A reversed map of `id` to `name` mapping + id_name_rev: BTreeMap<(KeyType, i32), String>, + /// The source and destination labels of a given relation label's id + relation_labels: BTreeMap>, + /// Is the table name mapped as id + is_table_id: bool, + /// Is the column name mapped as id + is_column_id: bool, + /// Entities + entities: Vec, + /// Relations + rels: Vec, +} + +impl Schema { + pub fn get_table_id(&self, name: &str) -> Option { + self.table_map.get(name).cloned() + } + + pub fn get_table_id_from_pb(&self, name: &common_pb::NameOrId) -> Option { + name.item.as_ref().and_then(|item| match item { + common_pb::name_or_id::Item::Name(name) => self.get_table_id(name), + common_pb::name_or_id::Item::Id(id) => Some(*id), + }) + } + + pub fn get_column_id(&self, name: &str) -> Option { + self.column_map.get(name).cloned() + } + + pub fn get_column_id_from_pb(&self, name: &common_pb::NameOrId) -> Option { + name.item.as_ref().and_then(|item| match item { + common_pb::name_or_id::Item::Name(name) => self.get_column_id(name), + common_pb::name_or_id::Item::Id(id) => Some(*id), + }) + } + + pub fn get_name(&self, id: i32, ty: KeyType) -> Option<&String> { + self.id_name_rev.get(&(ty, id)) + } + + pub fn get_relation_labels(&self, relation: &NameOrId) -> Option<&Vec<(LabelMeta, LabelMeta)>> { + match relation { + NameOrId::Str(name) => self.relation_labels.get(name), + NameOrId::Id(id) => self + .get_name(*id, KeyType::Relation) + .and_then(|name| self.relation_labels.get(name)), + } + } + + pub fn is_column_id(&self) -> bool { + self.is_column_id + } + + pub fn is_table_id(&self) -> bool { + self.is_table_id + } +} + +impl From<(Vec<(String, i32)>, Vec<(String, i32)>, Vec<(String, i32)>)> for Schema { + fn from(tuple: (Vec<(String, i32)>, Vec<(String, i32)>, Vec<(String, i32)>)) -> Self { + let (entities, relations, columns) = tuple; + let mut schema = Schema::default(); + schema.is_table_id = !entities.is_empty() || !relations.is_empty(); + schema.is_column_id = !columns.is_empty(); + + if schema.is_table_id { + for (name, id) in entities.into_iter() { + schema.table_map.insert(name.clone(), id); + schema + .id_name_rev + .insert((KeyType::Entity, id), name); + } + for (name, id) in relations.into_iter() { + schema.table_map.insert(name.clone(), id); + schema + .id_name_rev + .insert((KeyType::Relation, id), name); + } + } + if schema.is_column_id { + for (name, id) in columns.into_iter() { + schema.column_map.insert(name.clone(), id); + schema + .id_name_rev + .insert((KeyType::Column, id), name); + } + } + + schema + } +} + +impl JsonIO for Schema { + fn into_json(self, writer: W) -> io::Result<()> { + let entities_pb: Vec = if !self.entities.is_empty() { + self.entities.clone() + } else { + let mut entities = Vec::new(); + for (&(ty, id), name) in &self.id_name_rev { + if ty == KeyType::Entity { + entities.push(schema_pb::EntityMeta { + label: Some(schema_pb::LabelMeta { id, name: name.to_string() }), + columns: vec![], + }) + } + } + entities + }; + + let relations_pb: Vec = if !self.rels.is_empty() { + self.rels.clone() + } else { + let mut relations = Vec::new(); + for (&(ty, id), name) in &self.id_name_rev { + if ty == KeyType::Relation { + let mut relation_meta = schema_pb::RelationMeta { + label: Some(schema_pb::LabelMeta { id, name: name.to_string() }), + entity_pairs: vec![], + columns: vec![], + }; + if let Some(labels) = self.get_relation_labels(&id.into()) { + relation_meta.entity_pairs = labels + .iter() + .cloned() + .map(|(src, dst)| schema_pb::relation_meta::LabelPair { + src: Some(src.into()), + dst: Some(dst.into()), + }) + .collect(); + } + relations.push(relation_meta); + } + } + relations + }; + + let schema_pb = schema_pb::Schema { + entities: entities_pb, + relations: relations_pb, + is_table_id: self.is_table_id, + is_column_id: self.is_column_id, + }; + serde_json::to_writer_pretty(writer, &schema_pb)?; + Ok(()) + } + + fn from_json(reader: R) -> io::Result + where + Self: Sized, + { + let schema_pb = serde_json::from_reader::<_, schema_pb::Schema>(reader)?; + let mut schema = Schema::default(); + schema.entities = schema_pb.entities.clone(); + schema.rels = schema_pb.relations.clone(); + schema.is_table_id = schema_pb.is_table_id; + schema.is_column_id = schema_pb.is_column_id; + for entity in schema_pb.entities { + if schema_pb.is_table_id { + if let Some(label) = &entity.label { + if !schema.table_map.contains_key(&label.name) { + schema + .table_map + .insert(label.name.clone(), label.id); + schema + .id_name_rev + .insert((KeyType::Entity, label.id), label.name.clone()); + } + } + } + if schema_pb.is_column_id { + for column in entity.columns { + if let Some(key) = &column.key { + if !schema.column_map.contains_key(&key.name) { + schema + .column_map + .insert(key.name.clone(), key.id); + schema + .id_name_rev + .insert((KeyType::Column, key.id), key.name.clone()); + } + } + } + } + } + + for rel in schema_pb.relations { + if schema_pb.is_table_id { + if let Some(label) = &rel.label { + if !schema.table_map.contains_key(&label.name) { + schema + .table_map + .insert(label.name.clone(), label.id); + schema + .id_name_rev + .insert((KeyType::Relation, label.id), label.name.clone()); + } + } + } + if schema_pb.is_column_id { + for column in rel.columns { + if let Some(key) = &column.key { + if !schema.column_map.contains_key(&key.name) { + schema + .column_map + .insert(key.name.clone(), key.id); + schema + .id_name_rev + .insert((KeyType::Column, key.id), key.name.clone()); + } + } + } + } + if let Some(label) = &rel.label { + let pairs = schema + .relation_labels + .entry(label.name.clone()) + .or_default(); + for entity_pair in rel.entity_pairs { + if entity_pair.src.is_some() && entity_pair.dst.is_some() { + pairs.push(( + entity_pair.src.clone().unwrap().into(), + entity_pair.dst.clone().unwrap().into(), + )) + } + } + } + } + + Ok(schema) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +/// Record the runtime schema of the node in the logical plan, for it being the vertex/edge +pub struct NodeMeta { + /// The table names (labels) + tables: BTreeSet, + /// The required columns (columns) + columns: BTreeSet, + /// A flag to indicate that all columns are required + is_all_columns: bool, + /// Whether the current node accept columns + is_add_column: bool, +} + +impl NodeMeta { + pub fn insert_column(&mut self, col: NameOrId) { + if self.is_add_column { + self.columns.insert(col); + } + } + + pub fn get_columns(&self) -> &BTreeSet { + &self.columns + } + + pub fn insert_table(&mut self, table: NameOrId) { + self.tables.insert(table); + } + + pub fn get_tables(&self) -> &BTreeSet { + &self.tables + } +} + +#[derive(Clone, Debug)] +pub enum CurrNodeOpt { + Single(u32), + Union(Vec), +} + +impl Default for CurrNodeOpt { + fn default() -> Self { + CurrNodeOpt::Single(0) + } +} + +#[derive(Clone, Debug)] +pub enum NodeMetaOpt { + Single(Rc>), + Union(Vec>>), +} + +impl NodeMetaOpt { + pub fn insert_column(&mut self, col: NameOrId) { + match self { + NodeMetaOpt::Single(meta) => meta.borrow_mut().insert_column(col), + NodeMetaOpt::Union(metas) => { + for meta in metas { + meta.borrow_mut().insert_column(col.clone()); + } + } + } + } + + pub fn set_is_all_columns(&mut self, is_all_columns: bool) { + match self { + NodeMetaOpt::Single(meta) => meta.borrow_mut().is_all_columns = is_all_columns, + NodeMetaOpt::Union(metas) => { + for meta in metas { + meta.borrow_mut().is_all_columns = is_all_columns; + } + } + } + } + + pub fn is_all_columns(&self) -> bool { + match self { + NodeMetaOpt::Single(meta) => meta.borrow().is_all_columns, + NodeMetaOpt::Union(metas) => { + for meta in metas { + if meta.borrow().is_all_columns { + return true; + } + } + false + } + } + } + + pub fn set_is_add_column(&mut self, is_add_column: bool) { + match self { + NodeMetaOpt::Single(meta) => meta.borrow_mut().is_add_column = is_add_column, + NodeMetaOpt::Union(metas) => { + for meta in metas { + meta.borrow_mut().is_add_column = is_add_column; + } + } + } + } + + pub fn get_columns(&self) -> BTreeSet { + match self { + NodeMetaOpt::Single(meta) => meta.borrow().get_columns().clone(), + NodeMetaOpt::Union(metas) => { + let mut union_cols = BTreeSet::new(); + for meta in metas { + union_cols.append(&mut meta.borrow_mut().columns); + } + union_cols + } + } + } +} + +/// To record any metadata while processing the logical plan, including: +/// * The tables/columns required by a given node +/// * The tag-node mutual mappings +/// * The tag-id mappings, if preprocessing tag to id +/// * TODO etc. +#[derive(Default, Clone, Debug)] +pub struct PlanMeta { + /// To record all possible tables/columns of a node, which is typically referred from a tag + /// while processing projection, selection, groupby, orderby, and etc. For example, when + /// select the record via an expression "a.name == \"John\"", the tag "a" must refer to + /// some node in the logical plan, and the node requires the column of \"John\". Such + /// information is critical in distributed processing, as the computation may not align + /// with the storage to access the required column. Thus, such information can help + /// the computation route and fetch columns. + node_metas: BTreeMap>>, + /// The tag must refer to some valid nodes in the plan. + tag_nodes: BTreeMap>, + /// To ease the processing, tag may be transformed to an internal id. + /// This maintains the mappings + tag_ids: BTreeMap, + /// To record the current nodes' id in the logical plan. Note that nodes that have operators that + /// of `As` or `Selection` does not alter curr_node. + curr_node: CurrNodeOpt, + /// The maximal tag id that has been assigned, for mapping tag ids. + max_tag_id: u32, + /// Whether to preprocess the table name into id. + is_table_id: bool, + /// Whether to preprocess the column name into id. + is_column_id: bool, + /// Whether to preprocess the tag name into id. + is_tag_id: bool, + /// Whether to partition the task + is_partition: bool, +} + +// Some constructors +impl PlanMeta { + pub fn new(node_id: u32) -> Self { + let mut plan_meta = PlanMeta::default(); + plan_meta.curr_node = CurrNodeOpt::Single(node_id); + plan_meta.node_metas.entry(node_id).or_default(); + plan_meta + } + + pub fn with_store_conf(mut self, is_table_id: bool, is_column_id: bool) -> Self { + self.is_table_id = is_table_id; + self.is_column_id = is_column_id; + self + } + + pub fn with_tag_id(mut self) -> Self { + self.is_tag_id = true; + self + } + + pub fn with_partition(mut self) -> Self { + self.is_partition = true; + self + } +} + +impl PlanMeta { + pub fn curr_node_metas_mut(&mut self) -> NodeMetaOpt { + match &self.curr_node { + CurrNodeOpt::Single(node) => NodeMetaOpt::Single( + self.node_metas + .entry(*node) + .or_default() + .clone(), + ), + CurrNodeOpt::Union(nodes) => { + let mut node_metas = vec![]; + for node in nodes { + node_metas.push( + self.node_metas + .entry(*node) + .or_default() + .clone(), + ); + } + NodeMetaOpt::Union(node_metas) + } + } + } + + pub fn tag_node_metas_mut(&mut self, tag_opt: Option<&NameOrId>) -> IrResult { + if let Some(tag) = tag_opt { + if let Some(nodes) = self.tag_nodes.get(tag) { + if nodes.len() == 1 { + Ok(NodeMetaOpt::Single( + self.node_metas + .entry(nodes[0]) + .or_default() + .clone(), + )) + } else { + let mut node_metas = vec![]; + for node in nodes { + node_metas.push( + self.node_metas + .entry(*node) + .or_default() + .clone(), + ) + } + Ok(NodeMetaOpt::Union(node_metas)) + } + } else { + Err(IrError::TagNotExist(tag.clone())) + } + } else { + Ok(self.curr_node_metas_mut()) + } + } + + pub fn get_node_meta(&self, id: u32) -> Option>> { + self.node_metas.get(&id).cloned() + } + + pub fn curr_node_metas(&self) -> Option { + match &self.curr_node { + CurrNodeOpt::Single(node) => self + .node_metas + .get(node) + .map(|meta| NodeMetaOpt::Single(meta.clone())), + CurrNodeOpt::Union(nodes) => { + let mut node_metas = vec![]; + for node in nodes { + if let Some(node_meta) = self.node_metas.get(node) { + node_metas.push(node_meta.clone()); + } else { + return None; + } + } + Some(NodeMetaOpt::Union(node_metas)) + } + } + } + + pub fn insert_tag_nodes(&mut self, tag: NameOrId, nodes: Vec) { + self.tag_nodes + .entry(tag) + .or_default() + .extend(nodes.into_iter()); + } + + pub fn get_tag_nodes(&self, tag: &NameOrId) -> Option> { + self.tag_nodes.get(tag).cloned() + } + + /// Get the id (with a `true` indicator) of the given tag if it already presents, + /// otherwise, set and return the id as `self.max_tag_id` (with a `false` indicator). + pub fn get_or_set_tag_id(&mut self, tag: &NameOrId) -> (bool, u32) { + let entry = self.tag_ids.entry(tag.clone()); + match entry { + Entry::Occupied(o) => (true, *o.get()), + Entry::Vacant(v) => { + let new_tag_id = self.max_tag_id; + v.insert(new_tag_id); + self.max_tag_id += 1; + (false, new_tag_id) + } + } + } + + pub fn set_curr_node(&mut self, curr_node: u32) { + self.curr_node = CurrNodeOpt::Single(curr_node); + } + + pub fn set_union_curr_nodes(&mut self, nodes: Vec) { + if nodes.len() == 1 { + self.curr_node = CurrNodeOpt::Single(nodes[0]); + } else { + self.curr_node = CurrNodeOpt::Union(nodes); + } + } + + pub fn get_curr_nodes(&self) -> Vec { + match &self.curr_node { + CurrNodeOpt::Single(node) => vec![*node], + CurrNodeOpt::Union(nodes) => nodes.clone(), + } + } + + /// Return a dummy node for maintaining tag that refers to neither vertex nor edge + pub fn get_dummy_nodes(&self) -> Vec { + vec![0xffffffff] + } + + pub fn is_table_id(&self) -> bool { + self.is_table_id + } + + pub fn is_column_id(&self) -> bool { + self.is_column_id + } + + pub fn is_tag_id(&self) -> bool { + self.is_tag_id + } + + pub fn is_partition(&self) -> bool { + self.is_partition + } +} diff --git a/research/query_service/ir/core/src/plan/mod.rs b/research/query_service/ir/core/src/plan/mod.rs new file mode 100644 index 000000000000..c4b0fbc8c8f9 --- /dev/null +++ b/research/query_service/ir/core/src/plan/mod.rs @@ -0,0 +1,20 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +pub mod ffi; +pub mod logical; +pub mod meta; +pub mod patmat; +pub mod physical; diff --git a/research/query_service/ir/core/src/plan/patmat.rs b/research/query_service/ir/core/src/plan/patmat.rs new file mode 100644 index 000000000000..b92fef01df97 --- /dev/null +++ b/research/query_service/ir/core/src/plan/patmat.rs @@ -0,0 +1,1680 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! + +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap, VecDeque}; +use std::convert::{TryFrom, TryInto}; +use std::fmt::Debug; +use std::rc::Rc; + +use ir_common::error::{ParsePbError, ParsePbResult}; +use ir_common::expr_parse::str_to_expr_pb; +use ir_common::generated::algebra as pb; +use ir_common::generated::common as common_pb; +use ir_common::NameOrId; + +use crate::error::{IrError, IrResult}; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd)] +#[repr(i32)] +pub enum BindingOpt { + Vertex = 0, + Edge = 1, + Path = 2, +} + +/// A trait to abstract how to build a logical plan for `Pattern` operator. +pub trait MatchingStrategy { + fn build_logical_plan(&self) -> IrResult; +} + +pub trait AsBaseSentence: Debug + MatchingStrategy { + /// Get base sentence if any + fn get_base(&self) -> Option<&BaseSentence>; + /// Get tags for the sentence + fn get_tags(&self) -> &BTreeSet; +} + +/// Define the behavior of concatenating two matching basic sentences via `Composition` +pub trait BasicSentence: AsBaseSentence { + /// Composite works by extending the first sentence with the second one via + /// the **only** common tag. Composition cannot be applied when either sentence + /// has anti-semantics. + fn composite(&self, other: Rc) -> Option; + /// Get the start tag of the `BasicSentence`, must present + fn get_start_tag(&self) -> &NameOrId; + /// Get the end tag of the `BasicSentence`, which is optional + fn get_end_tag(&self) -> Option<&NameOrId>; + /// Get the join kind, for identifying if the basic sentence carries the + /// anti/semi join semantics, which can not be composited + fn get_join_kind(&self) -> pb::join::JoinKind; + /// Get the reverse sentence, which not only reverse the start and end tag (must present), + /// and the direction of all edge/path expansions if possible. + fn reverse(&mut self) -> bool; +} + +/// Define the behavior of concatenating two matching sentences via `Join` +pub trait Sentence: AsBaseSentence { + fn join(&self, other: Rc) -> Option; +} + +/// An internal representation of `pb::Sentence` +#[derive(Clone, Debug, PartialEq)] +pub struct BaseSentence { + /// The start tag of this sentence + start_tag: NameOrId, + /// The end tag, if any, of this sentence + end_tag: Option, + /// The tags bound to this sentence + tags: BTreeSet, + /// Use `pb::logical_plan::Operator` rather than `pb::Pattern::binder`, + /// to facilitate building the logical plan that may translate a tag into an `As` operator. + operators: Vec, + /// Is this a sentence with Anti(No)-semanatics + join_kind: pb::join::JoinKind, + /// What kind of entities this sentence binds to + end_as: BindingOpt, + /// The number of filters contained by the sentence + num_filters: usize, + /// The number of hops of edge expansions of the sentence + num_hops: usize, + /// While transforming into a logical plan, will add a `pb::As(start_tag)` operator + /// to the logical plan + has_as_opr: bool, +} + +impl BaseSentence { + pub fn set_has_as_opr(&mut self, has_as_opr: bool) { + self.has_as_opr = has_as_opr; + } +} + +impl TryFrom for BaseSentence { + type Error = ParsePbError; + + fn try_from(pb: pb::pattern::Sentence) -> Result { + use pb::pattern::binder::Item; + + if !pb.binders.is_empty() { + let mut operators = vec![]; + let mut num_filters = 0; + let mut num_hops = 0; + let start_tag: NameOrId = pb + .start + .clone() + .ok_or(ParsePbError::EmptyFieldError("Pattern::Sentence::start".to_string()))? + .try_into()?; + + let end_tag: Option = pb + .end + .clone() + .map(|tag| tag.try_into()) + .transpose()?; + + let mut tags = BTreeSet::new(); + tags.insert(start_tag.clone()); + if let Some(et) = &end_tag { + tags.insert(et.clone()); + } + + let mut end_as = BindingOpt::Vertex; + let size = pb.binders.len(); + for (id, binder) in pb.binders.into_iter().enumerate() { + let opr = match binder.item { + Some(Item::Vertex(v)) => { + num_filters += detect_filters(v.params.as_ref()); + Ok(v.into()) + } + Some(Item::Edge(e)) => { + if id == size - 1 { + if e.is_edge { + end_as = BindingOpt::Edge; + } + } + num_filters += detect_filters(e.params.as_ref()); + num_hops += 1; + Ok(e.into()) + } + Some(Item::Path(p)) => { + if id == size - 1 { + end_as = BindingOpt::Path; + } + if let Some(range) = &p.hop_range { + num_hops += range.upper as usize; + if let Some(base) = &p.base { + num_filters += detect_filters(base.params.as_ref()) * range.upper as usize; + } + } + Ok(p.into()) + } + None => Err(ParsePbError::EmptyFieldError("Pattern::Binder::item".to_string())), + }?; + operators.push(opr); + } + let join_kind = unsafe { std::mem::transmute(pb.join_kind) }; + if end_as == BindingOpt::Path { + Err(ParsePbError::Unsupported("binding a sentence to a path".to_string())) + } else { + Ok(Self { + start_tag, + end_tag, + tags, + operators, + join_kind, + end_as, + num_filters, + num_hops, + has_as_opr: true, + }) + } + } else { + Err(ParsePbError::EmptyFieldError("Pattern::Sentence::start".to_string())) + } + } +} + +impl MatchingStrategy for BaseSentence { + fn build_logical_plan(&self) -> IrResult { + let mut plan = pb::LogicalPlan { nodes: vec![] }; + let size = self.operators.len(); + if size == 0 { + Err(IrError::InvalidPattern("empty sentence".to_string())) + } else { + let mut child_offset = 1; + if self.has_as_opr { + plan.nodes.push(pb::logical_plan::Node { + // pb::NameOrId -> NameOrId never fails. + opr: Some(pb::As { alias: self.start_tag.clone().try_into().ok() }.into()), + children: vec![1], + }); + child_offset += 1; + } + + for (idx, opr) in self.operators.iter().enumerate() { + let child_id = idx as i32 + child_offset; + let node = if idx != size - 1 { + // A sentence is definitely a chain + pb::logical_plan::Node { opr: Some(opr.clone()), children: vec![child_id] } + } else { + if self.end_tag.is_some() { + pb::logical_plan::Node { opr: Some(opr.clone()), children: vec![child_id] } + } else { + pb::logical_plan::Node { opr: Some(opr.clone()), children: vec![] } + } + }; + plan.nodes.push(node); + } + if let Some(end_tag) = self.end_tag.clone() { + plan.nodes.push(pb::logical_plan::Node { + // pb::NameOrId -> NameOrId never fails. + opr: Some(pb::As { alias: end_tag.try_into().ok() }.into()), + children: vec![], + }); + } + + Ok(plan) + } + } +} + +impl AsBaseSentence for BaseSentence { + fn get_base(&self) -> Option<&BaseSentence> { + Some(self) + } + fn get_tags(&self) -> &BTreeSet { + &self.tags + } +} + +fn reverse_dir(dir: i32) -> i32 { + if dir == 0 { + 1 + } else if dir == 1 { + 0 + } else { + 2 + } +} + +impl BasicSentence for BaseSentence { + fn composite(&self, other: Rc) -> Option { + // can only composite two sentences that are both inner-join + if self.get_join_kind() != pb::join::JoinKind::Inner + || other.get_join_kind() != pb::join::JoinKind::Inner + { + return None; + } + let mut tags = self.get_tags().clone(); + // composition can happens if `self` sentence contains the start_tag of `other` + if !tags.contains(other.get_start_tag()) { + return None; + } + if tags.intersection(other.get_tags()).count() > 1 { + return None; + } + for tag in other.get_tags() { + tags.insert(tag.clone()); + } + + Some(CompoSentence { head: Rc::new(self.clone()), tail: Some(other), tags }) + } + + fn get_start_tag(&self) -> &NameOrId { + &self.start_tag + } + + fn get_end_tag(&self) -> Option<&NameOrId> { + self.end_tag.as_ref() + } + + fn get_join_kind(&self) -> pb::join::JoinKind { + self.join_kind + } + + fn reverse(&mut self) -> bool { + use pb::logical_plan::operator::Opr; + if let Some(end_tag) = self.end_tag.clone() { + let mut result = true; + let mut new_operators = self.operators.clone(); + for operator in new_operators.iter_mut() { + if let Some(opr) = operator.opr.as_mut() { + match opr { + Opr::Vertex(_) => {} + Opr::Edge(edge) => { + if edge.is_edge { + result = false; + break; + } + if let Some(params) = edge.params.as_mut() { + if !params.columns.is_empty() + || params.is_all_columns + || params.predicate.is_some() + { + result = false; + break; + } + } + edge.direction = reverse_dir(edge.direction); + } + Opr::Path(path) => { + if let Some(base) = path.base.as_mut() { + base.direction = reverse_dir(base.direction); + } + } + _ => {} + } + } + } + if result { + self.end_tag = Some(self.start_tag.clone()); + self.start_tag = end_tag; + self.operators = new_operators; + } + + result + } else { + false + } + } +} + +impl Sentence for BaseSentence { + fn join(&self, other: Rc) -> Option { + let common_tags: BTreeSet = self + .get_tags() + .intersection(&other.get_tags()) + .cloned() + .collect(); + + if !common_tags.is_empty() { + let tags: BTreeSet = self + .get_tags() + .union(&other.get_tags()) + .cloned() + .collect(); + + if let Some(base) = other.get_base() { + // cannot join two sentences that both have Non-inner join_type + if self.get_join_kind() != pb::join::JoinKind::Inner + && base.join_kind != pb::join::JoinKind::Inner + { + return None; + } + } + if self.get_join_kind() != pb::join::JoinKind::Inner { + Some(JoinSentence { + left: other, + // anti-sentence must be placed on the right + right: Some(Rc::new(self.clone())), + common_tags, + tags, + join_kind: pb::join::JoinKind::Anti, + }) + } else { + let join_kind = if let Some(base) = other.get_base() { + // if other is an non-inner sentence + base.join_kind + } else { + pb::join::JoinKind::Inner + }; + + Some(JoinSentence { + left: Rc::new(self.clone()), + right: Some(other), + common_tags, + tags, + join_kind, + }) + } + } else { + None + } + } +} + +/// Defines merging an array of `BaseSentence` with the same start and end tag +#[derive(Clone, Debug)] +pub struct MergedSentence { + /// The common start tag of the merged sentences + start_tag: NameOrId, + /// The common end tag, if any, of the merged sentences + end_tag: Option, + /// The tags bound to the merged sentences + tags: BTreeSet, + /// The merged sentences + bases: Vec, +} + +impl MergedSentence { + pub fn set_has_as_opr(&mut self, has_as_opr: bool) { + for base in self.bases.iter_mut() { + base.set_has_as_opr(has_as_opr); + } + } +} + +impl From for MergedSentence { + fn from(base: BaseSentence) -> Self { + Self { + start_tag: base.start_tag.clone(), + end_tag: base.end_tag.clone(), + tags: base.tags.clone(), + bases: vec![base], + } + } +} + +impl From for CompoSentence { + fn from(merged: MergedSentence) -> Self { + let tags = merged.tags.clone(); + Self { head: Rc::new(merged), tail: None, tags } + } +} + +impl TryFrom for JoinSentence { + type Error = IrError; + + fn try_from(merged: MergedSentence) -> IrResult { + if merged.bases.is_empty() { + Err(IrError::InvalidPattern("empty `MergedSentence` is not allowed".to_string())) + } else { + let tags = merged.tags.clone(); + let mut queue = merged + .bases + .into_iter() + .collect::>(); + let mut first: JoinSentence = queue.pop_front().unwrap().into(); + if first.join_kind != pb::join::JoinKind::Inner { + return Err(IrError::InvalidPattern( + "the first sentence of `MergedSentence` must have InnerJoin".to_string(), + )); + } + + while !queue.is_empty() { + let second = queue.pop_front().unwrap(); + let join_kind = second.join_kind; + first = JoinSentence { + left: Rc::new(first), + right: Some(Rc::new(second)), + common_tags: tags.clone(), + tags: tags.clone(), + join_kind, + }; + } + + Ok(first) + } + } +} + +impl MergedSentence { + pub fn new(mut s1: BaseSentence, mut s2: BaseSentence) -> Option { + if s1.start_tag == s2.start_tag && s1.end_tag == s2.end_tag { + if s1.join_kind != pb::join::JoinKind::Inner { + std::mem::swap(&mut s1, &mut s2); + } + Some(Self { + start_tag: s1.start_tag.clone(), + end_tag: s1.end_tag.clone(), + tags: s1.tags.clone(), + bases: vec![s1, s2], + }) + } else { + None + } + } + + pub fn merge(&mut self, base: BaseSentence) -> bool { + if self.start_tag == base.start_tag && self.end_tag == base.end_tag { + self.bases.push(base); + for i in self.bases.len() - 1..1 { + if self.bases[i].join_kind == pb::join::JoinKind::Inner + && self.bases[i - 1].join_kind != pb::join::JoinKind::Inner + { + self.bases.swap(i, i - 1); + } else { + break; + } + } + true + } else { + false + } + } +} + +impl MatchingStrategy for MergedSentence { + fn build_logical_plan(&self) -> IrResult { + JoinSentence::try_from(self.clone())?.build_logical_plan() + } +} + +impl AsBaseSentence for MergedSentence { + fn get_base(&self) -> Option<&BaseSentence> { + None + } + + fn get_tags(&self) -> &BTreeSet { + &self.tags + } +} + +impl BasicSentence for MergedSentence { + fn composite(&self, other: Rc) -> Option { + // can only composite two sentences that are both inner-join + if self.get_join_kind() != pb::join::JoinKind::Inner + || other.get_join_kind() != pb::join::JoinKind::Inner + { + return None; + } + let mut tags = self.get_tags().clone(); + // composition can happens if `self` sentence contains the start_tag of `other` + if !tags.contains(other.get_start_tag()) { + return None; + } + if tags.intersection(other.get_tags()).count() > 1 { + return None; + } + for tag in other.get_tags() { + tags.insert(tag.clone()); + } + + Some(CompoSentence { head: Rc::new(self.clone()), tail: Some(other), tags }) + } + + fn get_start_tag(&self) -> &NameOrId { + &self.start_tag + } + + fn get_end_tag(&self) -> Option<&NameOrId> { + self.end_tag.as_ref() + } + + fn get_join_kind(&self) -> pb::join::JoinKind { + pb::join::JoinKind::Inner + } + + fn reverse(&mut self) -> bool { + if let Some(end_tag) = self.end_tag.clone() { + let mut result = true; + let mut rev_bases = self.bases.clone(); + for base in rev_bases.iter_mut() { + if !base.reverse() { + result = false; + break; + } + } + if result { + self.end_tag = Some(self.start_tag.clone()); + self.start_tag = end_tag; + self.bases = rev_bases; + } + + result + } else { + false + } + } +} + +impl Sentence for MergedSentence { + fn join(&self, other: Rc) -> Option { + if let Ok(this) = JoinSentence::try_from(self.clone()) { + this.join(other) + } else { + None + } + } +} + +/// Define the sentences after composition +#[derive(Clone, Debug)] +pub struct CompoSentence { + /// The head sentence of the composition + head: Rc, + /// The tail sentence of the composition + tail: Option>, + /// The union tags of the sentences after composition + tags: BTreeSet, +} + +/// To obtain the nodes in `plan` that has no parent +fn get_first_nodes(plan: &pb::LogicalPlan) -> IrResult> { + let mut nodes: BTreeSet = (0..plan.nodes.len() as u32).collect(); + for node in &plan.nodes { + for child in &node.children { + nodes.remove(&(*child as u32)); + } + } + if nodes.is_empty() { + Err(IrError::InvalidPattern(format!("fail to obtain first nodes from plan {:?}", plan))) + } else { + Ok(nodes) + } +} + +impl MatchingStrategy for CompoSentence { + fn build_logical_plan(&self) -> IrResult { + let mut plan = self.head.build_logical_plan()?; + if let Some(tail) = &self.tail { + let start_tag = self.get_start_tag(); + let mut last_node = plan.nodes.len() as u32 - 1; + if start_tag == tail.get_start_tag() { + let mut expr_str = "@".to_string(); + match start_tag { + NameOrId::Str(s) => expr_str.push_str(s), + NameOrId::Id(i) => expr_str.push_str(&i.to_string()), + } + let project_node = pb::logical_plan::Node { + opr: Some(pb::logical_plan::Operator { + opr: Some(pb::logical_plan::operator::Opr::Project(pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: Some(str_to_expr_pb(expr_str)?), + alias: None, + }], + is_append: true, + })), + }), + children: vec![], + }; + if let Some(n) = plan.nodes.get_mut(last_node as usize) { + n.children.push(last_node as i32 + 1); + } + // update the last node as the newly added project node + last_node += 1; + plan.nodes.push(project_node); + } + let tail_plan = tail.build_logical_plan()?; + let tail_first_nodes: BTreeSet = get_first_nodes(&tail_plan)?; + // set the first nodes in tail plan as the children of last node of head plan. + if let Some(n) = plan.nodes.get_mut(last_node as usize) { + n.children.extend( + tail_first_nodes + .into_iter() + .map(|id| (id + last_node + 1) as i32), + ); + } + // push the nodes in tail_plan into the resulted plan + for mut node in tail_plan.nodes { + for child in node.children.iter_mut() { + // update the child node's id + *child += last_node as i32 + 1; + } + plan.nodes.push(node); + } + } + Ok(plan) + } +} + +impl AsBaseSentence for CompoSentence { + fn get_base(&self) -> Option<&BaseSentence> { + None + } + fn get_tags(&self) -> &BTreeSet { + &self.tags + } +} + +impl BasicSentence for CompoSentence { + fn composite(&self, other: Rc) -> Option { + // can only composite two sentences that are both inner-join + if self.get_join_kind() != pb::join::JoinKind::Inner + || other.get_join_kind() != pb::join::JoinKind::Inner + { + return None; + } + let mut tags = self.get_tags().clone(); + // composition can happens if `self` sentence contains the start_tag of `other` + if !tags.contains(other.get_start_tag()) { + return None; + } + if tags.intersection(other.get_tags()).count() > 1 { + return None; + } + for tag in other.get_tags() { + tags.insert(tag.clone()); + } + + Some(CompoSentence { head: Rc::new(self.clone()), tail: Some(other), tags }) + } + + fn get_start_tag(&self) -> &NameOrId { + self.head.get_start_tag() + } + + fn get_end_tag(&self) -> Option<&NameOrId> { + if let Some(tail) = &self.tail { + tail.get_end_tag() + } else { + None + } + } + + fn get_join_kind(&self) -> pb::join::JoinKind { + pb::join::JoinKind::Inner + } + + fn reverse(&mut self) -> bool { + false + } +} + +impl Sentence for CompoSentence { + fn join(&self, other: Rc) -> Option { + let common_tags: BTreeSet = self + .get_tags() + .intersection(&other.get_tags()) + .cloned() + .collect(); + if !common_tags.is_empty() { + let tags: BTreeSet = self + .get_tags() + .union(&other.get_tags()) + .cloned() + .collect(); + let join_kind = if let Some(base) = other.get_base() { + base.get_join_kind() + } else { + pb::join::JoinKind::Inner + }; + Some(JoinSentence { + left: Rc::new(self.clone()), + right: Some(other), + common_tags, + tags, + join_kind, + }) + } else { + None + } + } +} + +#[derive(Clone, Debug)] +pub struct JoinSentence { + /// The left sentence + left: Rc, + /// The right sentence + right: Option>, + /// The common tags of left and right for join + common_tags: BTreeSet, + /// The union tags of left and right + tags: BTreeSet, + /// The join kind, either `Inner`, or `Anti` + join_kind: pb::join::JoinKind, +} + +impl From for JoinSentence { + fn from(base: BaseSentence) -> Self { + let join_kind = base.join_kind; + let tags = base.tags.clone(); + Self { left: Rc::new(base), right: None, common_tags: tags.clone(), tags, join_kind } + } +} + +impl From for JoinSentence { + fn from(combo: CompoSentence) -> Self { + let join_kind = pb::join::JoinKind::Inner; + let tags = combo.tags.clone(); + Self { left: Rc::new(combo), right: None, common_tags: tags.clone(), tags, join_kind } + } +} + +fn detect_filters(params_opt: Option<&pb::QueryParams>) -> usize { + if let Some(params) = params_opt { + let mut count = 0; + // Simply count whether there is any predicate, without actually looking into + // the number of filter clauses + if params.predicate.is_some() { + count += 1; + } + // Simply count whether there is any column (for filtering) + if !params.columns.is_empty() { + count += 1; + } + count + } else { + 0 + } +} + +impl MatchingStrategy for JoinSentence { + fn build_logical_plan(&self) -> IrResult { + let mut plan = self.left.build_logical_plan()?; + let mut right_plan_opt = None; + if let Some(right) = &self.right { + right_plan_opt = Some(right.build_logical_plan()?); + } + if let Some(mut right_plan) = right_plan_opt { + let left_size = plan.nodes.len(); + let right_size = right_plan.nodes.len(); + let join_node_idx = (left_size + right_size) as i32; + + for (idx, node) in right_plan.nodes.iter_mut().enumerate() { + for child in node.children.iter_mut() { + *child += left_size as i32; + } + if idx == right_size - 1 { + node.children.push(join_node_idx); + } + } + if let Some(node) = plan.nodes.last_mut() { + node.children.push(join_node_idx); + } + plan.nodes.extend(right_plan.nodes.into_iter()); + let keys = self + .common_tags + .iter() + .cloned() + .map(|tag| common_pb::Variable { tag: tag.try_into().ok(), property: None }) + .collect::>(); + plan.nodes.push(pb::logical_plan::Node { + opr: Some( + pb::Join { + left_keys: keys.clone(), + right_keys: keys, + kind: unsafe { std::mem::transmute(self.join_kind) }, + } + .into(), + ), + children: vec![], + }); + } + + Ok(plan) + } +} + +impl AsBaseSentence for JoinSentence { + fn get_base(&self) -> Option<&BaseSentence> { + if self.right.is_none() { + self.left.get_base() + } else { + None + } + } + + fn get_tags(&self) -> &BTreeSet { + &self.tags + } +} + +impl Sentence for JoinSentence { + fn join(&self, other: Rc) -> Option { + let common_tags: BTreeSet = self + .tags + .intersection(&other.get_tags()) + .cloned() + .collect(); + let tags: BTreeSet = self + .tags + .union(&other.get_tags()) + .cloned() + .collect(); + if common_tags.is_empty() { + return None; + } + if let Some(base) = self.get_base() { + base.join(other) + } else if let Some(base) = other.get_base() { + base.join(Rc::new(self.clone())) + } else { + Some(JoinSentence { + left: Rc::new(self.clone()), + right: Some(other), + common_tags, + tags, + join_kind: pb::join::JoinKind::Inner, + }) + } + } +} + +/// A naive implementation of `MatchingStrategy` +#[derive(Clone, Default)] +pub struct NaiveStrategy { + /// To record the set of tags a given tag may connect to (out) and be connected with (in) + tag_adj_list: BTreeMap>, VecDeque>)>, + /// The matching sentence + sentences: BTreeMap<(NameOrId, Option), MergedSentence>, + /// The start tag is the start_tag of the first matching sentence + start_tag: NameOrId, +} + +impl From> for NaiveStrategy { + fn from(bases: Vec) -> Self { + let mut tag_out_set = BTreeMap::new(); + let mut tag_in_set = BTreeMap::new(); + let mut sentences: BTreeMap<_, MergedSentence> = BTreeMap::new(); + let mut match_start_tag = None; + for base in bases { + let start_tag = base.start_tag.clone(); + if match_start_tag.is_none() { + match_start_tag = Some(start_tag.clone()); + } + let end_tag_opt = base.end_tag.clone(); + let entry = sentences.entry((start_tag.clone(), end_tag_opt.clone())); + match entry { + Entry::Vacant(vac) => { + vac.insert(base.into()); + tag_out_set + .entry(start_tag.clone()) + .or_insert_with(BinaryHeap::new) + .push(end_tag_opt.clone()); + if let Some(end_tag) = end_tag_opt { + tag_in_set + .entry(end_tag.clone()) + .or_insert_with(BinaryHeap::new) + .push(Some(start_tag.clone())); + } + } + Entry::Occupied(mut occ) => { + occ.get_mut().merge(base); + } + } + } + let mut tag_adj_list = BTreeMap::new(); + for (k, v) in tag_out_set { + let out_list = v + .into_sorted_vec() + .into_iter() + .collect::>(); + tag_adj_list.insert(k, (out_list, VecDeque::new())); + } + for (k, v) in tag_in_set { + (tag_adj_list.entry(k).or_default()) + .1 + .extend(v.into_sorted_vec().into_iter()); + } + + Self { tag_adj_list, sentences, start_tag: match_start_tag.unwrap_or_default() } + } +} + +// TODO(longbin) For now, the either the `start` tag or `end` tag of each sentence in +// TODO(longbin) `pb::Pattern` does not exist outside of the match operator. +impl TryFrom for NaiveStrategy { + type Error = ParsePbError; + + fn try_from(pb: pb::Pattern) -> Result { + let bases = pb + .sentences + .into_iter() + .map(|s| s.try_into()) + .collect::>>()?; + if !bases.is_empty() { + Ok(Self::from(bases)) + } else { + Err(ParsePbError::EmptyFieldError("empty sentences in `Pattern`".to_string())) + } + } +} + +fn get_next_tag( + nbrs: &mut VecDeque>, visited: &BTreeSet, +) -> Option> { + let mut end_tag_opt = None; + let len = nbrs.len(); + let mut count = 0; + while !nbrs.is_empty() { + let test_tag = nbrs.pop_front().unwrap(); + let mut is_visited = false; + if let Some(tag) = &test_tag { + is_visited = visited.contains(tag); + } + if is_visited { + nbrs.push_back(test_tag); + count += 1; + if count == len { + // means all the neighbors have been visited + break; + } + } else { + end_tag_opt = Some(test_tag); + break; + } + } + + end_tag_opt +} + +impl NaiveStrategy { + fn get_start_tag(&self) -> Option { + if self.tag_adj_list.contains_key(&self.start_tag) { + Some(self.start_tag.clone()) + } else { + let mut max_cnt = 0; + let mut start_tag = None; + for (tag, (nbrs, _)) in &self.tag_adj_list { + if max_cnt < nbrs.len() { + max_cnt = nbrs.len(); + start_tag = Some(tag.clone()); + } + } + start_tag + } + } + + fn is_empty(&self) -> bool { + self.sentences.is_empty() + } + + fn do_dfs( + &mut self, start_tag: NameOrId, is_initial: bool, visited: &mut BTreeSet, + ) -> Option { + let mut result = None; + let mut should_remove = false; + visited.insert(start_tag.clone()); + if let Some((out_nbrs, in_nbrs)) = self.tag_adj_list.get_mut(&start_tag) { + let mut is_in_tag = false; + let mut next_tag_opt = get_next_tag(out_nbrs, visited); + if next_tag_opt.is_none() { + next_tag_opt = get_next_tag(in_nbrs, visited); + is_in_tag = true; + } + should_remove = out_nbrs.is_empty() && in_nbrs.is_empty(); + if let Some(next_tag) = next_tag_opt { + let tag_key = if is_in_tag { + // it is fine to unwrap as it must present + (next_tag.clone().unwrap(), Some(start_tag.clone())) + } else { + (start_tag.clone(), next_tag.clone()) + }; + if let Some(mut sentence) = self.sentences.remove(&tag_key) { + sentence.set_has_as_opr(is_initial); + if is_in_tag { + if !sentence.reverse() { + in_nbrs.push_back(next_tag.clone()); + self.sentences.insert(tag_key, sentence); + return None; + } + } + if let Some(next_start_tag) = next_tag { + if let Some(next_sentence) = self.do_dfs(next_start_tag, false, visited) { + result = sentence.composite(Rc::new(next_sentence)); + } else { + result = Some(sentence.into()) + } + } else { + result = Some(sentence.into()) + } + } + } + } + if should_remove { + self.tag_adj_list.remove(&start_tag); + } + visited.remove(&start_tag); + + result + } + + fn do_composition(&mut self) -> IrResult> { + let mut results = VecDeque::new(); + while !self.is_empty() { + let mut visited = BTreeSet::new(); + if let Some(start_tag) = self.get_start_tag() { + if let Some(sentence) = self.do_dfs(start_tag, true, &mut visited) { + results.push_back(sentence); + } else { + return Err(IrError::InvalidPattern("the pattern may be disconnected".to_string())); + } + } else { + if !self.is_empty() { + return Err(IrError::InvalidPattern("the pattern may be disconnected".to_string())); + } + } + } + Ok(results) + } +} + +impl MatchingStrategy for NaiveStrategy { + fn build_logical_plan(&self) -> IrResult { + let mut sentences = self + .clone() + .do_composition()? + .into_iter() + .map(|s| JoinSentence::from(s)) + .collect::>(); + + if !sentences.is_empty() { + let mut first = sentences.pop_front().unwrap(); + while !sentences.is_empty() { + let old_count = sentences.len(); + let mut new_count = 0; + for _ in 0..old_count { + let second = sentences.pop_front().unwrap(); + if let Some(join) = first.join(Rc::new(second.clone())) { + first = join; + } else { + sentences.push_back(second); + new_count += 1; + } + } + if new_count != 0 { + if new_count == old_count { + return Err(IrError::InvalidPattern( + "the matching pattern may be disconnected".to_string(), + )); + } + } else { + break; + } + } + first.build_logical_plan() + } else { + Err(IrError::InvalidPattern("empty sentences".to_string())) + } + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use super::*; + + #[allow(dead_code)] + fn query_params() -> pb::QueryParams { + pb::QueryParams { + tables: vec![], + columns: vec![], + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + } + } + + #[allow(dead_code)] + fn gen_sentence_x_out_y(x: &str, y: Option<&str>, is_edge: bool, is_anti: bool) -> BaseSentence { + let pb = pb::pattern::Sentence { + start: x.try_into().ok(), + binders: vec![pb::pattern::Binder { + item: Some(pb::pattern::binder::Item::Edge(pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params()), + is_edge, + alias: None, + })), + }], + end: y.and_then(|s| s.try_into().ok()), + join_kind: if is_anti { 5 } else { 0 }, + }; + + pb.try_into().unwrap() + } + + #[allow(dead_code)] + fn gen_sentence_x_out_y_all_columns(x: &str, y: Option<&str>) -> BaseSentence { + let mut params = query_params(); + params.is_all_columns = true; + let pb = pb::pattern::Sentence { + start: x.try_into().ok(), + binders: vec![pb::pattern::Binder { + item: Some(pb::pattern::binder::Item::Edge(pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(params), + is_edge: false, + alias: None, + })), + }], + end: y.and_then(|s| s.try_into().ok()), + join_kind: 0, + }; + + pb.try_into().unwrap() + } + + #[allow(dead_code)] + fn get_sentence_x_inv_y(x: &str, y: Option<&str>, is_anti: bool) -> BaseSentence { + let pb = pb::pattern::Sentence { + start: x.try_into().ok(), + binders: vec![pb::pattern::Binder { + item: Some(pb::pattern::binder::Item::Vertex(pb::GetV { + tag: None, + opt: 0, + params: Some(query_params()), + alias: None, + })), + }], + end: y.and_then(|s| s.try_into().ok()), + join_kind: if is_anti { 5 } else { 0 }, + }; + + pb.try_into().unwrap() + } + + #[test] + fn sentence_composition() { + // case 1. + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let b_out_c = gen_sentence_x_out_y("b", Some("c"), false, false); + let a_out_b_out_c = a_out_b.composite(Rc::new(b_out_c)).unwrap(); + assert_eq!( + a_out_b_out_c.tags, + vec!["a".into(), "b".into(), "c".into()] + .into_iter() + .collect() + ); + assert_eq!(a_out_b_out_c.get_start_tag().clone(), "a".into()); + assert_eq!(a_out_b_out_c.get_end_tag().unwrap().clone(), "c".into()); + + // case 2. + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let a_out_c = gen_sentence_x_out_y("a", Some("c"), false, false); + let a_out_b_out_c = a_out_b.composite(Rc::new(a_out_c)).unwrap(); + + assert_eq!( + a_out_b_out_c.tags, + vec!["a".into(), "b".into(), "c".into()] + .into_iter() + .collect() + ); + assert_eq!(a_out_b_out_c.get_start_tag().clone(), "a".into()); + assert_eq!(a_out_b_out_c.get_end_tag().unwrap().clone(), "c".into()); + + // case 3. + let a_out = gen_sentence_x_out_y("a", None, false, false); + let a_out_c = gen_sentence_x_out_y("a", Some("c"), false, false); + let a_out_out_c = a_out.composite(Rc::new(a_out_c)).unwrap(); + + assert_eq!(a_out_out_c.get_start_tag().clone(), "a".into()); + assert_eq!(a_out_out_c.get_end_tag().unwrap().clone(), "c".into()); + + // case 4: cannot composite without sharing tags + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let c_out_d = gen_sentence_x_out_y("c", Some("d"), false, false); + + assert!(a_out_b.composite(Rc::new(c_out_d)).is_none()); + + // case 5: cannot composite anti-sentence + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let b_out_c = gen_sentence_x_out_y("b", Some("c"), false, true); + + assert!(a_out_b.composite(Rc::new(b_out_c)).is_none()); + + // case 6: cannot composite while sharing more than two tags + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let a_out_b2 = gen_sentence_x_out_y("a", Some("b"), false, false); + + assert!(a_out_b.composite(Rc::new(a_out_b2)).is_none()); + } + + #[test] + fn sentence_into_logical_plan() { + // case 1. + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + + // As(a), Out(), As(b) + let plan = a_out_b.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 3); + println!("case 1: {:?}", plan.nodes); + + // case 2. + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let mut a_out_c = gen_sentence_x_out_y("a", Some("c"), false, false); + a_out_c.set_has_as_opr(false); + let a_out_b_out_c = a_out_b.composite(Rc::new(a_out_c)).unwrap(); + // As(a), Out(), As(b), Project(a), out(), As(c) + let plan = a_out_b_out_c.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 6); + println!("case 2: {:?}", plan.nodes); + + // case 3. merged case + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let merged = MergedSentence::new(a_out_b.clone(), a_out_b).unwrap(); + let plan = merged.build_logical_plan().unwrap(); + // As(a), Out(), As(b), As(a), out(), As(b), Join + assert_eq!(plan.nodes.len(), 7); + println!("case 3: {:?}", plan.nodes); + + // case 4. concatenate merged sentence + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let b_out_c = gen_sentence_x_out_y("b", Some("c"), false, false); + let mut merged = MergedSentence::new(b_out_c.clone(), b_out_c).unwrap(); + merged.set_has_as_opr(false); + let a_out_c = a_out_b.composite(Rc::new(merged)).unwrap(); + + let plan = a_out_c.build_logical_plan().unwrap(); + // As(a), Out(), As(b), Out(), As(c), Out(), As(c), Join + assert_eq!(plan.nodes.len(), 8); + println!("case 4: {:?}", plan.nodes); + } + + #[test] + fn sentence_join() { + // case 1. + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let a_out_b2 = a_out_b.clone(); + + let join = a_out_b.join(Rc::new(a_out_b2)).unwrap(); + assert_eq!( + join.common_tags, + vec!["a".into(), "b".into()] + .into_iter() + .collect() + ); + assert_eq!(join.join_kind, pb::join::JoinKind::Inner); + + let plan = join.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 7); + assert_eq!(plan.nodes.get(2).unwrap().children, vec![6]); + assert_eq!(plan.nodes.get(5).unwrap().children, vec![6]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("b".into()), property: None } + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("b".into()), property: None } + ], + kind: 0 + } + .into() + ); + + // case 2. + let a_out_b_anti = gen_sentence_x_out_y("a", Some("b"), false, true); + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + + let join = a_out_b_anti.join(Rc::new(a_out_b)).unwrap(); + assert_ne!(join.left.get_base().unwrap().join_kind, pb::join::JoinKind::Anti); + // anti-sentence moved to the right + assert_eq!( + join.right + .as_ref() + .unwrap() + .get_base() + .unwrap() + .join_kind, + pb::join::JoinKind::Anti + ); + assert_eq!( + join.common_tags, + vec!["a".into(), "b".into()] + .into_iter() + .collect() + ); + assert_eq!(join.join_kind, pb::join::JoinKind::Anti); + + // case 3: cannot join without sharing tags + let a_out_b = gen_sentence_x_out_y("a", Some("b"), false, false); + let c_out_d = gen_sentence_x_out_y("c", Some("d"), false, false); + + assert!(a_out_b.join(Rc::new(c_out_d)).is_none()); + + // case 4 + let a_out_b_anti = gen_sentence_x_out_y("a", Some("b"), false, true); + let a_out_b_anti2 = a_out_b_anti.clone(); + + assert!(a_out_b_anti + .join(Rc::new(a_out_b_anti2)) + .is_none()); + } + + #[test] + fn pattern_case1_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("b", Some("c"), false, false), + gen_sentence_x_out_y("b", Some("c"), false, false), + ]); + + // As(a), Out(), As(b), + // Join(id = 7) ( + // Out() (id = 3), As(c), + // Out() (id = 5), As(c), + // ) + let plan = strategy.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 8); + assert_eq!(plan.nodes.get(2).unwrap().children, vec![3, 5]); + assert_eq!(plan.nodes.get(4).unwrap().children, vec![7]); + assert_eq!(plan.nodes.get(6).unwrap().children, vec![7]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("b".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None } + ], + right_keys: vec![ + common_pb::Variable { tag: Some("b".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None } + ], + kind: 0 + } + .into() + ); + } + + #[test] + fn pattern_case2_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("b", Some("c"), false, false), + gen_sentence_x_out_y("a", Some("c"), false, false), + ]); + + // Join (id = 8) ( + // As(a), Out(), As(b), Out(), As(c) (id = 4) + // As(a), Out(), As(c) (id = 7) + // ) + let plan = strategy.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.get(4).unwrap().children, vec![8]); + assert_eq!( + plan.nodes.get(4).unwrap().opr.clone().unwrap(), + pb::As { alias: "c".try_into().ok() }.into() + ); + assert_eq!(plan.nodes.get(7).unwrap().children, vec![8]); + assert_eq!( + plan.nodes.get(7).unwrap().opr.clone().unwrap(), + pb::As { alias: "c".try_into().ok() }.into() + ); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None } + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None } + ], + kind: 0 // inner join + } + .into() + ); + } + + #[test] + fn pattern_case3_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("b", Some("c"), false, false), + gen_sentence_x_out_y("d", Some("c"), false, false), + gen_sentence_x_out_y("a", Some("d"), false, false), + gen_sentence_x_out_y("b", Some("d"), false, false), + ]); + // Join (opr_id = 14) + // As(b), Out(), as(d), (opr_id = 13), + // Join (opr_id = 10) ( + // As(a), Out(), As(b), Out(), As(c), In(), (this has been reversed), As(d) (opr_id = 6) + // As(a), Out(), As(d), (opr_id = 9) + // ) + // ) + let plan = strategy.build_logical_plan().unwrap(); + println!("{:#?}", plan.nodes); + + assert_eq!( + plan.nodes.get(5).unwrap().opr.clone().unwrap(), + pb::EdgeExpand { + v_tag: None, + direction: 1, // check this has been reversed from 0 to 1 + params: Some(query_params()), + is_edge: false, + alias: None + } + .into() + ); + assert_eq!(plan.nodes.get(6).unwrap().children, vec![10]); + assert_eq!(plan.nodes.get(9).unwrap().children, vec![10]); + assert_eq!( + plan.nodes.get(10).unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("d".into()), property: None }, + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("d".into()), property: None }, + ], + kind: 0 // inner join + } + .into() + ); + assert_eq!(plan.nodes.get(10).unwrap().children, vec![14]); + assert_eq!(plan.nodes.get(13).unwrap().children, vec![14]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("b".into()), property: None }, + common_pb::Variable { tag: Some("d".into()), property: None } + ], + right_keys: vec![ + common_pb::Variable { tag: Some("b".into()), property: None }, + common_pb::Variable { tag: Some("d".into()), property: None } + ], + kind: 0 // inner join + } + .into() + ); + } + + #[test] + fn pattern_case3_no_reverse_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("b", Some("c"), false, false), + gen_sentence_x_out_y_all_columns("d", Some("c")), + gen_sentence_x_out_y("a", Some("d"), false, false), + gen_sentence_x_out_y("b", Some("d"), false, false), + ]); + // Join (opr_id = 14) + // As(b), Out(), as(d), (opr_id = 13), + // Join (opr_id = 10) ( + // As(a), Out(), As(b), Out(), As(c), (opr_id = 4) + // As(a), Out(), As(d), Out(), As(c) (opr_id = 9) + // ) + // ) + let plan = strategy.build_logical_plan().unwrap(); + println!("{:#?}", plan.nodes); + + assert_eq!(plan.nodes.get(4).unwrap().children, vec![10]); + assert_eq!(plan.nodes.get(9).unwrap().children, vec![10]); + assert_eq!( + plan.nodes.get(10).unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None }, + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None }, + ], + kind: 0 // inner join + } + .into() + ); + assert_eq!(plan.nodes.get(10).unwrap().children, vec![14]); + assert_eq!(plan.nodes.get(13).unwrap().children, vec![14]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("b".into()), property: None }, + common_pb::Variable { tag: Some("d".into()), property: None } + ], + right_keys: vec![ + common_pb::Variable { tag: Some("b".into()), property: None }, + common_pb::Variable { tag: Some("d".into()), property: None } + ], + kind: 0 // inner join + } + .into() + ); + } + + #[test] + fn pattern_case4_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("b", Some("a"), false, false), + ]); + // Join (id = 6) ( + // As(a), Out(), As(b) (id = 2), + // As(b), Out(), As(a) (id = 5), + // ) + let plan = strategy.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 7); + assert_eq!(plan.nodes.get(2).unwrap().children, vec![6]); + assert_eq!(plan.nodes.get(5).unwrap().children, vec![6]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("b".into()), property: None }, + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("b".into()), property: None }, + ], + kind: 0 // inner join + } + .into() + ); + } + + #[test] + fn pattern_case5_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("a", Some("b"), false, true), + ]); + // Join (id = 6) ( + // As(a), Out(), As(b) (id = 2), + // As(a), Out(), As(b) (id = 5), + // ) + let plan = strategy.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 7); + assert_eq!(plan.nodes.get(2).unwrap().children, vec![6]); + assert_eq!(plan.nodes.get(5).unwrap().children, vec![6]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("b".into()), property: None }, + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("b".into()), property: None }, + ], + kind: 5 // anti join + } + .into() + ); + } + + #[test] + fn pattern_case6_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false), + gen_sentence_x_out_y("b", Some("c"), false, false), + gen_sentence_x_out_y("c", Some("a"), false, false), + ]); + // Join (id = 8) ( + // As(a), Out(), As(b), Out(), As(c) (id = 4), + // As(c), Out(), As(a) (id = 7), + // ) + let plan = strategy.build_logical_plan().unwrap(); + assert_eq!(plan.nodes.len(), 9); + assert_eq!(plan.nodes.get(4).unwrap().children, vec![8]); + assert_eq!(plan.nodes.get(7).unwrap().children, vec![8]); + assert_eq!( + plan.nodes.last().unwrap().opr.clone().unwrap(), + pb::Join { + left_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None }, + ], + right_keys: vec![ + common_pb::Variable { tag: Some("a".into()), property: None }, + common_pb::Variable { tag: Some("c".into()), property: None }, + ], + kind: 0 // inner join + } + .into() + ); + } + + #[test] + fn pattern_disconnected_into_logical_plan() { + let strategy = NaiveStrategy::from(vec![ + gen_sentence_x_out_y("a", Some("b"), false, false).into(), + gen_sentence_x_out_y("b", Some("c"), false, false).into(), + gen_sentence_x_out_y("d", Some("e"), false, false).into(), + ]); + + let result = strategy.build_logical_plan(); + match result.err().unwrap() { + IrError::InvalidPattern(_) => {} + _ => panic!("should produce invalid pattern error"), + } + } +} diff --git a/research/query_service/ir/core/src/plan/physical.rs b/research/query_service/ir/core/src/plan/physical.rs new file mode 100644 index 000000000000..b247c2f178cc --- /dev/null +++ b/research/query_service/ir/core/src/plan/physical.rs @@ -0,0 +1,1289 @@ +// +//! Copyright 2020 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. +//! +//! The physical module will help package a ir logical plan into a pegasus physical plan. +//! Basically, it will wrap each ir operator into a corresponding pegasus operator (map, flatmap, etc) +//! and then represent the udf as a BinarySource (byte array) that is directly encoded from the +//! protobuf structure. +//! + +use ir_common::generated::algebra as pb; +use ir_common::generated::common as common_pb; +use pegasus_client::builder::*; +use pegasus_server::pb as server_pb; +use prost::Message; + +use crate::error::{IrError, IrResult}; +use crate::plan::logical::{LogicalPlan, NodeType}; +use crate::plan::meta::PlanMeta; + +/// A trait for building physical plan (pegasus) from the logical plan +pub trait AsPhysical { + /// To add pegasus's `JobBuilder` + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()>; + + /// To conduct necessary post processing before transforming into a physical plan. + fn post_process(&mut self, _builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + Ok(()) + } +} + +#[derive(PartialEq)] +enum SimpleOpr { + Source, + Map, + FilterMap, + Flatmap, + Filter, + SortBy, + Dedup, + GroupBy, + Fold, + Sink, +} + +fn simple_add_job_builder( + builder: &mut JobBuilder, ir_opr: &M, opr: SimpleOpr, +) -> IrResult<()> { + let bytes = ir_opr.encode_to_vec(); + let _ = match opr { + SimpleOpr::Source => builder.add_source(bytes), + SimpleOpr::Map => builder.map(bytes), + SimpleOpr::FilterMap => builder.filter_map(bytes), + SimpleOpr::Flatmap => builder.flat_map(bytes), + SimpleOpr::Filter => builder.filter(bytes), + SimpleOpr::SortBy => builder.sort_by(bytes), + SimpleOpr::Dedup => builder.dedup(bytes), + SimpleOpr::GroupBy => builder.group_by(pegasus_server::pb::AccumKind::Custom, bytes), + SimpleOpr::Fold => builder.fold_custom(pegasus_server::pb::AccumKind::Custom, bytes), + SimpleOpr::Sink => { + builder.sink(bytes); + builder + } + }; + Ok(()) +} + +fn update_query_params( + params: &mut pb::QueryParams, columns: Vec, is_all_columns: bool, +) -> pb::QueryParams { + let mut new_params = params.clone(); + params.columns.clear(); + params.is_all_columns = false; + params.predicate = None; + + new_params.tables.clear(); + new_params.columns = columns; + new_params.is_all_columns = is_all_columns; + + new_params +} + +impl AsPhysical for pb::Project { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + simple_add_job_builder(builder, &pb::logical_plan::Operator::from(self.clone()), SimpleOpr::Map) + } +} + +impl AsPhysical for pb::Select { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + simple_add_job_builder(builder, &pb::logical_plan::Operator::from(self.clone()), SimpleOpr::Filter) + } +} + +impl AsPhysical for pb::Scan { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + let mut scan = self.clone(); + scan.post_process(builder, plan_meta)?; + simple_add_job_builder(builder, &pb::logical_plan::Operator::from(scan), SimpleOpr::Source) + } + + fn post_process(&mut self, _builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(params) = &mut self.params { + if let Some(node_metas) = plan_meta.curr_node_metas() { + let columns = node_metas.get_columns(); + let is_all_columns = node_metas.is_all_columns(); + if !columns.is_empty() || is_all_columns { + params.columns = columns + .into_iter() + .map(|tag| common_pb::NameOrId::from(tag)) + .collect(); + params.is_all_columns = is_all_columns; + } + } + Ok(()) + } else { + Err(IrError::MissingData("Scan::params".to_string())) + } + } +} + +impl AsPhysical for pb::EdgeExpand { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + let mut xpd = self.clone(); + xpd.post_process(builder, plan_meta) + // simple_add_job_builder(builder, &pb::logical_plan::Operator::from(self.clone()), SimpleOpr::Flatmap) + } + + fn post_process(&mut self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + let mut is_adding_auxilia = false; + let mut auxilia = pb::Auxilia { params: None, alias: None }; + if let Some(params) = self.params.as_mut() { + if let Some(node_metas) = plan_meta.curr_node_metas() { + let columns = node_metas.get_columns(); + let is_all_columns = node_metas.is_all_columns(); + if !columns.is_empty() || is_all_columns { + if !self.is_edge { + // Vertex expansion + // Move everything to Auxilia + auxilia.params = Some(update_query_params( + params, + columns + .into_iter() + .map(|tag| common_pb::NameOrId::from(tag)) + .collect(), + is_all_columns, + )); + auxilia.alias = self.alias.clone(); + self.alias = None; + is_adding_auxilia = true; + } else { + params.columns = columns + .into_iter() + .map(|tag| common_pb::NameOrId::from(tag)) + .collect(); + params.is_all_columns = is_all_columns; + } + } + } + } else { + return Err(IrError::MissingData("EdgeExpand::params".to_string())); + } + simple_add_job_builder( + builder, + &pb::logical_plan::Operator::from(self.clone()), + SimpleOpr::Flatmap, + )?; + if is_adding_auxilia { + if plan_meta.is_partition() { + let key_pb = common_pb::NameOrIdKey { key: None }; + builder.repartition(key_pb.encode_to_vec()); + } + pb::logical_plan::Operator::from(auxilia).add_job_builder(builder, plan_meta)?; + } + + Ok(()) + } +} + +impl AsPhysical for pb::PathExpand { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(range) = &self.hop_range { + if range.upper <= range.lower || range.lower <= 0 || range.upper <= 0 { + Err(IrError::InvalidRange(range.lower, range.upper)) + } else { + if let Some(base) = &self.base { + let path_start = pb::PathStart { + start_tag: self.start_tag.clone(), + is_whole_path: self.is_whole_path, + }; + simple_add_job_builder( + builder, + &pb::logical_plan::Operator::from(path_start), + SimpleOpr::Map, + )?; + let is_partition = plan_meta.is_partition(); + for _ in 0..(range.lower - 1) { + if is_partition { + let key_pb = common_pb::NameOrIdKey { key: None }; + builder.repartition(key_pb.encode_to_vec()); + } + pb::logical_plan::Operator::from(base.clone()) + .add_job_builder(builder, plan_meta)?; + } + let times = range.upper - range.lower; + if times == 1 { + if is_partition { + let key_pb = common_pb::NameOrIdKey { key: None }; + builder.repartition(key_pb.encode_to_vec()); + } + pb::logical_plan::Operator::from(base.clone()) + .add_job_builder(builder, plan_meta)?; + } else if times > 1 { + builder.iterate_emit(times as u32, move |plan| { + if is_partition { + let key_pb = common_pb::NameOrIdKey { key: None }; + plan.repartition(key_pb.encode_to_vec()); + } + plan.flat_map(pb::logical_plan::Operator::from(base.clone()).encode_to_vec()); + }); + } + let path_end = pb::PathEnd { alias: self.alias.clone() }; + simple_add_job_builder( + builder, + &pb::logical_plan::Operator::from(path_end), + SimpleOpr::Map, + ) + } else { + Err(IrError::MissingData("PathExpand::base".to_string())) + } + } + } else { + Err(IrError::MissingData("PathExpand::hop_range".to_string())) + } + } +} + +impl AsPhysical for pb::GetV { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + let mut getv = self.clone(); + getv.post_process(builder, plan_meta) + // simple_add_job_builder(builder, &pb::logical_plan::Operator::from(getv), SimpleOpr::Map) + } + + fn post_process(&mut self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + let mut is_adding_auxilia = false; + let mut auxilia = pb::Auxilia { params: None, alias: None }; + if let Some(params) = self.params.as_mut() { + if let Some(node_metas) = plan_meta.curr_node_metas() { + let columns = node_metas.get_columns(); + let is_all_columns = node_metas.is_all_columns(); + if !columns.is_empty() || is_all_columns { + auxilia.params = Some(update_query_params( + params, + columns + .into_iter() + .map(|tag| common_pb::NameOrId::from(tag)) + .collect(), + is_all_columns, + )); + auxilia.alias = self.alias.clone(); + self.alias = None; + is_adding_auxilia = true; + } + } + } else { + return Err(IrError::MissingData("GetV::params".to_string())); + } + simple_add_job_builder(builder, &pb::logical_plan::Operator::from(self.clone()), SimpleOpr::Map)?; + if is_adding_auxilia { + if plan_meta.is_partition() { + let key_pb = common_pb::NameOrIdKey { key: None }; + builder.repartition(key_pb.encode_to_vec()); + } + pb::logical_plan::Operator::from(auxilia).add_job_builder(builder, plan_meta)?; + } + + Ok(()) + } +} + +impl AsPhysical for pb::As { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + // Transform to `Auxilia` internally. + let auxilia = pb::Auxilia { params: None, alias: self.alias.clone() }; + auxilia.add_job_builder(builder, plan_meta) + } +} + +impl AsPhysical for pb::Auxilia { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + simple_add_job_builder( + builder, + &pb::logical_plan::Operator::from(self.clone()), + SimpleOpr::FilterMap, + ) + } +} + +impl AsPhysical for pb::Limit { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + if let Some(range) = &self.range { + if range.upper <= range.lower || range.lower < 0 || range.upper <= 0 { + Err(IrError::InvalidRange(range.lower, range.upper)) + } else { + builder.limit((range.upper - 1) as u32); + Ok(()) + } + } else { + Err(IrError::MissingData("Limit::range".to_string())) + } + } +} + +impl AsPhysical for pb::OrderBy { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + let opr = pb::logical_plan::Operator::from(self.clone()); + if self.limit.is_none() { + simple_add_job_builder(builder, &opr, SimpleOpr::SortBy) + } else { + let range = self.limit.clone().unwrap(); + if range.upper <= range.lower || range.lower < 0 || range.upper <= 0 { + Err(IrError::InvalidRange(range.lower, range.upper)) + } else { + let bytes = opr.encode_to_vec(); + builder.sort_limit_by((range.upper - 1) as i64, bytes); + Ok(()) + } + } + } +} + +impl AsPhysical for pb::Dedup { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + simple_add_job_builder(builder, &pb::logical_plan::Operator::from(self.clone()), SimpleOpr::Dedup) + } +} + +impl AsPhysical for pb::GroupBy { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + let opr = pb::logical_plan::Operator::from(self.clone()); + if self.mappings.is_empty() { + simple_add_job_builder(builder, &opr, SimpleOpr::Fold) + } else { + simple_add_job_builder(builder, &opr, SimpleOpr::GroupBy) + } + } +} + +impl AsPhysical for pb::Sink { + fn add_job_builder(&self, builder: &mut JobBuilder, _plan_meta: &mut PlanMeta) -> IrResult<()> { + simple_add_job_builder(builder, &pb::logical_plan::Operator::from(self.clone()), SimpleOpr::Sink) + } +} + +impl AsPhysical for pb::logical_plan::Operator { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + use pb::logical_plan::operator::Opr::*; + if let Some(opr) = &self.opr { + match opr { + Project(project) => project.add_job_builder(builder, plan_meta), + Select(select) => select.add_job_builder(builder, plan_meta), + Vertex(getv) => getv.add_job_builder(builder, plan_meta), + Edge(edgexpd) => edgexpd.add_job_builder(builder, plan_meta), + Path(pathxpd) => pathxpd.add_job_builder(builder, plan_meta), + Scan(scan) => scan.add_job_builder(builder, plan_meta), + Limit(limit) => limit.add_job_builder(builder, plan_meta), + OrderBy(orderby) => orderby.add_job_builder(builder, plan_meta), + Auxilia(auxilia) => auxilia.add_job_builder(builder, plan_meta), + As(as_opr) => as_opr.add_job_builder(builder, plan_meta), + Dedup(dedup) => dedup.add_job_builder(builder, plan_meta), + GroupBy(groupby) => groupby.add_job_builder(builder, plan_meta), + Sink(sink) => sink.add_job_builder(builder, plan_meta), + Union(_) => Ok(()), + _ => Err(IrError::Unsupported(format!("the operator {:?}", self))), + } + } else { + Err(IrError::MissingData("logical_plan::Operator::opr".to_string())) + } + } +} + +impl AsPhysical for NodeType { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + plan_meta.set_curr_node(self.borrow().id); + self.borrow() + .opr + .add_job_builder(builder, plan_meta) + } +} + +impl AsPhysical for LogicalPlan { + fn add_job_builder(&self, builder: &mut JobBuilder, plan_meta: &mut PlanMeta) -> IrResult<()> { + use pb::join::JoinKind; + use pb::logical_plan::operator::Opr::*; + let mut _prev_node_opt: Option = None; + let mut curr_node_opt = self.root(); + debug!("plan: {:#?}", self); + debug!("is_partition: {:?}", self.meta.is_partition()); + while curr_node_opt.is_some() { + let curr_node = curr_node_opt.as_ref().unwrap(); + if let Some(Apply(apply_opr)) = curr_node.borrow().opr.opr.as_ref() { + let mut sub_bldr = JobBuilder::default(); + if let Some(subplan) = self.extract_subplan(curr_node.clone()) { + subplan.add_job_builder(&mut sub_bldr, plan_meta)?; + let plan = sub_bldr.take_plan(); + builder.apply_join( + move |p| *p = plan.clone(), + pb::logical_plan::Operator::from(apply_opr.clone()).encode_to_vec(), + ); + } else { + return Err(IrError::MissingData("Apply::subplan".to_string())); + } + } else { + if let Some(Edge(edgexpd)) = curr_node.borrow().opr.opr.as_ref() { + let key_pb = common_pb::NameOrIdKey { key: edgexpd.v_tag.clone() }; + if plan_meta.is_partition() { + builder.repartition(key_pb.encode_to_vec()); + } + } + curr_node.add_job_builder(builder, plan_meta)?; + } + + _prev_node_opt = curr_node_opt.clone(); + + if curr_node.borrow().children.is_empty() { + break; + } else if curr_node.borrow().children.len() == 1 { + let next_node_id = curr_node.borrow().get_first_child().unwrap(); + curr_node_opt = self.get_node(next_node_id); + } else if curr_node.borrow().children.len() >= 2 { + let (merge_node_opt, subplans) = self.get_branch_plans(curr_node.clone()); + let mut plans: Vec = vec![]; + for subplan in subplans { + let mut sub_bldr = JobBuilder::new(builder.conf.clone()); + subplan.add_job_builder(&mut sub_bldr, plan_meta)?; + plans.push(sub_bldr.take_plan()); + } + + if let Some(merge_node) = merge_node_opt.clone() { + match &merge_node.borrow().opr.opr { + Some(Union(_)) => { + builder.merge(plans); + } + Some(Join(join_opr)) => { + if curr_node.borrow().children.len() > 2 { + // For now we only support joining two branches + return Err(IrError::Unsupported( + "joining more than two branches".to_string(), + )); + } + assert_eq!(plans.len(), 2); + let left_plan = plans.get(0).unwrap().clone(); + let right_plan = plans.get(1).unwrap().clone(); + + let join_kind = + unsafe { std::mem::transmute::(join_opr.kind) }; + let pegasus_join_kind = match join_kind { + JoinKind::Inner => server_pb::join::JoinKind::Inner, + JoinKind::LeftOuter => server_pb::join::JoinKind::LeftOuter, + JoinKind::RightOuter => server_pb::join::JoinKind::RightOuter, + JoinKind::FullOuter => server_pb::join::JoinKind::FullOuter, + JoinKind::Semi => server_pb::join::JoinKind::Semi, + JoinKind::Anti => server_pb::join::JoinKind::Anti, + JoinKind::Times => server_pb::join::JoinKind::Times, + }; + let mut join_bytes = vec![]; + pb::logical_plan::Operator::from(join_opr.clone()).encode(&mut join_bytes)?; + + builder.join(pegasus_join_kind, left_plan, right_plan, join_bytes); + } + None => return Err(IrError::MissingData("Union/Join::merge_node".to_string())), + _ => { + return Err(IrError::Unsupported( + "operators other than `Union` and `Join`".to_string(), + )) + } + } + } + curr_node_opt = merge_node_opt; + + if let Some(curr_node_clone) = curr_node_opt.clone() { + if curr_node_clone.borrow().children.len() <= 1 { + let next_id_opt = curr_node_clone.borrow().get_first_child(); + _prev_node_opt = curr_node_opt.clone(); + // the current node has been processed in this round, should skip to the next node + curr_node_opt = next_id_opt.and_then(|id| self.get_node(id)); + } + } + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + + use ir_common::expr_parse::str_to_expr_pb; + use ir_common::generated::algebra as pb; + use ir_common::generated::algebra::project::ExprAlias; + use ir_common::generated::common as common_pb; + + use super::*; + use crate::plan::logical::Node; + + #[allow(dead_code)] + fn query_params( + tables: Vec, columns: Vec, + ) -> pb::QueryParams { + pb::QueryParams { + tables, + columns, + is_all_columns: false, + limit: None, + predicate: None, + extra: HashMap::new(), + } + } + + #[allow(dead_code)] + fn build_scan(columns: Vec) -> pb::Scan { + pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], columns)), + idx_predicate: None, + } + } + + #[allow(dead_code)] + fn build_edgexpd( + is_edge: bool, columns: Vec, alias: Option, + ) -> pb::EdgeExpand { + pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], columns)), + is_edge, + alias, + } + } + + #[allow(dead_code)] + fn build_getv(alias: Option) -> pb::GetV { + pb::GetV { tag: None, opt: 1, params: Some(query_params(vec![], vec![])), alias } + } + + #[allow(dead_code)] + fn build_select(expr: &str) -> pb::Select { + pb::Select { predicate: str_to_expr_pb(expr.to_string()).ok() } + } + + #[allow(dead_code)] + fn build_project(expr: &str) -> pb::Project { + pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb(expr.to_string()).ok(), + alias: None, + }], + is_append: false, + } + } + + #[test] + fn post_process_edgexpd() { + // g.V().outE() + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + plan.append_operator_as_node(build_edgexpd(true, vec![], None).into(), vec![0]) + .unwrap(); + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(true, vec![], None)).encode_to_vec()); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan_meta = plan_meta.with_partition(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(true, vec![], None)).encode_to_vec()); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn post_process_edgexpd_columns_no_auxilia() { + // g.V().outE().has("creationDate", 20220101) + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + plan.append_operator_as_node(build_edgexpd(true, vec![], None).into(), vec![0]) + .unwrap(); + plan.append_operator_as_node(build_select("@.creationDate == 20220101").into(), vec![1]) + .unwrap(); + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan_meta = plan_meta.with_partition(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.flat_map( + pb::logical_plan::Operator::from(build_edgexpd(true, vec!["creationDate".into()], None)) + .encode_to_vec(), + ); + expected_builder.filter( + pb::logical_plan::Operator::from(build_select("@.creationDate == 20220101")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn post_process_edgexpd_columns_auxilia_shuffle() { + // g.V().out().has("birthday", 20220101) + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + plan.append_operator_as_node(build_edgexpd(false, vec![], None).into(), vec![0]) + .unwrap(); + plan.append_operator_as_node(build_select("@.birthday == 20220101").into(), vec![1]) + .unwrap(); + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(false, vec![], None)).encode_to_vec()); + expected_builder.filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["birthday".into()])), + alias: None, + }) + .encode_to_vec(), + ); + expected_builder.filter( + pb::logical_plan::Operator::from(build_select("@.birthday == 20220101")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan_meta = plan_meta.with_partition(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(false, vec![], None)).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["birthday".into()])), + alias: None, + }) + .encode_to_vec(), + ); + expected_builder.filter( + pb::logical_plan::Operator::from(build_select("@.birthday == 20220101")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn post_process_edgexpd_tag_auxilia_shuffle() { + // g.V().out().as('a').select('a').by(valueMap("name", "id", "age") + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + plan.append_operator_as_node(build_edgexpd(false, vec![], Some("a".into())).into(), vec![0]) + .unwrap(); + plan.append_operator_as_node(build_project("{@a.name, @a.id, @a.age}").into(), vec![1]) + .unwrap(); + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(false, vec![], None)).encode_to_vec()); + expected_builder.filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["age".into(), "id".into(), "name".into()])), + alias: Some("a".into()), + }) + .encode_to_vec(), + ); + expected_builder.map( + pb::logical_plan::Operator::from(build_project("{@a.name, @a.id, @a.age}")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan_meta = plan_meta.with_partition(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(false, vec![], None)).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["age".into(), "id".into(), "name".into()])), + alias: Some("a".into()), + }) + .encode_to_vec(), + ); + expected_builder.map( + pb::logical_plan::Operator::from(build_project("{@a.name, @a.id, @a.age}")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn post_process_edgexpd_tag_no_auxilia() { + // g.V().out().as('a').select('a') + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + plan.append_operator_as_node(build_edgexpd(false, vec![], Some("a".into())).into(), vec![0]) + .unwrap(); + plan.append_operator_as_node(build_project("@a").into(), vec![1]) + .unwrap(); + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan_meta = plan_meta.with_partition(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.flat_map( + pb::logical_plan::Operator::from(build_edgexpd(false, vec![], Some("a".into()))) + .encode_to_vec(), + ); + expected_builder.map(pb::logical_plan::Operator::from(build_project("@a")).encode_to_vec()); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn post_process_scan() { + let mut plan = LogicalPlan::default(); + // g.V().hasLabel("person").has("age", 27).valueMap("age", "name", "id") + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + // .hasLabel("person") + plan.append_operator_as_node(build_select("@.~label == \"person\"").into(), vec![0]) + .unwrap(); + // .has("age", 27) + plan.append_operator_as_node(build_select("@.age == 27").into(), vec![1]) + .unwrap(); + + // .valueMap("age", "name", "id") + plan.append_operator_as_node(build_project("{@.name, @.age, @.id}").into(), vec![2]) + .unwrap(); + + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source( + pb::logical_plan::Operator::from(build_scan(vec!["age".into(), "id".into(), "name".into()])) + .encode_to_vec(), + ); + + expected_builder.filter( + pb::logical_plan::Operator::from(build_select("@.~label == \"person\"")).encode_to_vec(), + ); + expected_builder + .filter(pb::logical_plan::Operator::from(build_select("@.age == 27")).encode_to_vec()); + expected_builder + .map(pb::logical_plan::Operator::from(build_project("{@.name, @.age, @.id}")).encode_to_vec()); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn post_process_getv_tag_auxilia_shuffle() { + // g.V().outE().inV().as('a').select('a').by(valueMap("name", "id", "age") + let mut plan = LogicalPlan::default(); + plan.append_operator_as_node(build_scan(vec![]).into(), vec![]) + .unwrap(); + plan.append_operator_as_node(build_edgexpd(true, vec![], None).into(), vec![0]) + .unwrap(); + plan.append_operator_as_node(build_getv(Some("a".into())).into(), vec![1]) + .unwrap(); + plan.append_operator_as_node(build_project("{@a.name, @a.id, @a.age}").into(), vec![2]) + .unwrap(); + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(true, vec![], None)).encode_to_vec()); + expected_builder.map(pb::logical_plan::Operator::from(build_getv(None)).encode_to_vec()); + expected_builder.filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["age".into(), "id".into(), "name".into()])), + alias: Some("a".into()), + }) + .encode_to_vec(), + ); + expected_builder.map( + pb::logical_plan::Operator::from(build_project("{@a.name, @a.id, @a.age}")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + + let mut job_builder = JobBuilder::default(); + let mut plan_meta = plan.meta.clone(); + plan_meta = plan_meta.with_partition(); + plan.add_job_builder(&mut job_builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(pb::logical_plan::Operator::from(build_scan(vec![])).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder + .flat_map(pb::logical_plan::Operator::from(build_edgexpd(true, vec![], None)).encode_to_vec()); + expected_builder.map(pb::logical_plan::Operator::from(build_getv(None)).encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["age".into(), "id".into(), "name".into()])), + alias: Some("a".into()), + }) + .encode_to_vec(), + ); + expected_builder.map( + pb::logical_plan::Operator::from(build_project("{@a.name, @a.id, @a.age}")).encode_to_vec(), + ); + expected_builder.sink(vec![]); + + assert_eq!(job_builder, expected_builder); + } + + #[test] + fn poc_plan_as_physical() { + // g.V().hasLabel("person").has("id", 10).out("knows").limit(10) + let source_opr = pb::logical_plan::Operator::from(pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec!["person".into()], vec![])), + idx_predicate: None, + }); + let select_opr = pb::logical_plan::Operator::from(pb::Select { + predicate: Some(str_to_expr_pb("@.id == 10".to_string()).unwrap()), + }); + let expand_opr = pb::logical_plan::Operator::from(pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec!["knows".into()], vec![])), + is_edge: false, + alias: None, + }); + let limit_opr = + pb::logical_plan::Operator::from(pb::Limit { range: Some(pb::Range { lower: 10, upper: 11 }) }); + let source_opr_bytes = source_opr.encode_to_vec(); + let select_opr_bytes = select_opr.encode_to_vec(); + let expand_opr_bytes = expand_opr.encode_to_vec(); + + let mut logical_plan = LogicalPlan::with_root(Node::new(0, source_opr)); + logical_plan + .append_operator_as_node(select_opr.clone(), vec![0]) + .unwrap(); // node 1 + logical_plan + .append_operator_as_node(expand_opr.clone(), vec![1]) + .unwrap(); // node 2 + logical_plan + .append_operator_as_node(limit_opr.clone(), vec![2]) + .unwrap(); // node 3 + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default(); + let _ = logical_plan.add_job_builder(&mut builder, &mut plan_meta); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr_bytes); + expected_builder.filter(select_opr_bytes); + expected_builder.flat_map(expand_opr_bytes); + expected_builder.limit(10); + expected_builder.sink(vec![]); + + assert_eq!(builder, expected_builder); + } + + #[test] + fn project_as_physical() { + let source_opr = pb::logical_plan::Operator::from(pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec!["person".into()], vec![])), + idx_predicate: None, + }); + + let project_opr = pb::logical_plan::Operator::from(pb::Project { + mappings: vec![ + ExprAlias { + expr: Some(str_to_expr_pb("10 * (@.class - 10)".to_string()).unwrap()), + alias: None, + }, + ExprAlias { expr: Some(str_to_expr_pb("@.age - 1".to_string()).unwrap()), alias: None }, + ], + is_append: false, + }); + + let mut logical_plan = LogicalPlan::with_root(Node::new(0, source_opr.clone())); + logical_plan + .append_operator_as_node(project_opr.clone(), vec![0]) + .unwrap(); // node 1 + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default(); + logical_plan + .add_job_builder(&mut builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr.encode_to_vec()); + expected_builder.map(project_opr.encode_to_vec()); + assert_eq!(builder, expected_builder); + } + + #[test] + fn path_expand_as_physical() { + let source_opr = pb::logical_plan::Operator::from(pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec!["person".into()], vec![])), + idx_predicate: None, + }); + + let edge_expand = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec!["knows".into()], vec![])), + is_edge: false, + alias: None, + }; + + let expand_opr = pb::logical_plan::Operator::from(edge_expand.clone()); + let path_start_opr = + pb::logical_plan::Operator::from(pb::PathStart { start_tag: None, is_whole_path: false }); + let path_opr = pb::logical_plan::Operator::from(pb::PathExpand { + base: Some(edge_expand.clone()), + start_tag: None, + is_whole_path: false, + alias: None, + hop_range: Some(pb::Range { lower: 1, upper: 4 }), + }); + let path_end_opr = pb::logical_plan::Operator::from(pb::PathEnd { alias: None }); + + let mut logical_plan = LogicalPlan::with_root(Node::new(0, source_opr.clone())); + logical_plan + .append_operator_as_node(path_opr.clone(), vec![0]) + .unwrap(); // node 1 + + // Case without partition + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default(); + logical_plan + .add_job_builder(&mut builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr.encode_to_vec()); + expected_builder.map(path_start_opr.encode_to_vec()); + expected_builder.iterate_emit(3, |plan| { + plan.flat_map(expand_opr.clone().encode_to_vec()); + }); + expected_builder.map(path_end_opr.encode_to_vec()); + + assert_eq!(builder, expected_builder); + + // Case with partition + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default(); + plan_meta = plan_meta.with_partition(); + logical_plan + .add_job_builder(&mut builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr.encode_to_vec()); + expected_builder.map(path_start_opr.encode_to_vec()); + expected_builder.iterate_emit(3, |plan| { + plan.repartition(vec![]) + .flat_map(expand_opr.clone().encode_to_vec()); + }); + expected_builder.map(path_end_opr.encode_to_vec()); + + assert_eq!(builder, expected_builder); + } + + #[test] + fn path_expand_exactly_as_physical() { + let source_opr = pb::logical_plan::Operator::from(pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec!["person".into()], vec![])), + idx_predicate: None, + }); + + let edge_expand = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec!["knows".into()], vec![])), + is_edge: false, + alias: None, + }; + + let expand_opr = pb::logical_plan::Operator::from(edge_expand.clone()); + let path_start_opr = + pb::logical_plan::Operator::from(pb::PathStart { start_tag: None, is_whole_path: false }); + let path_opr = pb::logical_plan::Operator::from(pb::PathExpand { + base: Some(edge_expand.clone()), + start_tag: None, + is_whole_path: false, + alias: None, + hop_range: Some(pb::Range { lower: 3, upper: 4 }), + }); + let path_end_opr = pb::logical_plan::Operator::from(pb::PathEnd { alias: None }); + + let mut logical_plan = LogicalPlan::with_root(Node::new(0, source_opr.clone())); + logical_plan + .append_operator_as_node(path_opr.clone(), vec![0]) + .unwrap(); // node 1 + // Case with partition + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default().with_partition(); + logical_plan + .add_job_builder(&mut builder, &mut plan_meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr.encode_to_vec()); + expected_builder.map(path_start_opr.encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.flat_map(expand_opr.clone().encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.flat_map(expand_opr.clone().encode_to_vec()); + expected_builder.repartition(vec![]); + expected_builder.flat_map(expand_opr.clone().encode_to_vec()); + expected_builder.map(path_end_opr.encode_to_vec()); + + assert_eq!(builder, expected_builder); + } + + #[test] + fn orderby_as_physical() { + let source_opr = pb::logical_plan::Operator::from(pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }); + + let orderby_opr = pb::logical_plan::Operator::from(pb::OrderBy { pairs: vec![], limit: None }); + + let topby_opr = pb::logical_plan::Operator::from(pb::OrderBy { + pairs: vec![], + limit: Some(pb::Range { lower: 10, upper: 11 }), + }); + + let source_opr_bytes = source_opr.encode_to_vec(); + let orderby_opr_bytes = orderby_opr.encode_to_vec(); + let topby_opr_bytes = topby_opr.encode_to_vec(); + + let mut logical_plan = LogicalPlan::with_root(Node::new(0, source_opr)); + logical_plan + .append_operator_as_node(orderby_opr.clone(), vec![0]) + .unwrap(); // node 1 + logical_plan + .append_operator_as_node(topby_opr.clone(), vec![1]) + .unwrap(); // node 2 + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default(); + let _ = logical_plan.add_job_builder(&mut builder, &mut plan_meta); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr_bytes); + expected_builder.sort_by(orderby_opr_bytes); + expected_builder.sort_limit_by(10, topby_opr_bytes); + + assert_eq!(builder, expected_builder); + } + + #[test] + fn apply_as_physical_case1() { + let mut plan = LogicalPlan::default(); + // g.V().as("v").where(out().as("o").has("lang", "java")).select("v").values("name") + plan.meta = plan.meta.with_partition(); + + // g.V("person") + let scan: pb::logical_plan::Operator = pb::Scan { + scan_opt: 0, + alias: Some("v".into()), + params: Some(query_params(vec![], vec!["name".into()])), + idx_predicate: None, + } + .into(); + + let opr_id = plan + .append_operator_as_node(scan.clone(), vec![]) + .unwrap(); + + // .out().as("o") + let mut expand = pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: false, + alias: Some("o".into()), + }; + + let root_id = plan + .append_operator_as_node(expand.clone().into(), vec![]) + .unwrap(); + + // .has("lang", "Java") + let select: pb::logical_plan::Operator = + pb::Select { predicate: str_to_expr_pb("@.lang == \"Java\"".to_string()).ok() }.into(); + plan.append_operator_as_node(select.clone(), vec![root_id]) + .unwrap(); + + let apply: pb::logical_plan::Operator = + pb::Apply { join_kind: 4, tags: vec![], subtask: root_id as i32, alias: None }.into(); + let opr_id = plan + .append_operator_as_node(apply.clone(), vec![opr_id]) + .unwrap(); + + let project: pb::logical_plan::Operator = pb::Project { + mappings: vec![pb::project::ExprAlias { + expr: str_to_expr_pb("@v.name".to_string()).ok(), + alias: None, + }], + is_append: true, + } + .into(); + plan.append_operator_as_node(project.clone(), vec![opr_id]) + .unwrap(); + + let mut builder = JobBuilder::default(); + let mut meta = plan.meta.clone(); + plan.add_job_builder(&mut builder, &mut meta) + .unwrap(); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(scan.encode_to_vec()); + expand.alias = None; + expected_builder.apply_join( + |plan| { + plan.repartition(vec![]) + .flat_map(pb::logical_plan::Operator::from(expand.clone()).encode_to_vec()) + .repartition(vec![]) + .filter_map( + pb::logical_plan::Operator::from(pb::Auxilia { + params: Some(query_params(vec![], vec!["lang".into()])), + alias: Some("o".into()), + }) + .encode_to_vec(), + ) + .filter(select.encode_to_vec()); + }, + apply.encode_to_vec(), + ); + expected_builder.map(project.encode_to_vec()); + expected_builder.sink(vec![]); + + assert_eq!(expected_builder, builder); + } + + #[test] + fn join_plan_as_physical() { + let source_opr = pb::logical_plan::Operator::from(pb::Scan { + scan_opt: 0, + alias: None, + params: Some(query_params(vec![], vec![])), + idx_predicate: None, + }); + let expand_opr = pb::logical_plan::Operator::from(pb::EdgeExpand { + v_tag: None, + direction: 0, + params: Some(query_params(vec![], vec![])), + is_edge: false, + alias: None, + }); + let join_opr = + pb::logical_plan::Operator::from(pb::Join { left_keys: vec![], right_keys: vec![], kind: 0 }); + let limit_opr = + pb::logical_plan::Operator::from(pb::Limit { range: Some(pb::Range { lower: 10, upper: 11 }) }); + + let source_opr_bytes = source_opr.encode_to_vec(); + let expand_opr_bytes = expand_opr.encode_to_vec(); + let join_opr_bytes = join_opr.encode_to_vec(); + + let mut logical_plan = LogicalPlan::with_root(Node::new(0, source_opr)); + logical_plan + .append_operator_as_node(expand_opr.clone(), vec![0]) + .unwrap(); // node 1 + logical_plan + .append_operator_as_node(expand_opr.clone(), vec![0]) + .unwrap(); // node 2 + logical_plan + .append_operator_as_node(expand_opr.clone(), vec![2]) + .unwrap(); // node 3 + logical_plan + .append_operator_as_node(join_opr.clone(), vec![1, 3]) + .unwrap(); // node 4 + logical_plan + .append_operator_as_node(pb::logical_plan::Operator::from(limit_opr.clone()), vec![4]) + .unwrap(); // node 5 + let mut builder = JobBuilder::default(); + let mut plan_meta = PlanMeta::default(); + let _ = logical_plan.add_job_builder(&mut builder, &mut plan_meta); + + let mut expected_builder = JobBuilder::default(); + expected_builder.add_source(source_opr_bytes); + let mut left_plan = Plan::default(); + let mut right_plan = Plan::default(); + left_plan.flat_map(expand_opr_bytes.clone()); + right_plan.flat_map(expand_opr_bytes.clone()); + right_plan.flat_map(expand_opr_bytes); + expected_builder.join(server_pb::join::JoinKind::Inner, left_plan, right_plan, join_opr_bytes); + expected_builder.limit(10); + + assert_eq!(builder, expected_builder); + } +} diff --git a/research/query_service/ir/graph_proxy/Cargo.toml b/research/query_service/ir/graph_proxy/Cargo.toml new file mode 100644 index 000000000000..d3f654d2e48a --- /dev/null +++ b/research/query_service/ir/graph_proxy/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "graph_proxy" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +log = "0.4" +lazy_static = "1.3.0" +graph_store = {path = "../../../graph_store"} +maxgraph-store = {path = "../../../../interactive_engine/executor/store"} +dyn_type = { path = "../../../dyn_type" } +ir_common = {path = "../common"} +pegasus = { path = "../../../engine/pegasus/pegasus" } +pegasus_common = { path = "../../../engine/pegasus/common" } +pegasus_network = { path = "../../../engine/pegasus/network" } +pegasus_server = { path = "../../../engine/pegasus/server-v0" } +runtime = { path="../runtime" } + +[features] +default = [] +proto_inplace = ["ir_common/proto_inplace"] diff --git a/research/query_service/ir/graph_proxy/src/exp_store/graph_partition.rs b/research/query_service/ir/graph_proxy/src/exp_store/graph_partition.rs new file mode 100644 index 000000000000..64ed36daa00d --- /dev/null +++ b/research/query_service/ir/graph_proxy/src/exp_store/graph_partition.rs @@ -0,0 +1,48 @@ +// +//! Copyright 2021 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use pegasus::api::function::FnResult; +use runtime::graph::partitioner::Partitioner; +use runtime::graph::ID; + +/// A simple partition utility that one server contains a single graph partition +pub struct SimplePartition { + pub num_servers: usize, +} + +impl Partitioner for SimplePartition { + fn get_partition(&self, id: &ID, workers: usize) -> FnResult { + let id_usize = *id as usize; + let magic_num = id_usize / self.num_servers; + // The partitioning logics is as follows: + // 1. `R = id - magic_num * num_servers = id % num_servers` routes a given id + // to the machine R that holds its data. + // 2. `R * workers` shifts the worker's id in the machine R. + // 3. `magic_num % workers` then picks up one of the workers in the machine R + // to do the computation. + Ok(((id_usize - magic_num * self.num_servers) * workers + magic_num % workers) as u64) + } + + fn get_worker_partitions(&self, job_workers: usize, worker_id: u32) -> FnResult>> { + // In graph that one server contains a single graph partition, + // we assign the first worker on current server to process (scan) the partition, + // and we assume the partition id is identity to the server id + if worker_id as usize % job_workers == 0 { + Ok(Some(vec![worker_id as u64 / job_workers as u64])) + } else { + Ok(None) + } + } +} diff --git a/research/query_service/ir/graph_proxy/src/exp_store/graph_query.rs b/research/query_service/ir/graph_proxy/src/exp_store/graph_query.rs new file mode 100644 index 000000000000..3709ecde3a1b --- /dev/null +++ b/research/query_service/ir/graph_proxy/src/exp_store/graph_query.rs @@ -0,0 +1,560 @@ +// +//! Copyright 2021 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::collections::HashMap; +use std::fmt; +use std::path::Path; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::sync::Arc; + +use dyn_type::{object, BorrowObject, Object}; +use graph_store::common::LabelId; +use graph_store::config::{JsonConf, DIR_GRAPH_SCHEMA, FILE_SCHEMA}; +use graph_store::ldbc::LDBCVertexParser; +use graph_store::prelude::{ + DefaultId, EdgeId, GlobalStoreTrait, GlobalStoreUpdate, GraphDBConfig, InternalId, LDBCGraphSchema, + LargeGraphDB, LocalEdge, LocalVertex, MutableGraphDB, Row, INVALID_LABEL_ID, +}; +use ir_common::{KeyId, NameOrId}; +use pegasus::api::function::FnResult; +use pegasus::configure_with_default; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; +use runtime::graph::element::{Edge, Vertex}; +use runtime::graph::property::{DefaultDetails, Details, DynDetails}; +use runtime::graph::{register_graph, Direction, GraphProxy, QueryParams, Statement, ID}; + +use crate::from_fn; +use crate::{filter_limit, limit_n}; + +lazy_static! { + pub static ref DATA_PATH: String = configure_with_default!(String, "DATA_PATH", "".to_string()); + pub static ref PARTITION_ID: usize = configure_with_default!(usize, "PARTITION_ID", 0); + pub static ref GRAPH: LargeGraphDB = _init_graph(); + static ref GRAPH_PROXY: Arc = initialize(); +} + +pub struct DemoGraph { + store: &'static LargeGraphDB, +} + +fn initialize() -> Arc { + lazy_static::initialize(&GRAPH); + Arc::new(DemoGraph { store: &GRAPH }) +} + +fn _init_graph() -> LargeGraphDB { + if DATA_PATH.is_empty() { + info!("Create and use the modern graph for demo."); + _init_modern_graph() + } else { + info!("Read the graph data from {:?} for demo.", *DATA_PATH); + GraphDBConfig::default() + .root_dir(&(*DATA_PATH)) + .partition(*PARTITION_ID) + .schema_file( + &(DATA_PATH.as_ref() as &Path) + .join(DIR_GRAPH_SCHEMA) + .join(FILE_SCHEMA), + ) + .open() + .expect("Open graph error") + } +} + +fn _init_modern_graph() -> LargeGraphDB { + let mut mut_graph: MutableGraphDB = GraphDBConfig::default().new(); + + let v1: DefaultId = LDBCVertexParser::to_global_id(1, 0); + let v2: DefaultId = LDBCVertexParser::to_global_id(2, 0); + let v3: DefaultId = LDBCVertexParser::to_global_id(3, 1); + let v4: DefaultId = LDBCVertexParser::to_global_id(4, 0); + let v5: DefaultId = LDBCVertexParser::to_global_id(5, 1); + let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); + + mut_graph.add_vertex(v1, [0, INVALID_LABEL_ID]); + mut_graph.add_vertex(v2, [0, INVALID_LABEL_ID]); + mut_graph.add_vertex(v3, [1, INVALID_LABEL_ID]); + mut_graph.add_vertex(v4, [0, INVALID_LABEL_ID]); + mut_graph.add_vertex(v5, [1, INVALID_LABEL_ID]); + mut_graph.add_vertex(v6, [0, INVALID_LABEL_ID]); + + let prop7 = Row::from(vec![object!(0.5)]); + let prop8 = Row::from(vec![object!(0.4)]); + let prop9 = Row::from(vec![object!(1.0)]); + let prop10 = Row::from(vec![object!(0.4)]); + let prop11 = Row::from(vec![object!(1.0)]); + let prop12 = Row::from(vec![object!(0.2)]); + + mut_graph + .add_edge_with_properties(v1, v2, 0, prop7) + .unwrap(); + mut_graph + .add_edge_with_properties(v1, v3, 1, prop8) + .unwrap(); + mut_graph + .add_edge_with_properties(v1, v4, 0, prop9) + .unwrap(); + mut_graph + .add_edge_with_properties(v4, v3, 1, prop10) + .unwrap(); + mut_graph + .add_edge_with_properties(v4, v5, 1, prop11) + .unwrap(); + mut_graph + .add_edge_with_properties(v6, v3, 1, prop12) + .unwrap(); + + let prop1 = Row::from(vec![object!(1), object!("marko"), object!(29)]); + let prop2 = Row::from(vec![object!(2), object!("vadas"), object!(27)]); + let prop3 = Row::from(vec![object!(3), object!("lop"), object!("java")]); + let prop4 = Row::from(vec![object!(4), object!("josh"), object!(32)]); + let prop5 = Row::from(vec![object!(5), object!("ripple"), object!("java")]); + let prop6 = Row::from(vec![object!(6), object!("peter"), object!(35)]); + + mut_graph + .add_or_update_vertex_properties(v1, prop1) + .unwrap(); + mut_graph + .add_or_update_vertex_properties(v2, prop2) + .unwrap(); + mut_graph + .add_or_update_vertex_properties(v3, prop3) + .unwrap(); + mut_graph + .add_or_update_vertex_properties(v4, prop4) + .unwrap(); + mut_graph + .add_or_update_vertex_properties(v5, prop5) + .unwrap(); + mut_graph + .add_or_update_vertex_properties(v6, prop6) + .unwrap(); + + let modern_graph_schema = r#" + { + "vertex_type_map": { + "person": 0, + "software": 1 + }, + "edge_type_map": { + "knows": 0, + "created": 1 + }, + "vertex_prop": { + "person": [ + [ + "id", + "ID" + ], + [ + "name", + "String" + ], + [ + "age", + "Integer" + ] + ], + "software": [ + [ + "id", + "ID" + ], + [ + "name", + "String" + ], + [ + "lang", + "String" + ] + ] + }, + "edge_prop": { + "knows": [ + [ + "start_id", + "ID" + ], + [ + "end_id", + "ID" + ], + [ + "weight", + "Double" + ] + ], + "created": [ + [ + "start_id", + "ID" + ], + [ + "end_id", + "ID" + ], + [ + "weight", + "Double" + ] + ] + } + } + "#; + let schema = LDBCGraphSchema::from_json(modern_graph_schema.to_string()).expect("Parse schema error!"); + + mut_graph.into_graph(schema) +} + +impl GraphProxy for DemoGraph { + fn scan_vertex(&self, params: &QueryParams) -> FnResult + Send>> { + // DemoGraph contains a single graph partition on each server, + // therefore, there's no need to use the specific partition id for query. + // Besides, we guarantee only one worker (on each server) is going to scan (with params.partitions.is_some()) + if params.partitions.is_some() { + let label_ids = encode_storage_vertex_label(¶ms.labels); + let store = self.store; + let props = params.columns.clone(); + let result = self + .store + .get_all_vertices(label_ids.as_ref()) + .map(move |v| to_runtime_vertex(v, store, props.clone())); + + Ok(filter_limit!(result, params.filter, params.limit)) + } else { + Ok(Box::new(std::iter::empty())) + } + } + + fn scan_edge(&self, params: &QueryParams) -> FnResult + Send>> { + if params.partitions.is_some() { + let label_ids = encode_storage_edge_label(¶ms.labels); + let store = self.store; + let result = self + .store + .get_all_edges(label_ids.as_ref()) + .map(move |e| to_runtime_edge(e, store)); + + Ok(filter_limit!(result, params.filter, params.limit)) + } else { + Ok(Box::new(std::iter::empty())) + } + } + + fn get_vertex( + &self, ids: &[ID], params: &QueryParams, + ) -> FnResult + Send>> { + let mut result = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(local_vertex) = self.store.get_vertex(*id as DefaultId) { + let v = to_runtime_vertex(local_vertex, self.store, params.columns.clone()); + result.push(v); + } + } + Ok(filter_limit!(result.into_iter(), params.filter, params.limit)) + } + + fn get_edge( + &self, ids: &[ID], params: &QueryParams, + ) -> FnResult + Send>> { + let mut result = Vec::with_capacity(ids.len()); + for id in ids { + let eid = encode_store_e_id(id); + if let Some(local_edge) = self.store.get_edge(eid) { + let e = to_runtime_edge(local_edge, self.store); + result.push(e); + } + } + Ok(filter_limit!(result.into_iter(), params.filter, params.limit)) + } + + fn prepare_explore_vertex( + &self, direction: Direction, params: &QueryParams, + ) -> FnResult>> { + let edge_label_ids = encode_storage_edge_label(params.labels.as_ref()); + let filter = params.filter.clone(); + let limit = params.limit.clone(); + let graph = self.store; + + let stmt = from_fn(move |v: ID| { + let iter = match direction { + Direction::Out => graph.get_out_vertices(v as DefaultId, edge_label_ids.as_ref()), + Direction::In => graph.get_in_vertices(v as DefaultId, edge_label_ids.as_ref()), + Direction::Both => graph.get_both_vertices(v as DefaultId, edge_label_ids.as_ref()), + } + .map(move |v| to_runtime_vertex(v, graph, None)); + Ok(filter_limit!(iter, filter, limit)) + }); + Ok(stmt) + } + + fn prepare_explore_edge( + &self, direction: Direction, params: &QueryParams, + ) -> FnResult>> { + let edge_label_ids = encode_storage_edge_label(¶ms.labels); + let filter = params.filter.clone(); + let limit = params.limit.clone(); + let graph = self.store; + let stmt = from_fn(move |v: ID| { + let iter = match direction { + Direction::Out => graph.get_out_edges(v as DefaultId, edge_label_ids.as_ref()), + Direction::In => graph.get_in_edges(v as DefaultId, edge_label_ids.as_ref()), + Direction::Both => graph.get_both_edges(v as DefaultId, edge_label_ids.as_ref()), + } + .map(move |e| to_runtime_edge(e, graph)); + Ok(filter_limit!(iter, filter, limit)) + }); + Ok(stmt) + } +} + +#[allow(dead_code)] +pub fn create_demo_graph() { + lazy_static::initialize(&GRAPH_PROXY); + register_graph(GRAPH_PROXY.clone()); +} + +#[inline] +fn to_runtime_vertex( + v: LocalVertex, store: &'static LargeGraphDB, + prop_keys: Option>, +) -> Vertex { + // For vertices, we query properties via vid + let label = encode_runtime_v_label(&v); + let details = LazyVertexDetails::new(v.get_id(), prop_keys, store); + Vertex::new(v.get_id() as ID, Some(label), DynDetails::new(details)) +} + +#[inline] +fn to_runtime_edge( + e: LocalEdge, _store: &'static LargeGraphDB, +) -> Edge { + let id = encode_runtime_e_id(&e); + let label = encode_runtime_e_label(&e); + let mut properties = HashMap::new(); + // TODO: For edges, we clone all properties by default for now. But we'd better get properties on demand + if let Some(mut prop_vals) = e.clone_all_properties() { + for (prop, obj) in prop_vals.drain() { + properties.insert(prop.into(), obj as Object); + } + } + + Edge::with_from_src( + id, + Some(label), + e.get_src_id() as ID, + e.get_dst_id() as ID, + e.is_from_start(), + DynDetails::new(DefaultDetails::new(properties)), + ) +} + +/// LazyVertexDetails is used for local property fetching optimization. +/// That is, the required properties will not be materialized until LazyVertexDetails need to be shuffled. +#[allow(dead_code)] +struct LazyVertexDetails { + pub id: DefaultId, + // prop_keys specify the properties we query for, + // Specifically, Some(vec![]) indicates we need all properties + // and None indicates we do not need any property, + prop_keys: Option>, + inner: AtomicPtr>, + store: &'static LargeGraphDB, +} + +impl_as_any!(LazyVertexDetails); + +impl LazyVertexDetails { + pub fn new( + id: DefaultId, prop_keys: Option>, + store: &'static LargeGraphDB, + ) -> Self { + LazyVertexDetails { id, prop_keys, inner: AtomicPtr::default(), store } + } +} + +impl fmt::Debug for LazyVertexDetails { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LazyVertexDetails") + .field("id", &self.id) + .field("inner", &self.inner) + .finish() + } +} + +impl Details for LazyVertexDetails { + fn get_property(&self, key: &NameOrId) -> Option { + if let NameOrId::Str(key) = key { + let mut ptr = self.inner.load(Ordering::SeqCst); + if ptr.is_null() { + if let Some(v) = self.store.get_vertex(self.id) { + let v = Box::new(v); + let new_ptr = Box::into_raw(v); + let swapped = self.inner.swap(new_ptr, Ordering::SeqCst); + if swapped.is_null() { + ptr = new_ptr; + } else { + unsafe { + std::ptr::drop_in_place(new_ptr); + } + ptr = swapped + }; + } else { + return None; + } + } + + unsafe { (*ptr).get_property(key) } + } else { + info!("Have not support getting property by prop_id in experiments store yet"); + None + } + } + + fn get_all_properties(&self) -> Option> { + let mut all_props = HashMap::new(); + if let Some(prop_keys) = self.prop_keys.as_ref() { + // the case of get_all_properties from vertex; + if prop_keys.is_empty() { + let mut ptr = self.inner.load(Ordering::SeqCst); + if ptr.is_null() { + if let Some(v) = self.store.get_vertex(self.id) { + let v = Box::new(v); + let new_ptr = Box::into_raw(v); + let swapped = self.inner.swap(new_ptr, Ordering::SeqCst); + if swapped.is_null() { + ptr = new_ptr; + } else { + unsafe { + std::ptr::drop_in_place(new_ptr); + } + ptr = swapped + }; + } else { + return None; + } + } + unsafe { + if let Some(prop_key_vals) = (*ptr).clone_all_properties() { + all_props = prop_key_vals + .into_iter() + .map(|(prop_key, prop_val)| (prop_key.into(), prop_val as Object)) + .collect(); + } else { + return None; + } + } + } else { + // the case of get_all_properties with prop_keys pre-specified + for key in prop_keys.iter() { + if let Some(prop) = self.get_property(&key) { + all_props.insert(key.clone(), prop.try_to_owned().unwrap()); + } else { + all_props.insert(key.clone(), Object::None); + } + } + } + } + Some(all_props) + } +} + +impl Drop for LazyVertexDetails { + fn drop(&mut self) { + let ptr = self.inner.load(Ordering::SeqCst); + if !ptr.is_null() { + unsafe { + std::ptr::drop_in_place(ptr); + } + } + } +} + +/// Edge's ID is encoded by its internal index +fn encode_runtime_e_id(e: &LocalEdge) -> ID { + let ei = e.get_edge_id(); + ei.1 as ID +} + +pub fn encode_store_e_id(e: &ID) -> EdgeId { + // TODO(longbin) To only use in current partition + (0, *e as usize) +} + +fn encode_runtime_v_label(v: &LocalVertex) -> NameOrId { + NameOrId::Id(v.get_label()[0] as KeyId) +} + +fn encode_runtime_e_label(e: &LocalEdge) -> NameOrId { + NameOrId::Id(e.get_label() as KeyId) +} + +/// Transform string-typed labels into a id-typed labels. +/// `is_true_label` records whether the label is an actual label, or already transformed into +/// an id-type. +fn labels_to_ids(labels: &Vec, is_vertex: bool) -> Option> { + if labels.is_empty() { + None + } else { + Some( + labels + .iter() + .map(|label| match label { + NameOrId::Str(s) => { + let label_id = if is_vertex { + (*GRAPH).get_schema().get_vertex_label_id(s) + } else { + (*GRAPH) + .get_schema() + .get_edge_label_id(s) + .map(|id| id) + }; + label_id.unwrap_or(INVALID_LABEL_ID) + } + NameOrId::Id(id) => *id as LabelId, + }) + .collect::>(), + ) + } +} + +fn encode_storage_vertex_label(labels: &Vec) -> Option> { + labels_to_ids(labels, true) +} + +fn encode_storage_edge_label(labels: &Vec) -> Option> { + labels_to_ids(labels, false) +} + +#[cfg(test)] +mod tests { + use graph_store::ldbc::LDBCVertexParser; + use graph_store::prelude::{DefaultId, GlobalStoreTrait}; + + use super::GRAPH; + + #[test] + fn it_works() { + let v1: DefaultId = LDBCVertexParser::to_global_id(1, 0); + let v2: DefaultId = LDBCVertexParser::to_global_id(2, 0); + let v4: DefaultId = LDBCVertexParser::to_global_id(4, 0); + + let out_iter = GRAPH.get_out_vertices(v1, Some(&vec![0])); + let out: Vec = out_iter.map(|v| v.get_id()).collect(); + assert_eq!(out, vec![v4, v2]); + } +} diff --git a/research/query_service/ir/graph_proxy/src/exp_store/mod.rs b/research/query_service/ir/graph_proxy/src/exp_store/mod.rs new file mode 100644 index 000000000000..6a7b66a4e02c --- /dev/null +++ b/research/query_service/ir/graph_proxy/src/exp_store/mod.rs @@ -0,0 +1,41 @@ +// +//! Copyright 2021 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +mod graph_partition; +mod graph_query; + +pub use graph_partition::SimplePartition; +pub use graph_query::create_demo_graph; +use runtime::IRJobCompiler; + +use crate::InitializeJobCompiler; + +pub struct QueryExpGraph { + num_servers: usize, +} + +impl QueryExpGraph { + pub fn new(num_servers: usize) -> Self { + QueryExpGraph { num_servers } + } +} + +impl InitializeJobCompiler for QueryExpGraph { + fn initialize_job_compiler(&self) -> IRJobCompiler { + create_demo_graph(); + let partitioner = SimplePartition { num_servers: self.num_servers.clone() }; + IRJobCompiler::new(partitioner) + } +} diff --git a/research/query_service/ir/graph_proxy/src/gs_store/graph_partition.rs b/research/query_service/ir/graph_proxy/src/gs_store/graph_partition.rs new file mode 100644 index 000000000000..33bdfbd5b881 --- /dev/null +++ b/research/query_service/ir/graph_proxy/src/gs_store/graph_partition.rs @@ -0,0 +1,194 @@ +// +//! Copyright 2021 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use maxgraph_store::api::graph_partition::GraphPartitionManager; +use maxgraph_store::api::{PartitionId, VertexId}; +use pegasus::api::function::FnResult; +use runtime::error::FnExecError; +use runtime::graph::partitioner::Partitioner; +use runtime::graph::ID; + +/// A partition utility that one server contains multiple graph partitions for MaxGraph (V2) Store +pub struct MaxGraphMultiPartition { + graph_partition_manager: Arc, +} + +#[allow(dead_code)] +impl MaxGraphMultiPartition { + pub fn new(graph_partition_manager: Arc) -> Self { + MaxGraphMultiPartition { graph_partition_manager } + } +} + +impl Partitioner for MaxGraphMultiPartition { + fn get_partition(&self, id: &ID, worker_num_per_server: usize) -> FnResult { + // The partitioning logics is as follows: + // 1. `partition_id = self.graph_partition_manager.get_partition_id(*id as VertexId)` routes a given id + // to the partition that holds its data. + // 2. `server_index = partition_id % self.num_servers as u64` routes the partition id to the + // server R that holds the partition + // 3. `worker_index = partition_id % worker_num_per_server` picks up one worker to do the computation. + // 4. `server_index * worker_num_per_server + worker_index` computes the worker index in server R + // to do the computation. + let vid = *id as VertexId; + let worker_num_per_server = worker_num_per_server as u64; + let partition_id = self + .graph_partition_manager + .get_partition_id(vid) as u64; + let server_index = self + .graph_partition_manager + .get_server_id(partition_id as PartitionId) + .ok_or(FnExecError::query_store_error("get server id failed in graph_partition_manager"))? + as u64; + let worker_index = partition_id % worker_num_per_server; + Ok(server_index * worker_num_per_server + worker_index as u64) + } + + fn get_worker_partitions(&self, job_workers: usize, worker_id: u32) -> FnResult>> { + // Get worker partition list logic is as follows: + // 1. `process_partition_list = self.graph_partition_manager.get_process_partition_list()` + // get all partitions on current server + // 2. 'pid % job_workers' picks one worker to do the computation. + // and 'pid % job_workers == worker_id % job_workers' checks if current worker is the picked worker + let mut worker_partition_list = vec![]; + let process_partition_list = self + .graph_partition_manager + .get_process_partition_list(); + for pid in process_partition_list { + if pid % (job_workers as u32) == worker_id % (job_workers as u32) { + worker_partition_list.push(pid as u64) + } + } + info!( + "job_workers {:?}, worker id: {:?}, worker_partition_list {:?}", + job_workers, worker_id, worker_partition_list + ); + Ok(Some(worker_partition_list)) + } +} + +/// A partition utility that one server contains multiple graph partitions for Vineyard +/// Starting gaia with vineyard will pre-allocate partitions for each worker to process, +/// thus we use graph_partitioner together with partition_worker_mapping for data routing. +pub struct VineyardMultiPartition { + graph_partition_manager: Arc, + // mapping of partition id -> worker id + partition_worker_mapping: Arc>>>, + // mapping of worker id -> partition list + worker_partition_list_mapping: Arc>>>>, + num_servers: usize, +} + +impl VineyardMultiPartition { + pub fn new( + graph_partition_manager: Arc, + partition_worker_mapping: Arc>>>, + worker_partition_list_mapping: Arc>>>>, num_servers: usize, + ) -> VineyardMultiPartition { + VineyardMultiPartition { + graph_partition_manager, + partition_worker_mapping, + worker_partition_list_mapping, + num_servers, + } + } +} + +impl Partitioner for VineyardMultiPartition { + fn get_partition(&self, id: &ID, worker_num_per_server: usize) -> FnResult { + // The partitioning logics is as follows: + // 1. `partition_id = self.graph_partition_manager.get_partition_id(*id as VertexId)` routes a given id + // to the partition that holds its data. + // 2. get worker_id by the prebuild partition_worker_map, which specifies partition_id -> worker_id + + // Firstly, we check if the job parallelism is identical to the pre-allocated parallelism, + let parallelism = self + .worker_partition_list_mapping + .read() + .unwrap() + .as_ref() + .map_or(0, |map| map.len()); + if self.num_servers * worker_num_per_server != parallelism { + Err(FnExecError::query_store_error( + "Job parallelism is not identical to the pre-allocated parallelism", + ))? + } else { + let vid = *id as VertexId; + let partition_id = self + .graph_partition_manager + .get_partition_id(vid) as PartitionId; + if let Ok(partition_worker_mapping) = self.partition_worker_mapping.read() { + if let Some(partition_worker_mapping) = partition_worker_mapping.as_ref() { + if let Some(worker_id) = partition_worker_mapping.get(&partition_id) { + Ok(*worker_id as u64) + } else { + Err(FnExecError::query_store_error( + "get worker id failed in VineyardMultiPartition", + ))? + } + } else { + Err(FnExecError::query_store_error( + "partition_worker_mapping is not initialized in VineyardMultiPartition", + ))? + } + } else { + Err(FnExecError::query_store_error( + "read partition_worker_mapping in VineyardMultiPartition failed", + ))? + } + } + } + + fn get_worker_partitions(&self, job_workers: usize, worker_id: u32) -> FnResult>> { + // If only one worker each server, it will process all partitions + if job_workers == 1 { + Ok(Some( + self.graph_partition_manager + .get_process_partition_list() + .into_iter() + .map(|pid| pid as u64) + .collect(), + )) + } + // Vineyard will pre-allocate the worker_partition_list mapping + else if let Ok(worker_partition_list_mapping) = self.worker_partition_list_mapping.read() { + if let Some(worker_partition_list_mapping) = worker_partition_list_mapping.as_ref() { + if let Some(partition_list) = worker_partition_list_mapping.get(&worker_id) { + Ok(Some( + partition_list + .iter() + .map(|pid| *pid as u64) + .collect(), + )) + } else { + Err(FnExecError::query_store_error( + "get worker partitions failed in VineyardMultiPartition", + ))? + } + } else { + Err(FnExecError::query_store_error( + "worker_partition_list is not initialized in VineyardMultiPartition", + ))? + } + } else { + Err(FnExecError::query_store_error( + "read worker_partition_list failed in VineyardMultiPartition", + ))? + } + } +} diff --git a/research/query_service/ir/graph_proxy/src/gs_store/graph_query.rs b/research/query_service/ir/graph_proxy/src/gs_store/graph_query.rs new file mode 100644 index 000000000000..9c84383c8e44 --- /dev/null +++ b/research/query_service/ir/graph_proxy/src/gs_store/graph_query.rs @@ -0,0 +1,453 @@ +// +//! Copyright 2021 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use dyn_type::{Object, Primitives}; +use graph_store::utils::IterList; +use ir_common::NameOrId as Label; +use ir_common::{KeyId, NameOrId}; +use maxgraph_store::api::graph_partition::GraphPartitionManager; +use maxgraph_store::api::prelude::Property; +use maxgraph_store::api::*; +use maxgraph_store::api::{Edge as StoreEdge, Vertex as StoreVertex}; +use maxgraph_store::api::{PropId, SnapshotId}; +use pegasus::api::function::FnResult; +use runtime::error::{FnExecError, FnExecResult}; +use runtime::graph::element::{Edge, Vertex}; +use runtime::graph::property::{DefaultDetails, DynDetails}; +use runtime::graph::{Direction, GraphProxy, QueryParams, Statement, ID}; +use runtime::register_graph; + +use crate::from_fn; +use crate::{filter_limit, limit_n}; + +// Should be identical to the param_name given by compiler +const SNAPSHOT_ID: &str = "SID"; +// This will refer to the latest graph +const DEFAULT_SNAPSHOT_ID: SnapshotId = SnapshotId::max_value() - 1; + +pub struct GraphScopeStore +where + V: StoreVertex + 'static, + VI: Iterator + Send + 'static, + E: StoreEdge + 'static, + EI: Iterator + Send + 'static, +{ + store: Arc>, + partition_manager: Arc, +} + +#[allow(dead_code)] +pub fn create_gs_store( + store: Arc>, + partition_manager: Arc, +) where + V: StoreVertex + 'static, + VI: Iterator + Send + 'static, + E: StoreEdge + 'static, + EI: Iterator + Send + 'static, +{ + let graph = GraphScopeStore { store, partition_manager }; + register_graph(Arc::new(graph)); +} + +impl GraphProxy for GraphScopeStore +where + V: StoreVertex + 'static, + VI: Iterator + Send + 'static, + E: StoreEdge + 'static, + EI: Iterator + Send + 'static, +{ + fn scan_vertex(&self, params: &QueryParams) -> FnResult + Send>> { + if let Some(partitions) = params.partitions.as_ref() { + let store = self.store.clone(); + let si = params + .get_extra_param(SNAPSHOT_ID) + .map(|s| { + s.parse::() + .unwrap_or(DEFAULT_SNAPSHOT_ID) + }) + .unwrap_or(DEFAULT_SNAPSHOT_ID); + let label_ids = encode_storage_label(params.labels.as_ref())?; + let prop_ids = encode_storage_prop_keys(params.columns.as_ref())?; + let filter = params.filter.clone(); + let partitions: Vec = partitions + .iter() + .map(|pid| *pid as PartitionId) + .collect(); + let result = store + .get_all_vertices( + si, + label_ids.as_ref(), + // None means no filter condition pushed down to storage as not supported yet. Same as follows. + None, + // None means no need to dedup by properties. Same as follows. + None, + prop_ids.as_ref(), + // Zero limit means no limit. Same as follows. + params.limit.unwrap_or(0), + // Each worker will scan the partitions pre-allocated in source operator. Same as follows. + partitions.as_ref(), + ) + .map(move |v| to_runtime_vertex(&v)); + + Ok(filter_limit!(result, filter, None)) + } else { + Ok(Box::new(std::iter::empty())) + } + } + + fn scan_edge(&self, params: &QueryParams) -> FnResult + Send>> { + if let Some(partitions) = params.partitions.as_ref() { + let store = self.store.clone(); + let si = params + .get_extra_param(SNAPSHOT_ID) + .map(|s| { + s.parse::() + .unwrap_or(DEFAULT_SNAPSHOT_ID) + }) + .unwrap_or(DEFAULT_SNAPSHOT_ID); + let label_ids = encode_storage_label(params.labels.as_ref())?; + let prop_ids = encode_storage_prop_keys(params.columns.as_ref())?; + let filter = params.filter.clone(); + let partitions: Vec = partitions + .iter() + .map(|pid| *pid as PartitionId) + .collect(); + let result = store + .get_all_edges( + si, + label_ids.as_ref(), + None, + None, + prop_ids.as_ref(), + params.limit.unwrap_or(0), + partitions.as_ref(), + ) + .map(move |e| to_runtime_edge(&e)); + + Ok(filter_limit!(result, filter, None)) + } else { + Ok(Box::new(std::iter::empty())) + } + } + + fn get_vertex( + &self, ids: &[ID], params: &QueryParams, + ) -> FnResult + Send>> { + let store = self.store.clone(); + let si = params + .get_extra_param(SNAPSHOT_ID) + .map(|s| { + s.parse::() + .unwrap_or(DEFAULT_SNAPSHOT_ID) + }) + .unwrap_or(DEFAULT_SNAPSHOT_ID); + let prop_ids = encode_storage_prop_keys(params.columns.as_ref())?; + let filter = params.filter.clone(); + let partition_label_vertex_ids = + get_partition_label_vertex_ids(ids, self.partition_manager.clone()); + let result = store + .get_vertex_properties(si, partition_label_vertex_ids, prop_ids.as_ref()) + .map(move |v| to_runtime_vertex(&v)); + + Ok(filter_limit!(result, filter, None)) + } + + fn get_edge( + &self, _ids: &[ID], _params: &QueryParams, + ) -> FnResult + Send>> { + // TODO(bingqing): adapt get_edge when graphscope support this + Err(FnExecError::query_store_error("GraphScope storage does not support get_edge for now"))? + } + + fn prepare_explore_vertex( + &self, direction: Direction, params: &QueryParams, + ) -> FnResult>> { + let filter = params.filter.clone(); + let limit = params.limit.clone(); + let store = self.store.clone(); + let partition_manager = self.partition_manager.clone(); + let si = params + .get_extra_param(SNAPSHOT_ID) + .map(|s| { + s.parse::() + .unwrap_or(DEFAULT_SNAPSHOT_ID) + }) + .unwrap_or(DEFAULT_SNAPSHOT_ID); + let edge_label_ids = encode_storage_label(params.labels.as_ref())?; + + let stmt = from_fn(move |v: ID| { + let src_id = get_partition_vertex_ids(v, partition_manager.clone()); + let iter = match direction { + Direction::Out => store.get_out_vertex_ids( + si, + src_id, + edge_label_ids.as_ref(), + None, + None, + limit.unwrap_or(0), + ), + Direction::In => store.get_in_vertex_ids( + si, + src_id, + edge_label_ids.as_ref(), + None, + None, + limit.unwrap_or(0), + ), + Direction::Both => { + let mut iters = vec![]; + let out_iter = store.get_out_vertex_ids( + si, + src_id.clone(), + edge_label_ids.as_ref(), + None, + None, + limit.clone().unwrap_or(0), + ); + iters.push(out_iter); + let in_iter = store.get_in_vertex_ids( + si, + src_id, + edge_label_ids.as_ref(), + None, + None, + limit.unwrap_or(0), + ); + iters.push(in_iter); + Box::new(IterList::new(iters)) + } + }; + let iters = iter.map(|(_src, vi)| vi).collect(); + let iter_list = IterList::new(iters).map(move |v| to_runtime_vertex(&v)); + Ok(filter_limit!(iter_list, filter, None)) + }); + Ok(stmt) + } + + fn prepare_explore_edge( + &self, direction: Direction, params: &QueryParams, + ) -> FnResult>> { + let store = self.store.clone(); + let si = params + .get_extra_param(SNAPSHOT_ID) + .map(|s| { + s.parse::() + .unwrap_or(DEFAULT_SNAPSHOT_ID) + }) + .unwrap_or(DEFAULT_SNAPSHOT_ID); + let partition_manager = self.partition_manager.clone(); + let filter = params.filter.clone(); + let limit = params.limit.clone(); + let edge_label_ids = encode_storage_label(params.labels.as_ref())?; + let prop_ids = encode_storage_prop_keys(params.columns.as_ref())?; + + let stmt = from_fn(move |v: ID| { + let src_id = get_partition_vertex_ids(v, partition_manager.clone()); + let iter = match direction { + Direction::Out => store.get_out_edges( + si, + src_id, + edge_label_ids.as_ref(), + None, + None, + prop_ids.as_ref(), + limit.unwrap_or(0), + ), + Direction::In => store.get_in_edges( + si, + src_id, + edge_label_ids.as_ref(), + None, + None, + prop_ids.as_ref(), + limit.unwrap_or(0), + ), + Direction::Both => { + let mut iter = vec![]; + let out_iter = store.get_out_edges( + si, + src_id.clone(), + edge_label_ids.as_ref(), + None, + None, + prop_ids.as_ref(), + limit.clone().unwrap_or(0), + ); + iter.push(out_iter); + let in_iter = store.get_in_edges( + si, + src_id, + edge_label_ids.as_ref(), + None, + None, + prop_ids.as_ref(), + limit.unwrap_or(0), + ); + iter.push(in_iter); + Box::new(IterList::new(iter)) + } + }; + let iters = iter.map(|(_src, ei)| ei).collect(); + let iter_list = IterList::new(iters).map(move |e| to_runtime_edge(&e)); + Ok(filter_limit!(iter_list, filter, None)) + }); + Ok(stmt) + } +} + +#[inline] +fn to_runtime_vertex(v: &V) -> Vertex { + let id = v.get_id() as ID; + let label = encode_runtime_v_label(v); + let properties = v + .get_properties() + .map(|(prop_id, prop_val)| encode_runtime_property(prop_id, prop_val)) + .collect(); + Vertex::new(id, Some(label), DynDetails::new(DefaultDetails::new(properties))) +} + +#[inline] +fn to_runtime_edge(e: &E) -> Edge { + let id = e.get_edge_id() as ID; + let label = encode_runtime_e_label(e); + let properties = e + .get_properties() + .map(|(prop_id, prop_val)| encode_runtime_property(prop_id, prop_val)) + .collect(); + let mut edge = Edge::new( + id, + Some(label), + e.get_src_id() as ID, + e.get_dst_id() as ID, + DynDetails::new(DefaultDetails::new(properties)), + ); + edge.set_src_label(Some(NameOrId::Id(e.get_src_label_id() as KeyId))); + edge.set_dst_label(Some(NameOrId::Id(e.get_dst_label_id() as KeyId))); + edge +} + +/// in maxgraph store, Option>: None means we need all properties, +/// and Some means we need given properties (and Some(vec![]) means we do not need any property) +/// while in ir, None means we do not need any properties, +/// and Some means we need given properties (and Some(vec![]) means we need all properties) +#[inline] +fn encode_storage_prop_keys(prop_names: Option<&Vec>) -> FnExecResult>> { + if let Some(prop_names) = prop_names { + if prop_names.is_empty() { + Ok(None) + } else { + let encoded_prop_ids = prop_names + .iter() + .map(|prop_key| match prop_key { + NameOrId::Str(_) => Err(FnExecError::query_store_error( + "encode storage prop key error, should provide prop_id", + )), + NameOrId::Id(prop_id) => Ok(*prop_id as PropId), + }) + .collect::, _>>()?; + Ok(Some(encoded_prop_ids)) + } + } else { + Ok(Some(vec![])) + } +} + +#[inline] +fn encode_storage_label(labels: &Vec