Skip to content

Commit

Permalink
[GIE Compiler] Introduce GraphPlanner to unify logical and physical…
Browse files Browse the repository at this point in the history
… plan building (#2838)

<!--
Thanks for your contribution! please review
https://github.com/alibaba/GraphScope/blob/main/CONTRIBUTING.md before
opening an issue.
-->

## What do these changes do?
1. re-design `PhysicalBuilder` to build physical plan from logical plan,
`FfiPhysicalBuilder` is one of the implementations which will build
physical plan in ir core structure
2. introduce `GraphPlanner` to unify logical and physical plan building
phases
3. other minor fix: refine the structure of `IrMeta` which contains
schema info
<!-- Please give a short brief about these changes. -->

## Related issue number

<!-- Are there any issues opened that will be resolved by merging this
change? -->

Fixes

---------

Co-authored-by: Longbin Lai <longbin.lailb@alibaba-inc.com>
  • Loading branch information
shirly121 and longbinlai committed Jun 8, 2023
1 parent a8b9dce commit 1ba6102
Show file tree
Hide file tree
Showing 35 changed files with 643 additions and 424 deletions.
11 changes: 11 additions & 0 deletions interactive_engine/compiler/Makefile
Expand Up @@ -20,6 +20,9 @@ target.revision:=0.0.1-SNAPSHOT

QUIET_OPT := --quiet

query:=
physical:=

build:
cd $(CUR_DIR)/.. && \
mvn clean install -DskipTests -Drevision=${target.revision} -Pexperimental ${QUIET_OPT} && \
Expand Down Expand Up @@ -57,6 +60,14 @@ run:
-Dpegasus.hosts=${pegasus.hosts} \
com.alibaba.graphscope.gremlin.service.GraphServiceMain

# make physical_plan query='<query in string>' physical='<path to the physical output file>'
physical_plan:
cd $(CUR_DIR) && $(java) \
-cp ".:./target/libs/*:./target/compiler-0.0.1-SNAPSHOT.jar" \
-Djna.library.path=../executor/ir/target/release \
-Dgraph.schema=${graph.schema} \
com.alibaba.graphscope.common.ir.tools.GraphPlanner "$(query)" "$(physical)"

# start rpc server
# make run graph.schema:=../executor/ir/core/resource/ldbc_schema.json
ldbc_test:
Expand Down
3 changes: 3 additions & 0 deletions interactive_engine/compiler/src/main/antlr4/CypherGS.g4
Expand Up @@ -26,6 +26,9 @@ oC_Statement
: oC_Query ;

oC_Query
: oC_RegularQuery ;

oC_RegularQuery
: oC_Match ( SP? oC_With )* ( SP oC_Return ) ;

oC_Match
Expand Down
Expand Up @@ -599,7 +599,7 @@ public Pointer createParams(QueryParams params) {
}

public IrPlan(IrMeta meta, InterOpCollection opCollection) {
irCoreLib.setSchema(meta.getSchemaJson());
irCoreLib.setSchema(meta.getSchema().schemaJson());
this.ptrPlan = irCoreLib.initLogicalPlan();
// add snapshot to QueryParams
for (InterOpBase op : opCollection.unmodifiableCollection()) {
Expand All @@ -611,9 +611,10 @@ public IrPlan(IrMeta meta, InterOpCollection opCollection) {
} else if (op instanceof GetVOp && ((GetVOp) op).getParams().isPresent()) {
params = ((GetVOp) op).getParams().get();
}
if (params != null && meta.isAcquireSnapshot()) {
if (params != null && meta.getSnapshotId().isAcquired()) {
params.addExtraParams(
QueryParams.SNAPSHOT_CONFIG_NAME, String.valueOf(meta.getSnapshotId()));
QueryParams.SNAPSHOT_CONFIG_NAME,
String.valueOf(meta.getSnapshotId().getId()));
}
}
appendInterOpCollection(-1, opCollection);
Expand Down
@@ -0,0 +1,26 @@
/*
* Copyright 2020 Alibaba Group Holding Limited.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.alibaba.graphscope.common.antlr4;

import org.antlr.v4.runtime.tree.ParseTree;

/**
* parse DSL statement to antlr tree
*/
public interface Antlr4Parser {
ParseTree parse(String statement);
}

This file was deleted.

@@ -0,0 +1,60 @@
/*
* Copyright 2020 Alibaba Group Holding Limited.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.alibaba.graphscope.common.ir.runtime;

import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
import com.alibaba.graphscope.common.ir.tools.LogicalPlan;

import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelShuttle;
import org.apache.calcite.rel.RelVisitor;
import org.checkerframework.checker.nullness.qual.Nullable;

/**
* build physical plan from logical plan of a regular query
* @param <T>
* @param <R>
*/
public abstract class RegularPhysicalBuilder<T, R> extends PhysicalBuilder<R> {
protected RelShuttle relShuttle;

protected RegularPhysicalBuilder(LogicalPlan logicalPlan, RelShuttle relShuttle) {
super(logicalPlan);
this.relShuttle = relShuttle;
}

protected void initialize() {
if (this.logicalPlan.getRegularQuery() != null && !this.logicalPlan.isReturnEmpty()) {
RelNode regularQuery = this.logicalPlan.getRegularQuery();
RelVisitor relVisitor =
new RelVisitor() {
@Override
public void visit(RelNode node, int ordinal, @Nullable RelNode parent) {
super.visit(node, ordinal, parent);
appendNode((PhysicalNode) node.accept(relShuttle));
}
};
relVisitor.go(regularQuery);
}
}

/**
* append {@code PhysicalNode} to the physical plan
* @param node
*/
protected abstract void appendNode(PhysicalNode<T> node);
}
Expand Up @@ -16,10 +16,13 @@

package com.alibaba.graphscope.common.ir.runtime.ffi;

import com.alibaba.graphscope.common.config.Configs;
import com.alibaba.graphscope.common.config.PegasusConfig;
import com.alibaba.graphscope.common.intermediate.ArgUtils;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject;
import com.alibaba.graphscope.common.ir.rel.GraphLogicalSort;
import com.alibaba.graphscope.common.ir.rel.GraphRelShuttleWrapper;
import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalExpand;
import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalGetV;
import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalPathExpand;
Expand All @@ -28,53 +31,55 @@
import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch;
import com.alibaba.graphscope.common.ir.rel.type.group.GraphGroupKeys;
import com.alibaba.graphscope.common.ir.rex.RexGraphVariable;
import com.alibaba.graphscope.common.ir.runtime.RegularPhysicalBuilder;
import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter;
import com.alibaba.graphscope.common.ir.runtime.type.LogicalNode;
import com.alibaba.graphscope.common.ir.runtime.type.LogicalPlan;
import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
import com.alibaba.graphscope.common.ir.tools.AliasInference;
import com.alibaba.graphscope.common.ir.tools.LogicalPlan;
import com.alibaba.graphscope.common.ir.tools.config.GraphOpt;
import com.alibaba.graphscope.common.jna.IrCoreLibrary;
import com.alibaba.graphscope.common.jna.type.*;
import com.alibaba.graphscope.common.jna.type.FfiData;
import com.alibaba.graphscope.common.jna.type.FfiPbPointer;
import com.alibaba.graphscope.common.jna.type.FfiResult;
import com.alibaba.graphscope.common.jna.type.ResultCode;
import com.alibaba.graphscope.common.store.IrMeta;
import com.alibaba.graphscope.gaia.proto.OuterExpression;
import com.google.common.base.Preconditions;
import com.sun.jna.Pointer;
import com.sun.jna.ptr.IntByReference;

import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.hint.RelHint;
import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexVariable;
import org.apache.commons.lang3.StringUtils;

import java.util.List;

/**
* build ffi logical plan in ir core by jna invocation
* build physical plan from logical plan of a regular query, the physical plan is actually denoted by ir core structure (FFI Pointer)
*/
public class FfiLogicalPlan extends LogicalPlan<Pointer, byte[]> {
public class FfiPhysicalBuilder extends RegularPhysicalBuilder<Pointer, byte[]> {
private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE;

private final IrMeta irMeta;
private final Configs graphConfig;
private final Pointer ptrPlan;

private int lastIdx;

public FfiLogicalPlan(RelOptCluster cluster, IrMeta irMeta, List<RelHint> hints) {
super(cluster, hints);
checkFfiResult(LIB.setSchema(irMeta.getSchemaJson()));
public FfiPhysicalBuilder(Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan) {
super(
logicalPlan,
new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId())));
this.graphConfig = graphConfig;
this.irMeta = irMeta;
checkFfiResult(LIB.setSchema(irMeta.getSchema().schemaJson()));
this.ptrPlan = LIB.initLogicalPlan();
this.lastIdx = -1;
initialize();
}

@Override
public void appendNode(LogicalNode<Pointer> node) {
if (isReturnEmpty()) {
throw new IllegalArgumentException(
"should not append any node to the logical pb if returnEmpty is set to true");
}
public void appendNode(PhysicalNode<Pointer> node) {
IntByReference oprIdx = new IntByReference(this.lastIdx);
RelNode original = node.getOriginal();
if (original instanceof GraphLogicalSource) {
Expand Down Expand Up @@ -126,7 +131,6 @@ public void appendNode(LogicalNode<Pointer> node) {

@Override
public String explain() {
if (isReturnEmpty()) return StringUtils.EMPTY;
FfiResult res = LIB.printPlanAsJson(this.ptrPlan);
if (res == null || res.code != ResultCode.Success) {
throw new IllegalStateException("print plan in ir core fail, msg : %s" + res, null);
Expand All @@ -135,11 +139,10 @@ public String explain() {
}

@Override
public byte[] toPhysical() {
if (isReturnEmpty()) return null;
public byte[] build() {
appendSink(new IntByReference(this.lastIdx));
int servers = Integer.valueOf(hints.get(0).kvOptions.get("servers"));
int workers = Integer.valueOf(hints.get(0).kvOptions.get("workers"));
int servers = PegasusConfig.PEGASUS_HOSTS.get(graphConfig).split(",").length;
int workers = PegasusConfig.PEGASUS_WORKER_NUM.get(graphConfig);
FfiData.ByValue ffiData = LIB.buildPhysicalPlan(ptrPlan, workers, servers);
checkFfiResult(ffiData.error);
byte[] bytes = ffiData.getBytes();
Expand All @@ -162,10 +165,10 @@ private void checkFfiResult(FfiResult res) {
}

private boolean isColumnId() {
return Boolean.valueOf(hints.get(0).kvOptions.get("isColumnId"));
return this.irMeta.getSchema().isColumnId();
}

private void appendMatch(LogicalNode<Pointer> node, IntByReference oprIdx) {
private void appendMatch(PhysicalNode<Pointer> node, IntByReference oprIdx) {
// append dummy source
Pointer ptrScan = LIB.initScanOperator(Utils.ffiScanOpt(GraphOpt.Source.VERTEX));
checkFfiResult(LIB.appendScanOperator(ptrPlan, ptrScan, oprIdx.getValue(), oprIdx));
Expand Down

0 comments on commit 1ba6102

Please sign in to comment.