[GIE Compiler] Introduce GraphPlanner to unify logical and physical plan building

interactive_engine/compiler/Makefile

@@ -20,6 +20,9 @@ target.revision:=0.0.1-SNAPSHOT
 
 QUIET_OPT := --quiet
 
+query:=
+physical:=
+
 build:
 	cd $(CUR_DIR)/.. && \
 		mvn clean install -DskipTests -Drevision=${target.revision} -Pexperimental ${QUIET_OPT} && \
@@ -57,6 +60,14 @@ run:
 	  -Dpegasus.hosts=${pegasus.hosts} \
 	  com.alibaba.graphscope.gremlin.service.GraphServiceMain
 
+# make physical_plan query='<query in string>' physical='<path to the physical output file>'
+physical_plan:
+	cd $(CUR_DIR) && $(java) \
+		-cp ".:./target/libs/*:./target/compiler-0.0.1-SNAPSHOT.jar" \
+		-Djna.library.path=../executor/ir/target/release \
+		-Dgraph.schema=${graph.schema} \
+		com.alibaba.graphscope.common.ir.tools.GraphPlanner "$(query)" "$(physical)"
+
 # start rpc server
 # make run graph.schema:=../executor/ir/core/resource/ldbc_schema.json 
 ldbc_test:

interactive_engine/compiler/src/main/antlr4/CypherGS.g4

@@ -26,6 +26,9 @@ oC_Statement
       :  oC_Query ;
 
 oC_Query
+     :  oC_RegularQuery ;
+
+oC_RegularQuery
      :  oC_Match ( SP? oC_With )* ( SP oC_Return ) ;
 
 oC_Match

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java

@@ -599,7 +599,7 @@ public Pointer createParams(QueryParams params) {
     }
 
     public IrPlan(IrMeta meta, InterOpCollection opCollection) {
-        irCoreLib.setSchema(meta.getSchemaJson());
+        irCoreLib.setSchema(meta.getSchema().schemaJson());
         this.ptrPlan = irCoreLib.initLogicalPlan();
         // add snapshot to QueryParams
         for (InterOpBase op : opCollection.unmodifiableCollection()) {
@@ -611,9 +611,10 @@ public IrPlan(IrMeta meta, InterOpCollection opCollection) {
             } else if (op instanceof GetVOp && ((GetVOp) op).getParams().isPresent()) {
                 params = ((GetVOp) op).getParams().get();
             }
-            if (params != null && meta.isAcquireSnapshot()) {
+            if (params != null && meta.getSnapshotId().isAcquired()) {
                 params.addExtraParams(
-                        QueryParams.SNAPSHOT_CONFIG_NAME, String.valueOf(meta.getSnapshotId()));
+                        QueryParams.SNAPSHOT_CONFIG_NAME,
+                        String.valueOf(meta.getSnapshotId().getId()));
             }
         }
         appendInterOpCollection(-1, opCollection);

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/antlr4/Antlr4Parser.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.graphscope.common.antlr4;
+
+import org.antlr.v4.runtime.tree.ParseTree;
+
+/**
+ * parse DSL statement to antlr tree
+ */
+public interface Antlr4Parser {
+    ParseTree parse(String statement);
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/RegularPhysicalBuilder.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.graphscope.common.ir.runtime;
+
+import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
+import com.alibaba.graphscope.common.ir.tools.LogicalPlan;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
+import org.apache.calcite.rel.RelVisitor;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+/**
+ * build physical plan from logical plan of a regular query
+ * @param <T>
+ * @param <R>
+ */
+public abstract class RegularPhysicalBuilder<T, R> extends PhysicalBuilder<R> {
+    protected RelShuttle relShuttle;
+
+    protected RegularPhysicalBuilder(LogicalPlan logicalPlan, RelShuttle relShuttle) {
+        super(logicalPlan);
+        this.relShuttle = relShuttle;
+    }
+
+    protected void initialize() {
+        if (this.logicalPlan.getRegularQuery() != null && !this.logicalPlan.isReturnEmpty()) {
+            RelNode regularQuery = this.logicalPlan.getRegularQuery();
+            RelVisitor relVisitor =
+                    new RelVisitor() {
+                        @Override
+                        public void visit(RelNode node, int ordinal, @Nullable RelNode parent) {
+                            super.visit(node, ordinal, parent);
+                            appendNode((PhysicalNode) node.accept(relShuttle));
+                        }
+                    };
+            relVisitor.go(regularQuery);
+        }
+    }
+
+    /**
+     * append {@code PhysicalNode} to the physical plan
+     * @param node
+     */
+    protected abstract void appendNode(PhysicalNode<T> node);
+}

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiLogicalPlan.java → interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java

@@ -16,10 +16,13 @@
 
 package com.alibaba.graphscope.common.ir.runtime.ffi;
 
+import com.alibaba.graphscope.common.config.Configs;
+import com.alibaba.graphscope.common.config.PegasusConfig;
 import com.alibaba.graphscope.common.intermediate.ArgUtils;
 import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate;
 import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject;
 import com.alibaba.graphscope.common.ir.rel.GraphLogicalSort;
+import com.alibaba.graphscope.common.ir.rel.GraphRelShuttleWrapper;
 import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalExpand;
 import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalGetV;
 import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalPathExpand;
@@ -28,53 +31,55 @@
 import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch;
 import com.alibaba.graphscope.common.ir.rel.type.group.GraphGroupKeys;
 import com.alibaba.graphscope.common.ir.rex.RexGraphVariable;
+import com.alibaba.graphscope.common.ir.runtime.RegularPhysicalBuilder;
 import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter;
-import com.alibaba.graphscope.common.ir.runtime.type.LogicalNode;
-import com.alibaba.graphscope.common.ir.runtime.type.LogicalPlan;
+import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode;
 import com.alibaba.graphscope.common.ir.tools.AliasInference;
+import com.alibaba.graphscope.common.ir.tools.LogicalPlan;
 import com.alibaba.graphscope.common.ir.tools.config.GraphOpt;
 import com.alibaba.graphscope.common.jna.IrCoreLibrary;
-import com.alibaba.graphscope.common.jna.type.*;
+import com.alibaba.graphscope.common.jna.type.FfiData;
+import com.alibaba.graphscope.common.jna.type.FfiPbPointer;
+import com.alibaba.graphscope.common.jna.type.FfiResult;
+import com.alibaba.graphscope.common.jna.type.ResultCode;
 import com.alibaba.graphscope.common.store.IrMeta;
 import com.alibaba.graphscope.gaia.proto.OuterExpression;
 import com.google.common.base.Preconditions;
 import com.sun.jna.Pointer;
 import com.sun.jna.ptr.IntByReference;
 
-import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.hint.RelHint;
 import org.apache.calcite.rel.logical.LogicalFilter;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexVariable;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
 
 /**
- * build ffi logical plan in ir core by jna invocation
+ * build physical plan from logical plan of a regular query, the physical plan is actually denoted by ir core structure (FFI Pointer)
  */
-public class FfiLogicalPlan extends LogicalPlan<Pointer, byte[]> {
+public class FfiPhysicalBuilder extends RegularPhysicalBuilder<Pointer, byte[]> {
     private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE;
-
+    private final IrMeta irMeta;
+    private final Configs graphConfig;
     private final Pointer ptrPlan;
-
     private int lastIdx;
 
-    public FfiLogicalPlan(RelOptCluster cluster, IrMeta irMeta, List<RelHint> hints) {
-        super(cluster, hints);
-        checkFfiResult(LIB.setSchema(irMeta.getSchemaJson()));
+    public FfiPhysicalBuilder(Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan) {
+        super(
+                logicalPlan,
+                new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId())));
+        this.graphConfig = graphConfig;
+        this.irMeta = irMeta;
+        checkFfiResult(LIB.setSchema(irMeta.getSchema().schemaJson()));
         this.ptrPlan = LIB.initLogicalPlan();
         this.lastIdx = -1;
+        initialize();
     }
 
     @Override
-    public void appendNode(LogicalNode<Pointer> node) {
-        if (isReturnEmpty()) {
-            throw new IllegalArgumentException(
-                    "should not append any node to the logical pb if returnEmpty is set to true");
-        }
+    public void appendNode(PhysicalNode<Pointer> node) {
         IntByReference oprIdx = new IntByReference(this.lastIdx);
         RelNode original = node.getOriginal();
         if (original instanceof GraphLogicalSource) {
@@ -126,7 +131,6 @@ public void appendNode(LogicalNode<Pointer> node) {
 
     @Override
     public String explain() {
-        if (isReturnEmpty()) return StringUtils.EMPTY;
         FfiResult res = LIB.printPlanAsJson(this.ptrPlan);
         if (res == null || res.code != ResultCode.Success) {
             throw new IllegalStateException("print plan in ir core fail, msg : %s" + res, null);
@@ -135,11 +139,10 @@ public String explain() {
     }
 
     @Override
-    public byte[] toPhysical() {
-        if (isReturnEmpty()) return null;
+    public byte[] build() {
         appendSink(new IntByReference(this.lastIdx));
-        int servers = Integer.valueOf(hints.get(0).kvOptions.get("servers"));
-        int workers = Integer.valueOf(hints.get(0).kvOptions.get("workers"));
+        int servers = PegasusConfig.PEGASUS_HOSTS.get(graphConfig).split(",").length;
+        int workers = PegasusConfig.PEGASUS_WORKER_NUM.get(graphConfig);
         FfiData.ByValue ffiData = LIB.buildPhysicalPlan(ptrPlan, workers, servers);
         checkFfiResult(ffiData.error);
         byte[] bytes = ffiData.getBytes();
@@ -162,10 +165,10 @@ private void checkFfiResult(FfiResult res) {
     }
 
     private boolean isColumnId() {
-        return Boolean.valueOf(hints.get(0).kvOptions.get("isColumnId"));
+        return this.irMeta.getSchema().isColumnId();
     }
 
-    private void appendMatch(LogicalNode<Pointer> node, IntByReference oprIdx) {
+    private void appendMatch(PhysicalNode<Pointer> node, IntByReference oprIdx) {
         // append dummy source
         Pointer ptrScan = LIB.initScanOperator(Utils.ffiScanOpt(GraphOpt.Source.VERTEX));
         checkFfiResult(LIB.appendScanOperator(ptrPlan, ptrScan, oprIdx.getValue(), oprIdx));