[GIE] fuse expand with count operators nested in apply for optimizati…

…on (#1987)

* [IR Proto] add fused op in IR

* [IR Runtime] Implement for FusedOp

* [IR Proto] add ExpandOpt in EdgeExpand

* [IR Runtime] Implemnt for expand_degree in EdgeExpand

* minor refine: rename in ExpandOpt

* [IR Proto] remove is_edge from EdgeExpand

* [CI Tests] fix all related ci tests

* format the codes

* [GIE] Change `is_edge` to `expand_opt` in ffi/logical plan/physical plan.

* [GIE] Implement fusing Apply + EdgeExpand + Count operators in IR-core.

* [IR Compiler] fuse expand with count nested in apply

* [GIE] Implement fusing Apply + EdgeExpand + Count operators in IR-runtime.

* [GIE] FlatMap FusedOperator instead of FilterMap

* [IR Compiler] add ci tests of group().by(outE().count())

* [IR Compiler] add ExpandFusionStepStrategy and remove ElementFusionStrategy

* [GIE] Further fuse Select(a) + Expand + Count

* [IR Compiler] refine codes

* [IR Compiler] minor fix

* [IR Compiler] set timeout of compiler to 3000s

* [IR Compiler] minor fix

* [IR Compiler] add copyrights

Co-authored-by: BingqingLyu <lv_bingqing@163.com>
Co-authored-by: longbin.lailb <longbin.lailb@alibaba-inc.com>

interactive_engine/compiler/Makefile

@@ -56,7 +56,7 @@ run:
 	  com.alibaba.graphscope.gremlin.service.GraphServiceMain
 
 # start rpc server
-# make run graph.schema:=../core/resource/ldbc_schema.json 
+# make run graph.schema:=../executor/ir/core/resource/ldbc_schema.json 
 ldbc_test:
 	mvn test -Dtest=com.alibaba.graphscope.integration.ldbc.IrLdbcTest $(OSFLAG)
 

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java

@@ -128,14 +128,13 @@ public Pointer apply(InterOpBase baseOp) {
                             baseOp.getClass(), "direction", "not present");
                 }
                 FfiDirection ffiDirection = (FfiDirection) direction.get().applyArg();
-                Optional<OpArg> edgeOpt = op.getIsEdge();
+                Optional<OpArg> edgeOpt = op.getExpandOpt();
                 if (!edgeOpt.isPresent()) {
                     throw new InterOpIllegalArgException(
                             baseOp.getClass(), "edgeOpt", "not present");
                 }
-                Boolean isEdge = (Boolean) edgeOpt.get().applyArg();
-                Pointer expand = irCoreLib.initEdgexpdOperator(isEdge, ffiDirection);
-
+                FfiExpandOpt expandOpt = (FfiExpandOpt) edgeOpt.get().applyArg();
+                Pointer expand = irCoreLib.initEdgexpdOperator(expandOpt, ffiDirection);
                 // set params
                 Optional<QueryParams> paramsOpt = op.getParams();
                 if (paramsOpt.isPresent()) {
@@ -146,7 +145,6 @@ public Pointer apply(InterOpBase baseOp) {
                                 baseOp.getClass(), "params", "setEdgexpdParams returns " + e1.msg);
                     }
                 }
-
                 Optional<OpArg> aliasOpt = baseOp.getAlias();
                 if (aliasOpt.isPresent() && ClassUtils.equalClass(baseOp, ExpandOp.class)) {
                     FfiAlias.ByValue alias = (FfiAlias.ByValue) aliasOpt.get().applyArg();

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/FrontendConfig.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.alibaba.graphscope.common.config;
 
 public class FrontendConfig {

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/config/GraphConfig.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.alibaba.graphscope.common.config;
 
 public class GraphConfig {

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/InterOpCollection.java

@@ -23,7 +23,6 @@
 import com.alibaba.graphscope.common.intermediate.process.InterOpProcessor;
 import com.alibaba.graphscope.common.intermediate.process.SinkOutputProcessor;
 import com.alibaba.graphscope.common.intermediate.process.SubGraphProjectProcessor;
-import com.alibaba.graphscope.common.intermediate.strategy.ElementFusionStrategy;
 import com.alibaba.graphscope.common.intermediate.strategy.InterOpStrategy;
 import com.alibaba.graphscope.common.intermediate.strategy.TopKStrategy;
 
@@ -37,8 +36,7 @@
 // collection of intermediate operators
 public class InterOpCollection {
     private List<InterOpBase> opCollection;
-    private static List<InterOpStrategy> strategies =
-            Arrays.asList(TopKStrategy.INSTANCE, ElementFusionStrategy.INSTANCE);
+    private static List<InterOpStrategy> strategies = Arrays.asList(TopKStrategy.INSTANCE);
     // order matters, process SubGraphProject before the SinkOutput
     private static List<InterOpProcessor> processors =
             Arrays.asList(SubGraphProjectProcessor.INSTANCE, SinkOutputProcessor.INSTANCE);

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/MatchSentence.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.alibaba.graphscope.common.intermediate;
 
 import com.alibaba.graphscope.common.jna.type.FfiAlias;

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/AsNoneOp.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.alibaba.graphscope.common.intermediate.operator;
 
 // to represent identity in gremlin

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/ExpandOp.java

@@ -21,21 +21,21 @@
 public class ExpandOp extends InterOpBase {
     public ExpandOp() {
         super();
-        this.isEdge = Optional.empty();
+        this.expandOpt = Optional.empty();
         this.direction = Optional.empty();
         this.params = Optional.empty();
     }
 
-    // out or outE
-    private Optional<OpArg> isEdge;
+    // FfiExpandOpt
+    private Optional<OpArg> expandOpt;
 
     // in/out/both
     private Optional<OpArg> direction;
 
     private Optional<QueryParams> params;
 
-    public Optional<OpArg> getIsEdge() {
-        return isEdge;
+    public Optional<OpArg> getExpandOpt() {
+        return expandOpt;
     }
 
     public Optional<OpArg> getDirection() {
@@ -50,8 +50,8 @@ public void setDirection(OpArg direction) {
         this.direction = Optional.of(direction);
     }
 
-    public void setEdgeOpt(OpArg isEdge) {
-        this.isEdge = Optional.of(isEdge);
+    public void setEdgeOpt(OpArg expandOpt) {
+        this.expandOpt = Optional.of(expandOpt);
     }
 
     public void setParams(QueryParams params) {

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/MatchOp.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.alibaba.graphscope.common.intermediate.operator;
 
 import java.util.Optional;

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/PathExpandOp.java

@@ -34,8 +34,8 @@ public PathExpandOp(ExpandOp other) {
         if (other.getAlias().isPresent()) {
             setAlias(other.getAlias().get());
         }
-        if (other.getIsEdge().isPresent()) {
-            setEdgeOpt(other.getIsEdge().get());
+        if (other.getExpandOpt().isPresent()) {
+            setEdgeOpt(other.getExpandOpt().get());
         }
         if (other.getDirection().isPresent()) {
             setDirection(other.getDirection().get());

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/operator/SelectOp.java

@@ -22,11 +22,8 @@ public class SelectOp extends InterOpBase {
     public SelectOp() {
         super();
         this.predicate = Optional.empty();
-        this.type = FilterType.NONE;
     }
 
-    private FilterType type;
-
     private Optional<OpArg> predicate;
 
     public Optional<OpArg> getPredicate() {
@@ -36,18 +33,4 @@ public Optional<OpArg> getPredicate() {
     public void setPredicate(OpArg predicate) {
         this.predicate = Optional.of(predicate);
     }
-
-    public void setType(FilterType type) {
-        this.type = type;
-    }
-
-    public FilterType getType() {
-        return type;
-    }
-
-    public enum FilterType {
-        NONE,
-        HAS,
-        WHERE,
-    }
 }

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/IrCoreLibrary.java

@@ -60,7 +60,7 @@ FfiResult.ByValue orEquivPredicate(
 
     FfiResult.ByValue setScanAlias(Pointer scan, FfiAlias.ByValue alias);
 
-    Pointer initEdgexpdOperator(boolean isEdge, FfiDirection direction);
+    Pointer initEdgexpdOperator(FfiExpandOpt expandOpt, FfiDirection direction);
 
     FfiResult.ByValue appendEdgexpdOperator(
             Pointer plan, Pointer edgeXpd, int parent, IntByReference oprIdx);

interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/jna/type/FfiBinderOpt.java

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Alibaba Group Holding Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.alibaba.graphscope.common.jna.type;
 
 import com.alibaba.graphscope.common.jna.IntEnum;