Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[GAIA-IR] support more patterns of aggregate functions, i.e min/max/toList/toSet/avg/countDistinct #1847

Merged
merged 12 commits into from
Jul 28, 2022
Merged

[GAIA-IR] support more patterns of aggregate functions, i.e min/max/toList/toSet/avg/countDistinct #1847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
d29874c
[IR Compiler] support more aggregate functions sum/min/max/mean/fold/…
shirly121 Jul 13, 2022
ee37c6f
support aggregations on variables in group
shirly121 Jul 13, 2022
7284b28
Merge branch 'main' into ir_compiler_aggFn
shirly121 Jul 15, 2022
55b8349
[IR Compiler] fix bugs of count_distinct
shirly121 Jul 15, 2022
19ca15b
[IR Compiler] support group().by(..).by('str')
shirly121 Jul 15, 2022
df1d057
[IR Compiler] add comments for ExprArg
shirly121 Jul 15, 2022
a6d9c27
[IR Compiler] print query and job conf name with ir plan
shirly121 Jun 21, 2022
c491f67
Merge branch 'main' into ir_compiler_aggFn
shirly121 Jul 20, 2022
a21e980
[Gaia/IR] Add bug case of select keys tag in group by
longbinlai Jul 26, 2022
b0e2529
[GIE/IR] Fix a bug of maintaining tag id in GroupBy keys.
longbinlai Jul 27, 2022
a8ceba5
[GIE/IR] Fix a bug of referring to previous nodes in apply operator
longbinlai Jul 27, 2022
9c44a80
Merge branch 'ir_fix_group_tag' into ir_compiler_aggFn
longbinlai Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 36 additions & 2 deletions research/query_service/ir/compiler/src/main/antlr4/GremlinGS.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ traversalMethod
| traversalMethod_group // group()
| traversalMethod_groupCount // groupCount()
| traversalMethod_values // values()
| traversalMethod_count // count()
| traversalMethod_is // is()
| traversalMethod_where // where()
| traversalMethod_inV // inV()
Expand All @@ -80,6 +79,7 @@ traversalMethod
| traversalMethod_match // match()
| traversalMethod_subgraph // subgraph()
| traversalMethod_bothV // bothV()
| traversalMethod_aggregate_func
| traversalMethod_hasNot // hasNot()
;

Expand Down Expand Up @@ -267,14 +267,28 @@ traversalMethod_group_keyby
// group().by(...).by(fold().as("value"))
// group().by(...).by(count())
// group().by(...).by(count().as("value"))
// group().by(...).by("name") = group().by(...).by(values("name").fold())
// group().by(...).by(sum()/min()/max()/mean()/fold())
// group().by(...).by(select("a").count()/sum()/min()/max()/mean()/fold())
// group().by(...).by(select("a").by("name").count()/sum()/min()/max()/mean()/fold())
// group().by(...).by(select("a").values("name").count()/sum()/min()/max()/mean()/fold())
// group().by(...).by(dedup().count()) = countDistinct
// group().by(...).by(dedup().fold()) = toSet
traversalMethod_group_valueby
: 'by' LPAREN RPAREN
| 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_aggregate_func (DOT traversalMethod_as)? RPAREN
| 'by' LPAREN stringLiteral RPAREN
| 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? (traversalMethod_select DOT)? (traversalMethod_values DOT)? traversalMethod_aggregate_func (DOT traversalMethod_as)? RPAREN
| 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_dedup DOT traversalMethod_count (DOT traversalMethod_as)? RPAREN
| 'by' LPAREN (ANON_TRAVERSAL_ROOT DOT)? traversalMethod_dedup DOT traversalMethod_fold (DOT traversalMethod_as)? RPAREN
;

traversalMethod_aggregate_func
: traversalMethod_count
| traversalMethod_fold
| traversalMethod_sum
| traversalMethod_min
| traversalMethod_max
| traversalMethod_mean
;

// count in global scope
Expand All @@ -293,6 +307,26 @@ traversalMethod_fold
: 'fold' LPAREN RPAREN
;

// sum in global scope
traversalMethod_sum
: 'sum' LPAREN RPAREN
;

// min in global scope
traversalMethod_min
: 'min' LPAREN RPAREN
;

// max in global scope
traversalMethod_max
: 'max' LPAREN RPAREN
;

// mean in global scope
traversalMethod_mean
: 'mean' LPAREN RPAREN
;

// is(27)
// is(P.eq(27))
traversalMethod_is
Expand Down
8 changes: 8 additions & 0 deletions ...ervice/ir/compiler/src/main/java/com/alibaba/graphscope/common/intermediate/ArgAggFn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ public FfiAlias.ByValue getAlias() {
return alias;
}

public void setVar(FfiVariable.ByValue var) {
this.var = var;
}

public void setAggregate(FfiAggOpt aggregate) {
this.aggregate = aggregate;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down
14 changes: 9 additions & 5 deletions ...ce/ir/compiler/src/main/java/com/alibaba/graphscope/gremlin/InterOpCollectionBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,17 @@ public InterOpCollection build() throws OpArgIllegalException, UnsupportedStepEx
opList.add(StepTransformFactory.HAS_STEP.apply(step));
} else if (Utils.equalClass(step, RangeGlobalStep.class)) {
opList.add(StepTransformFactory.LIMIT_STEP.apply(step));
} else if (Utils.equalClass(step, PropertyMapStep.class)) {
opList.add(StepTransformFactory.VALUE_MAP_STEP.apply(step));
} else if (Utils.equalClass(step, DedupGlobalStep.class)) {
opList.add(StepTransformFactory.DEDUP_STEP.apply(step));
} else if (Utils.equalClass(step, CountGlobalStep.class)) {
opList.add(StepTransformFactory.COUNT_STEP.apply(step));
} else if (Utils.equalClass(step, PropertiesStep.class)) {
} else if (Utils.equalClass(step, CountGlobalStep.class)
|| Utils.equalClass(step, SumGlobalStep.class)
|| Utils.equalClass(step, MaxGlobalStep.class)
|| Utils.equalClass(step, MinGlobalStep.class)
|| Utils.equalClass(step, FoldStep.class)
|| Utils.equalClass(step, MeanGlobalStep.class)) {
opList.add(StepTransformFactory.AGGREGATE_STEP.apply(step));
} else if (Utils.equalClass(step, PropertiesStep.class)
|| Utils.equalClass(step, PropertyMapStep.class)) {
opList.add(StepTransformFactory.VALUES_STEP.apply(step));
} else if (Utils.equalClass(step, IsStep.class)) {
opList.add(StepTransformFactory.IS_STEP.apply(step));
Expand Down
88 changes: 62 additions & 26 deletions .../compiler/src/main/java/com/alibaba/graphscope/gremlin/antlr4/TraversalMethodVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,34 +464,41 @@ public Traversal visitTraversalMethod_group_keyby(
public Traversal visitTraversalMethod_group_valueby(
GremlinGSParser.TraversalMethod_group_valuebyContext ctx) {
int childCount = ctx.getChildCount();
if (childCount == 3) {
return graphTraversal.by();
} else if (childCount == 4 && ctx.traversalMethod_aggregate_func() != null) {
TraversalMethodVisitor nestedVisitor =
new TraversalMethodVisitor(
gvisitor, GremlinAntlrToJava.getTraversalSupplier().get());
Traversal nestedTraversal =
nestedVisitor.visitTraversalMethod_aggregate_func(
ctx.traversalMethod_aggregate_func());
return graphTraversal.by(nestedTraversal);
} else if (childCount >= 6 && ctx.traversalMethod_aggregate_func() != null) {
TraversalMethodVisitor nestedVisitor =
new TraversalMethodVisitor(
gvisitor, GremlinAntlrToJava.getTraversalSupplier().get());
Traversal nestedTraversal =
if (childCount == 3) return graphTraversal.by();
if (ctx.stringLiteral() != null) {
return graphTraversal.by(GenericLiteralVisitor.getStringLiteral(ctx.stringLiteral()));
}
TraversalMethodVisitor nestedVisitor =
new TraversalMethodVisitor(
gvisitor, GremlinAntlrToJava.getTraversalSupplier().get());
Traversal nestedTraversal = null;
if (ctx.traversalMethod_dedup() != null) {
nestedTraversal = nestedVisitor.visitTraversalMethod_dedup(ctx.traversalMethod_dedup());
if (ctx.traversalMethod_count() != null) {
nestedTraversal =
nestedVisitor.visitTraversalMethod_count(ctx.traversalMethod_count());
} else if (ctx.traversalMethod_fold() != null) {
nestedTraversal =
nestedVisitor.visitTraversalMethod_fold(ctx.traversalMethod_fold());
}
} else if (ctx.traversalMethod_aggregate_func() != null) {
if (ctx.traversalMethod_select() != null) {
nestedVisitor.visitTraversalMethod_select(ctx.traversalMethod_select());
}
if (ctx.traversalMethod_values() != null) {
nestedVisitor.visitTraversalMethod_values(ctx.traversalMethod_values());
}
nestedTraversal =
nestedVisitor.visitTraversalMethod_aggregate_func(
ctx.traversalMethod_aggregate_func());
if (ctx.traversalMethod_as() != null) {
nestedTraversal = nestedVisitor.visitTraversalMethod_as(ctx.traversalMethod_as());
}
return graphTraversal.by(nestedTraversal);
} else {
throw new UnsupportedEvalException(
ctx.getClass(),
"supported pattern is [group().by(..).by(count())] or"
+ " [group().by(..).by(fold())] or [group().by(..).by(count().as('..'))] or"
+ " [group().by(..).by(fold().as('..'))]");
}
if (ctx.traversalMethod_as() != null) {
nestedTraversal = nestedVisitor.visitTraversalMethod_as(ctx.traversalMethod_as());
}
if (nestedTraversal == null) {
throw new UnsupportedEvalException(ctx.getClass(), "aggregate function should exist");
}
return graphTraversal.by(nestedTraversal);
}

@Override
Expand All @@ -501,12 +508,41 @@ public Traversal visitTraversalMethod_aggregate_func(
return visitTraversalMethod_count(ctx.traversalMethod_count());
} else if (ctx.traversalMethod_fold() != null) {
return visitTraversalMethod_fold(ctx.traversalMethod_fold());
} else if (ctx.traversalMethod_sum() != null) {
return visitTraversalMethod_sum(ctx.traversalMethod_sum());
} else if (ctx.traversalMethod_min() != null) {
return visitTraversalMethod_min(ctx.traversalMethod_min());
} else if (ctx.traversalMethod_max() != null) {
return visitTraversalMethod_max(ctx.traversalMethod_max());
} else if (ctx.traversalMethod_mean() != null) {
return visitTraversalMethod_mean(ctx.traversalMethod_mean());
} else {
throw new UnsupportedEvalException(
ctx.getClass(), "supported pattern is [count()] or [fold()]");
ctx.getClass(),
"supported aggregation functions are count/sum/min/max/mean/fold");
}
}

@Override
public Traversal visitTraversalMethod_sum(GremlinGSParser.TraversalMethod_sumContext ctx) {
return graphTraversal.sum();
}

@Override
public Traversal visitTraversalMethod_min(GremlinGSParser.TraversalMethod_minContext ctx) {
return graphTraversal.min();
}

@Override
public Traversal visitTraversalMethod_max(GremlinGSParser.TraversalMethod_maxContext ctx) {
return graphTraversal.max();
}

@Override
public Traversal visitTraversalMethod_mean(GremlinGSParser.TraversalMethod_meanContext ctx) {
return graphTraversal.mean();
}

@Override
public Traversal visitTraversalMethod_count(GremlinGSParser.TraversalMethod_countContext ctx) {
return graphTraversal.count();
Expand Down
72 changes: 36 additions & 36 deletions ...piler/src/main/java/com/alibaba/graphscope/gremlin/integration/graph/RemoteTestGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,14 +157,14 @@
test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasTest",
method = "g_EX11X_outV_outE_hasXid_10AsStringX",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest",
method = "g_V_whereXinXcreatedX_count_isX1XX_valuesXnameX",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest",
method = "g_V_whereXinXcreatedX_count_isXgte_2XX_valuesXnameX",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest",
// method = "g_V_whereXinXcreatedX_count_isX1XX_valuesXnameX",
// reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsTest",
// method = "g_V_whereXinXcreatedX_count_isXgte_2XX_valuesXnameX",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeTest",
method = "g_V_outE_valuesXweightX_fold_orderXlocalX_skipXlocal_2X",
Expand Down Expand Up @@ -306,10 +306,10 @@
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest",
method = "g_V_both_both_count",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest",
method = "g_V_whereXinXkknowsX_outXcreatedX_count_is_0XX_name",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest",
// method = "g_V_whereXinXkknowsX_outXcreatedX_count_is_0XX_name",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest",
method = "g_V_hasLabelXpersonX_asXpX_VXsoftwareX_addInEXuses_pX",
Expand Down Expand Up @@ -386,10 +386,10 @@
// @Graph.OptOut(test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest",
// method = "g_V_order_byXname_incrX_name",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest",
method = "g_V_order_byXoutE_count_descX",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest",
// method = "g_V_order_byXoutE_count_descX",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest",
method = "g_V_order_byXname_a1_b1X_byXname_b2_a2X_name",
Expand Down Expand Up @@ -671,10 +671,10 @@
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
method = "g_V_hasXlangX_groupXaX_byXlangX_byXnameX_out_capXaX",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
method = "g_V_group_byXoutE_countX_byXnameX",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
// method = "g_V_group_byXoutE_countX_byXnameX",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest",
method = "g_V_groupXaX_byXlabelX_byXoutE_weight_sumX_capXaX",
Expand All @@ -689,18 +689,18 @@
method =
"g_V_unionXrepeatXoutX_timesX2X_groupCountXmX_byXlangXX__repeatXinX_timesX2X_groupCountXmX_byXnameXX_capXmX",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_groupCount_byXbothE_countX",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
// method = "g_V_groupCount_byXbothE_countX",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_repeatXout_groupCountXaX_byXnameXX_timesX2X_capXaX",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_outXcreatedX_groupCount_byXnameX",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
// method = "g_V_outXcreatedX_groupCount_byXnameX",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_outXcreatedX_groupCountXxX_capXxX",
Expand All @@ -709,10 +709,10 @@
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_both_groupCountXaX_out_capXaX_selectXkeysX_unfold_both_groupCountXaX_capXaX",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_outXcreatedX_name_groupCount",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
// method = "g_V_outXcreatedX_name_groupCount",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest",
method = "g_V_outXcreatedX_groupCountXaX_byXnameX_capXaX",
Expand Down Expand Up @@ -828,10 +828,10 @@
test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
method = "g_V_chooseXlabel_is_person__unionX__out_lang__out_nameX__in_labelX_groupCount",
reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
method = "g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX",
reason = "unsupported")
// @Graph.OptOut(
// test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
// method = "g_VX1_2X_unionXoutE_count__inE_count__outE_weight_sumX",
// reason = "unsupported")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.branch.UnionTest",
method =
Expand Down
3 changes: 2 additions & 1 deletion ...src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ public ThrowingConsumer<Context> select(Context ctx) {
new GremlinTestResultProcessor(
ctx,
GremlinResultAnalyzer.analyze(traversal),
testGraph));
testGraph),
script);
});
return op;
default:
Expand Down
Loading