Skip to content

Commit

Permalink
Plan CONTAINS and ENDS WITH as ordered
Browse files Browse the repository at this point in the history
This relies on the fact that the native indexes (which are the only
indexes that support order) can only serve contains and ends-with queries
using a full scan and filter. If we somehow arrive at a new index
implementation where this is not the case, the index capabitlity interface
has to be enriched to answer ordering questions for specific queries.
  • Loading branch information
fickludd committed Sep 18, 2018
1 parent 02e3562 commit 4c5a73a
Show file tree
Hide file tree
Showing 18 changed files with 93 additions and 72 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ case class NodeIndexContainsScan(idName: String,
label: LabelToken,
property: IndexedProperty,
valueExpr: Expression,
argumentIds: Set[String])
argumentIds: Set[String],
indexOrder: IndexOrder)
(implicit idGen: IdGen)
extends IndexLeafPlan(idGen) {

Expand All @@ -43,5 +44,5 @@ case class NodeIndexContainsScan(idName: String,
override def availableCachedNodeProperties: Map[Property, CachedNodeProperty] = property.asAvailablePropertyMap(idName)

override def copyWithoutGettingValues: NodeIndexContainsScan =
NodeIndexContainsScan(idName, label, IndexedProperty(property.propertyKeyToken, DoNotGetValue), valueExpr, argumentIds)(SameId(this.id))
NodeIndexContainsScan(idName, label, IndexedProperty(property.propertyKeyToken, DoNotGetValue), valueExpr, argumentIds, indexOrder)(SameId(this.id))
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ case class NodeIndexEndsWithScan(idName: String,
label: LabelToken,
property: IndexedProperty,
valueExpr: Expression,
argumentIds: Set[String])
argumentIds: Set[String],
indexOrder: IndexOrder)
(implicit idGen: IdGen)
extends IndexLeafPlan(idGen) {

Expand All @@ -43,5 +44,5 @@ case class NodeIndexEndsWithScan(idName: String,
override def availableCachedNodeProperties: Map[Property, CachedNodeProperty] = property.asAvailablePropertyMap(idName)

override def copyWithoutGettingValues: NodeIndexEndsWithScan =
NodeIndexEndsWithScan(idName, label, IndexedProperty(property.propertyKeyToken, DoNotGetValue), valueExpr, argumentIds)(SameId(this.id))
NodeIndexEndsWithScan(idName, label, IndexedProperty(property.propertyKeyToken, DoNotGetValue), valueExpr, argumentIds, indexOrder)(SameId(this.id))
}
Original file line number Diff line number Diff line change
Expand Up @@ -254,14 +254,15 @@ case class LogicalPlanProducer(cardinalityModel: CardinalityModel, planningAttri
solvedHint: Option[UsingIndexHint],
valueExpr: Expression,
argumentIds: Set[String],
providedOrder: ProvidedOrder,
context: LogicalPlanningContext): LogicalPlan = {
val solved = RegularPlannerQuery(queryGraph = QueryGraph.empty
.addPatternNodes(idName)
.addPredicates(solvedPredicates: _*)
.addHints(solvedHint)
.addArgumentIds(argumentIds.toIndexedSeq)
)
annotate(NodeIndexContainsScan(idName, label, property, valueExpr, argumentIds), solved, ProvidedOrder.empty, context)
annotate(NodeIndexContainsScan(idName, label, property, valueExpr, argumentIds, toIndexOrder(providedOrder)), solved, providedOrder, context)
}

def planNodeIndexEndsWithScan(idName: String,
Expand All @@ -271,14 +272,15 @@ case class LogicalPlanProducer(cardinalityModel: CardinalityModel, planningAttri
solvedHint: Option[UsingIndexHint],
valueExpr: Expression,
argumentIds: Set[String],
providedOrder: ProvidedOrder,
context: LogicalPlanningContext): LogicalPlan = {
val solved = RegularPlannerQuery(queryGraph = QueryGraph.empty
.addPatternNodes(idName)
.addPredicates(solvedPredicates: _*)
.addHints(solvedHint)
.addArgumentIds(argumentIds.toIndexedSeq)
)
annotate(NodeIndexEndsWithScan(idName, label, property, valueExpr, argumentIds), solved, ProvidedOrder.empty, context)
annotate(NodeIndexEndsWithScan(idName, label, property, valueExpr, argumentIds, toIndexOrder(providedOrder)), solved, providedOrder, context)
}

def planNodeHashJoin(nodes: Set[String], left: LogicalPlan, right: LogicalPlan, hints: Seq[UsingJoinHint], context: LogicalPlanningContext): LogicalPlan = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,11 @@ case class alignGetValueFromIndexBehavior(query: PlannerQuery, lpp: LogicalPlanP

case x: NodeIndexContainsScan =>
val alignedProperty = withAlignedGetValueBehavior(x.idName, x.property)
NodeIndexContainsScan(x.idName, x.label, alignedProperty, x.valueExpr, x.argumentIds)(attributes.copy(x.id))
NodeIndexContainsScan(x.idName, x.label, alignedProperty, x.valueExpr, x.argumentIds, x.indexOrder)(attributes.copy(x.id))

case x: NodeIndexEndsWithScan =>
val alignedProperty = withAlignedGetValueBehavior(x.idName, x.property)
NodeIndexEndsWithScan(x.idName, x.label, alignedProperty, x.valueExpr, x.argumentIds)(attributes.copy(x.id))
NodeIndexEndsWithScan(x.idName, x.label, alignedProperty, x.valueExpr, x.argumentIds, x.indexOrder)(attributes.copy(x.id))

case x: NodeIndexScan =>
val alignedProperty = withAlignedGetValueBehavior(x.idName, x.property)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,13 @@ object indexScanLeafPlanner extends LeafPlanner with LeafPlanFromExpression {
// MATCH (n:User) WHERE n.prop CONTAINS 'substring' RETURN n
case predicate@Contains(prop@Property(Variable(name), propertyKey), expr) =>
val plans = produce(name, propertyKey.name, qg, requiredOrder, prop, CTString, predicate,
// We discard the provided order since `CONTAINS` can never provide order regardless of index capability
(name, label, prop, pred, hints, args, _) => lpp.planNodeIndexContainsScan(name, label, prop, pred, hints, expr, args, context), context)
lpp.planNodeIndexContainsScan(_, _, _, _, _, expr, _, _, context), context)
maybeLeafPlans(name, plans)

// MATCH (n:User) WHERE n.prop ENDS WITH 'substring' RETURN n
case predicate@EndsWith(prop@Property(Variable(name), propertyKey), expr) =>
val plans = produce(name, propertyKey.name, qg, requiredOrder, prop, CTString, predicate,
// We discard the provided order since `ENDS WITH` can never provide order regardless of index capability
(name, label, prop, pred, hints, args, _) => lpp.planNodeIndexEndsWithScan(name, label, prop, pred, hints, expr, args, context), context)
lpp.planNodeIndexEndsWithScan(_, _, _, _, _, expr, _, _, context), context)
maybeLeafPlans(name, plans)

// MATCH (n:User) WHERE exists(n.prop) RETURN n
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,44 +189,44 @@ class IndexWithProvidedOrderPlanningIntegrationTest extends CypherFunSuite with
)
}

test("Order by index backed property should NOT plan with provided order (contains scan)") {
// This is supported because internally all kernel indexes which support ordering will just scan and filter to serve contains
test("Order by index backed property should plan with provided order (contains scan)") {
val plan = new given {
indexOn("Awesome", "prop").providesOrder(AscIndexOrder)
} getLogicalPlanFor "MATCH (n:Awesome) WHERE n.prop CONTAINS 'foo' RETURN n.prop ORDER BY n.prop"

plan._2 should equal(
Projection(
Sort(
Projection(
NodeIndexContainsScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty),
Projection(
NodeIndexContainsScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty,
IndexOrderAscending),
Map(" FRESHID55" -> Property(Variable("n")(pos), PropertyKeyName("prop")(pos))(pos))),
Seq(Ascending(" FRESHID55"))),
Map("n.prop" -> Variable(" FRESHID55")(pos)))
)
}

test("Order by index backed property should NOT plan with provided order (ends with scan)") {
// This is supported because internally all kernel indexes which support ordering will just scan and filter to serve ends with
test("Order by index backed property should plan with provided order (ends with scan)") {
val plan = new given {
indexOn("Awesome", "prop").providesOrder(AscIndexOrder)
} getLogicalPlanFor "MATCH (n:Awesome) WHERE n.prop ENDS WITH 'foo' RETURN n.prop ORDER BY n.prop"

plan._2 should equal(
Projection(
Sort(
Projection(
NodeIndexEndsWithScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty),
Map(" FRESHID56" -> Property(Variable("n")(pos), PropertyKeyName("prop")(pos))(pos))),
Seq(Ascending(" FRESHID56"))),
Projection(
NodeIndexEndsWithScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty,
IndexOrderAscending),
Map(" FRESHID56" -> Property(Variable("n")(pos), PropertyKeyName("prop")(pos))(pos))),
Map("n.prop" -> Variable(" FRESHID56")(pos)))
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,8 @@ class IndexWithValuesPlanningIntegrationTest extends CypherFunSuite with Logical
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), GetValue),
StringLiteral("foo")(pos),
Set.empty),
Set.empty,
IndexOrderNone),
Map(cachedNodePropertyProj("n", "prop"))
)
)
Expand All @@ -788,7 +789,8 @@ class IndexWithValuesPlanningIntegrationTest extends CypherFunSuite with Logical
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty),
Set.empty,
IndexOrderNone),
Map(propertyProj("n", "prop")))
)
}
Expand All @@ -806,7 +808,8 @@ class IndexWithValuesPlanningIntegrationTest extends CypherFunSuite with Logical
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty),
Set.empty,
IndexOrderNone),
Map(propertyProj("n", "foo")))
)
}
Expand All @@ -825,7 +828,8 @@ class IndexWithValuesPlanningIntegrationTest extends CypherFunSuite with Logical
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), GetValue),
StringLiteral("foo")(pos),
Set.empty),
Set.empty,
IndexOrderNone),
Map(cachedNodePropertyProj("n", "prop"))
)
)
Expand All @@ -843,7 +847,8 @@ class IndexWithValuesPlanningIntegrationTest extends CypherFunSuite with Logical
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty),
Set.empty,
IndexOrderNone),
Map(propertyProj("n", "prop")))
)
}
Expand All @@ -861,7 +866,8 @@ class IndexWithValuesPlanningIntegrationTest extends CypherFunSuite with Logical
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("foo")(pos),
Set.empty),
Set.empty,
IndexOrderNone),
Map(propertyProj("n", "foo")))
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ class LeafPlanningIntegrationTest extends CypherFunSuite with LogicalPlanningTes
LabelToken("Person", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("name") _, PropertyKeyId(0)), DoNotGetValue),
StringLiteral("substring")_,
Set.empty)
Set.empty,
IndexOrderNone)
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, DoNotGetValue), `stringLiteral`, _)) => ()
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, DoNotGetValue), `stringLiteral`, _, _)) => ()
}
}
}
Expand All @@ -394,7 +394,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, CanGetValue), `stringLiteral`, _)) => ()
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, CanGetValue), `stringLiteral`, _, _)) => ()
}
}
}
Expand All @@ -410,7 +410,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, DoNotGetValue), `stringLiteral`, _)) => ()
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, DoNotGetValue), `stringLiteral`, _, _)) => ()
}
}
}
Expand All @@ -426,7 +426,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, CanGetValue), `stringLiteral`, _)) => ()
case Seq(NodeIndexContainsScan(`idName`, _, IndexedProperty(_, CanGetValue), `stringLiteral`, _, _)) => ()
}
}
}
Expand All @@ -444,7 +444,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexContainsScan(`idName`, _, _, `stringLiteral`, _)) => ()
case Seq(NodeIndexContainsScan(`idName`, _, _, `stringLiteral`, _, _)) => ()
}

resultPlans.map(p => ctx.planningAttributes.solveds.get(p.id).queryGraph) should beLike {
Expand All @@ -466,7 +466,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexContainsScan(`idName`, _, _, `stringLiteral`, _)) => ()
case Seq(NodeIndexContainsScan(`idName`, _, _, `stringLiteral`, _, _)) => ()
}

resultPlans.map(p => ctx.planningAttributes.solveds.get(p.id).queryGraph) should beLike {
Expand All @@ -486,7 +486,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexEndsWithScan(`idName`, _, IndexedProperty(_, DoNotGetValue), `stringLiteral`, _)) => ()
case Seq(NodeIndexEndsWithScan(`idName`, _, IndexedProperty(_, DoNotGetValue), `stringLiteral`, _, _)) => ()
}
}
}
Expand All @@ -502,7 +502,7 @@ class IndexScanLeafPlannerTest extends CypherFunSuite with LogicalPlanningTestSu

// then
resultPlans should beLike {
case Seq(NodeIndexEndsWithScan(`idName`, _, IndexedProperty(_, CanGetValue), `stringLiteral`, _)) => ()
case Seq(NodeIndexEndsWithScan(`idName`, _, IndexedProperty(_, CanGetValue), `stringLiteral`, _, _)) => ()
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,30 +36,35 @@ class AlignGetValueFromIndexBehaviorTest extends CypherFunSuite with LogicalPlan
LabelToken("Awesome", LabelId(0)),
Seq(IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), getValue)),
SingleQueryExpression(SignedDecimalIntegerLiteral("42") _),
Set.empty, IndexOrderNone)
Set.empty,
IndexOrderNone)
val uniqueIndexSeek: IndexOperator = getValue => NodeUniqueIndexSeek(
"n",
LabelToken("Awesome", LabelId(0)),
Seq(IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), getValue)),
SingleQueryExpression(SignedDecimalIntegerLiteral("42") _),
Set.empty, IndexOrderNone)
Set.empty,
IndexOrderNone)
val indexContainsScan: IndexOperator = getValue => NodeIndexContainsScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), getValue),
StringLiteral("foo")(pos),
Set.empty)
Set.empty,
IndexOrderNone)
val indexEndsWithScan: IndexOperator = getValue => NodeIndexEndsWithScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), getValue),
StringLiteral("foo")(pos),
Set.empty)
Set.empty,
IndexOrderNone)
val indexScan: IndexOperator = getValue => NodeIndexScan(
"n",
LabelToken("Awesome", LabelId(0)),
IndexedProperty(PropertyKeyToken(PropertyKeyName("prop") _, PropertyKeyId(0)), getValue),
Set.empty, IndexOrderNone)
Set.empty,
IndexOrderNone)

val indexOperators = Seq(indexSeek, uniqueIndexSeek, indexContainsScan, indexEndsWithScan, indexScan)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,8 @@ object LogicalPlanConverter {
children(1).asInstanceOf[expressionsv3_5.LabelToken],
IndexedProperty(children(2).asInstanceOf[expressionsv3_5.PropertyKeyToken], DoNotGetValue),
children(3).asInstanceOf[Expressionv3_5],
children(4).asInstanceOf[Set[String]]
children(4).asInstanceOf[Set[String]],
IndexOrderNone
)(ids.convertId(plan))

case (plan: plansV3_4.NodeIndexEndsWithScan, children: Seq[AnyRef]) =>
Expand All @@ -170,7 +171,8 @@ object LogicalPlanConverter {
children(1).asInstanceOf[expressionsv3_5.LabelToken],
IndexedProperty(children(2).asInstanceOf[expressionsv3_5.PropertyKeyToken], DoNotGetValue),
children(3).asInstanceOf[Expressionv3_5],
children(4).asInstanceOf[Set[String]]
children(4).asInstanceOf[Set[String]],
IndexOrderNone
)(ids.convertId(plan))

case (plan: plansV3_4.NodeIndexScan, children: Seq[AnyRef]) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,13 @@ case class checkForIndexLimitation(planContext: PlanContext) extends Notificatio
def apply(plan: LogicalPlan): Set[InternalNotification] = {

plan.treeFold[Set[InternalNotification]](Set.empty) {
case NodeIndexContainsScan(_, label, property, _, _) =>
case NodeIndexContainsScan(_, label, property, _, _, _) =>
acc =>
val notifications = getLimitations(label, property.propertyKeyToken).collect {
case SlowContains => SuboptimalIndexForConstainsQueryNotification(label.name, Seq(property.propertyKeyToken.name))
}
(acc ++ notifications, None)
case NodeIndexEndsWithScan(_, label, property, _, _) =>
case NodeIndexEndsWithScan(_, label, property, _, _, _) =>
acc =>
val notifications = getLimitations(label, property.propertyKeyToken).collect {
case SlowContains => SuboptimalIndexForEndsWithQueryNotification(label.name, Seq(property.propertyKeyToken.name))
Expand Down

0 comments on commit 4c5a73a

Please sign in to comment.