Permalink
Browse files

Improve subquery fusion for Union operations

- Add a new phase `reorderOperations` to push mapping Binds down into
  Unions before merging to Comprehensions. It is overly conservative
  and skips any mapping that could potentially be hoisted to the
  client side later (and even when this happens, an unnecessary
  subquery remains). A better solution would require `hoistClientOps`
  to operate before this phase on an unmerged tree.

- Remove `retype` flag from `infer` and `withInferredType` (no longer
  used). If you really need to retype, created an untyped copy first
  (extra traversal but no extra copying needed).

- Fix typing bugs in `hoistClientOps`.

- Remove unnecessary Symbols from `Union`.

- Allow top-level `Union` nodes in `QueryBuilder` and `hoistClientOps`.

- Improve error reporting in `VerifyTypes`.

- Change the handling of `CollectionCast` nodes in `CodeGen` and
  `hoistClientOps` to allow CollectionCast at the top level.
  • Loading branch information...
szeiger committed Jul 30, 2015
1 parent 6b783d1 commit ffee22c45c3e050f1de19bf9af9cec521cb8b01e
@@ -28,6 +28,7 @@
<logger name="slick.compiler.RewriteJoins" level="${log.qcomp.rewriteJoins:-inherited}" />
<logger name="slick.compiler.RemoveTakeDrop" level="${log.qcomp.removeTakeDrop:-inherited}" />
<logger name="slick.compiler.ResolveZipJoins" level="${log.qcomp.resolveZipJoins:-inherited}" />
<logger name="slick.compiler.ReorderOperations" level="${log.qcomp.reorderOperations:-inherited}" />
<logger name="slick.compiler.MergeToComprehensions" level="${log.qcomp.mergeToComprehensions:-inherited}" />
<logger name="slick.compiler.FixRowNumberOrdering" level="${log.qcomp.fixRowNumberOrdering:-inherited}" />
<logger name="slick.compiler.HoistClientOps" level="${log.qcomp.hoistClientOps:-inherited}" />
@@ -28,6 +28,7 @@
<logger name="slick.compiler.RewriteJoins" level="${log.qcomp.rewriteJoins:-inherited}" />
<logger name="slick.compiler.RemoveTakeDrop" level="${log.qcomp.removeTakeDrop:-inherited}" />
<logger name="slick.compiler.ResolveZipJoins" level="${log.qcomp.resolveZipJoins:-inherited}" />
<logger name="slick.compiler.ReorderOperations" level="${log.qcomp.reorderOperations:-inherited}" />
<logger name="slick.compiler.MergeToComprehensions" level="${log.qcomp.mergeToComprehensions:-inherited}" />
<logger name="slick.compiler.FixRowNumberOrdering" level="${log.qcomp.fixRowNumberOrdering:-inherited}" />
<logger name="slick.compiler.HoistClientOps" level="${log.qcomp.hoistClientOps:-inherited}" />
@@ -42,7 +42,7 @@ class JoinTest extends AsyncTest[RelationalTestDB] {
c <- categories
p <- posts if c.id === p.category
} yield (p.id, c.id, c.name, p.title)).sortBy(_._1)
_ <- q1.map(p => (p._1, p._2)).result.map(_ shouldBe List((2,1), (3,2), (4,3), (5,2)))
_ <- mark("q1", q1.map(p => (p._1, p._2)).result).map(_ shouldBe List((2,1), (3,2), (4,3), (5,2)))
// Explicit inner join
q2 = (for {
(c,p) <- categories join posts on (_.id === _.category)
@@ -498,8 +498,9 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
val q11e = q10.drop(7)
val q11f = q10.take(6).drop(2).filter(_ =!= 5)
val q12 = as.filter(_.id <= as.map(_.id).max-1).map(_.a)
//val q5 = q1.length
val q13 = (as.filter(_.id < 2) union as.filter(_.id > 2)).map(_.id)
val q14 = q13.to[Set]
val q15 = (as.map(a => a.id.?).filter(_ < 2) unionAll as.map(a => a.id.?).filter(_ > 2)).map(_.get).to[Set]
if(tdb.driver == H2Driver) {
assertNesting(q1, 1)
@@ -521,6 +522,10 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
assertNesting(q11d, 1)
assertNesting(q11e, 1)
assertNesting(q11f, 2)
assertNesting(q12, 2)
assertNesting(q13, 2)
assertNesting(q14, 2)
//assertNesting(q15, 2) //TODO
}
for {
@@ -547,6 +552,9 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
_ <- mark("q11e", q11e.result).map(_ shouldBe Seq(8, 9))
_ <- mark("q11f", q11f.result).map(_ shouldBe Seq(3, 4, 6))
_ <- mark("q12", q12.result).map(_ shouldBe Seq("a", "a"))
_ <- mark("q13", q13.result).map(_.toSet shouldBe Set(1, 3))
_ <- mark("q14", q14.result).map(_ shouldBe Set(1, 3))
_ <- mark("q15", q15.result).map(_ shouldBe Set(1, 3))
} yield ()
}
@@ -49,17 +49,17 @@ final case class ResultSetMapping(generator: TermSymbol, from: Node, map: Node)
def generators = Seq((generator, from))
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(generator = gen(0))
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val from2 = from.infer(scope, typeChildren, retype)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val from2 = from.infer(scope, typeChildren)
val (map2, newType) = from2.nodeType match {
case CollectionType(cons, elem) =>
val map2 = map.infer(scope + (generator -> elem), typeChildren, retype)
val map2 = map.infer(scope + (generator -> elem), typeChildren)
(map2, CollectionType(cons, map2.nodeType))
case t =>
val map2 = map.infer(scope + (generator -> t), typeChildren, retype)
val map2 = map.infer(scope + (generator -> t), typeChildren)
(map2, map2.nodeType)
}
withChildren(Vector(from2, map2)) :@ (if(!hasType || retype) newType else nodeType)
withChildren(Vector(from2, map2)) :@ (if(!hasType) newType else nodeType)
}
def nodeMapServerSide(keepType: Boolean, r: Node => Node) = {
val this2 = mapScopedChildren {
@@ -47,23 +47,23 @@ final case class Comprehension(sym: TermSymbol, from: Node, select: Node, where:
def generators = Seq((sym, from))
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(sym = gen.head)
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
// Assign type to "from" Node and compute the resulting scope
val f2 = from.infer(scope, typeChildren, retype)
val f2 = from.infer(scope, typeChildren)
val genScope = scope + (sym -> f2.nodeType.asCollectionType.elementType)
// Assign types to "select", "where", "groupBy", "orderBy", "having", "fetch" and "offset" Nodes
val s2 = select.infer(genScope, typeChildren, retype)
val w2 = mapOrNone(where)(_.infer(genScope, typeChildren, retype))
val g2 = mapOrNone(groupBy)(_.infer(genScope, typeChildren, retype))
val s2 = select.infer(genScope, typeChildren)
val w2 = mapOrNone(where)(_.infer(genScope, typeChildren))
val g2 = mapOrNone(groupBy)(_.infer(genScope, typeChildren))
val o = orderBy.map(_._1)
val o2 = mapOrNone(o)(_.infer(genScope, typeChildren, retype))
val h2 = mapOrNone(having)(_.infer(genScope, typeChildren, retype))
val fetch2 = mapOrNone(fetch)(_.infer(genScope, typeChildren, retype))
val offset2 = mapOrNone(offset)(_.infer(genScope, typeChildren, retype))
val o2 = mapOrNone(o)(_.infer(genScope, typeChildren))
val h2 = mapOrNone(having)(_.infer(genScope, typeChildren))
val fetch2 = mapOrNone(fetch)(_.infer(genScope, typeChildren))
val offset2 = mapOrNone(offset)(_.infer(genScope, typeChildren))
// Check if the nodes changed
val same = (f2 eq from) && (s2 eq select) && w2.isEmpty && g2.isEmpty && o2.isEmpty && h2.isEmpty && fetch2.isEmpty && offset2.isEmpty
val newType =
if(!hasType || retype) CollectionType(f2.nodeType.asCollectionType.cons, s2.nodeType.asCollectionType.elementType)
if(!hasType) CollectionType(f2.nodeType.asCollectionType.cons, s2.nodeType.asCollectionType.elementType)
else nodeType
if(same && newType == nodeType) this else {
copy(
@@ -9,10 +9,10 @@ final case class Insert(tableSym: TermSymbol, table: Node, linear: Node) extends
def generators = Vector((tableSym, table))
def rebuild(l: Node, r: Node) = copy(table = l, linear = r)
def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(tableSym = gen(0))
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val table2 = table.infer(scope, typeChildren, retype)
val lin2 = linear.infer(scope + (tableSym -> table2.nodeType), typeChildren, retype)
withChildren(Vector(table2, lin2)) :@ (if(!hasType || retype) lin2.nodeType else nodeType)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val table2 = table.infer(scope, typeChildren)
val lin2 = linear.infer(scope + (tableSym -> table2.nodeType), typeChildren)
withChildren(Vector(table2, lin2)) :@ (if(!hasType) lin2.nodeType else nodeType)
}
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
}
@@ -66,10 +66,10 @@ trait Node extends Dumpable {
* the children are only type-checked again if ``typeChildren`` is true. if ``retype`` is also
* true, the existing type of this node is replaced. If this node does not yet have a type, the
* types of all children are computed first. */
final def infer(scope: Type.Scope = Map.empty, typeChildren: Boolean = false, retype: Boolean = false): Self =
if(hasType && !typeChildren) this else withInferredType(scope, typeChildren, retype)
final def infer(scope: Type.Scope = Map.empty, typeChildren: Boolean = false): Self =
if(hasType && !typeChildren) this else withInferredType(scope, typeChildren)
protected[this] def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self
protected[this] def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self
def getDumpInfo = {
val (objName, mainInfo) = this match {
@@ -99,9 +99,9 @@ trait SimplyTypedNode extends Node {
protected def buildType: Type
final def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val this2: Self = mapChildren(_.infer(scope, typeChildren, retype), !retype)
if(!hasType || retype) (this2 :@ this2.buildType).asInstanceOf[Self] else this2
final def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val this2: Self = mapChildren(_.infer(scope, typeChildren), keepType = true)
if(!hasType) (this2 :@ this2.buildType).asInstanceOf[Self] else this2
}
}
@@ -236,13 +236,13 @@ abstract class FilteredQuery extends Node {
case _ => ""
})
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val from2 = from.infer(scope, typeChildren, retype)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val from2 = from.infer(scope, typeChildren)
val genScope = scope + (generator -> from2.nodeType.asCollectionType.elementType)
val ch2: IndexedSeq[Node] = children.map { ch =>
if(ch eq from) from2 else ch.infer(genScope, typeChildren, retype)
if(ch eq from) from2 else ch.infer(genScope, typeChildren)
}(collection.breakOut)
(withChildren(ch2) :@ (if(!hasType || retype) ch2.head.nodeType else nodeType)).asInstanceOf[Self]
(withChildren(ch2) :@ (if(!hasType) ch2.head.nodeType else nodeType)).asInstanceOf[Self]
}
}
@@ -304,12 +304,12 @@ final case class GroupBy(fromGen: TermSymbol, from: Node, by: Node, identity: Ty
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(fromGen = gen(0))
def generators = Seq((fromGen, from))
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = identity.toString)
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val from2 = from.infer(scope, typeChildren, retype)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val from2 = from.infer(scope, typeChildren)
val from2Type = from2.nodeType.asCollectionType
val by2 = by.infer(scope + (fromGen -> from2Type.elementType), typeChildren, retype)
val by2 = by.infer(scope + (fromGen -> from2Type.elementType), typeChildren)
withChildren(Vector(from2, by2)) :@ (
if(!hasType || retype)
if(!hasType)
CollectionType(from2Type.cons, ProductType(IndexedSeq(NominalType(identity, by2.nodeType), CollectionType(TypedCollectionTypeConstructor.seq, from2Type.elementType))))
else nodeType)
}
@@ -352,33 +352,31 @@ final case class Join(leftGen: TermSymbol, rightGen: TermSymbol, left: Node, rig
def generators = Seq((leftGen, left), (rightGen, right))
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) =
copy(leftGen = gen(0), rightGen = gen(1))
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val left2 = left.infer(scope, typeChildren, retype)
val right2 = right.infer(scope, typeChildren, retype)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val left2 = left.infer(scope, typeChildren)
val right2 = right.infer(scope, typeChildren)
val left2Type = left2.nodeType.asCollectionType
val right2Type = right2.nodeType.asCollectionType
val on2 = on.infer(scope + (leftGen -> left2Type.elementType) + (rightGen -> right2Type.elementType), typeChildren, retype)
val on2 = on.infer(scope + (leftGen -> left2Type.elementType) + (rightGen -> right2Type.elementType), typeChildren)
val (joinedLeftType, joinedRightType) = jt match {
case JoinType.LeftOption => (left2Type.elementType, OptionType(right2Type.elementType))
case JoinType.RightOption => (OptionType(left2Type.elementType), right2Type.elementType)
case JoinType.OuterOption => (OptionType(left2Type.elementType), OptionType(right2Type.elementType))
case _ => (left2Type.elementType, right2Type.elementType)
}
withChildren(Vector(left2, right2, on2)) :@ (
if(!hasType || retype) CollectionType(left2Type.cons, ProductType(IndexedSeq(joinedLeftType, joinedRightType)))
if(!hasType) CollectionType(left2Type.cons, ProductType(IndexedSeq(joinedLeftType, joinedRightType)))
else nodeType)
}
}
/** A union of type
* (CollectionType(c, t), CollectionType(_, t)) => CollectionType(c, t). */
final case class Union(left: Node, right: Node, all: Boolean, leftGen: TermSymbol = new AnonSymbol, rightGen: TermSymbol = new AnonSymbol) extends BinaryNode with DefNode with SimplyTypedNode {
final case class Union(left: Node, right: Node, all: Boolean) extends BinaryNode with SimplyTypedNode {
type Self = Union
protected[this] def rebuild(left: Node, right: Node) = copy(left = left, right = right)
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = if(all) "all" else "")
override def childNames = Seq("left "+leftGen, "right "+rightGen)
def generators = Seq((leftGen, left), (rightGen, right))
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(leftGen = gen(0), rightGen = gen(1))
override def childNames = Seq("left", "right")
protected def buildType = left.nodeType
}
@@ -393,13 +391,12 @@ final case class Bind(generator: TermSymbol, from: Node, select: Node) extends B
def generators = Seq((generator, from))
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(generator = gen(0))
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val from2 = from.infer(scope, typeChildren, retype)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val from2 = from.infer(scope, typeChildren)
val from2Type = from2.nodeType.asCollectionType
val select2 = select.infer(scope + (generator -> from2Type.elementType), typeChildren, retype)
val select2 = select.infer(scope + (generator -> from2Type.elementType), typeChildren)
withChildren(Vector(from2, select2)) :@ (
if(!hasType || retype)
CollectionType(from2Type.cons, select2.nodeType.asCollectionType.elementType)
if(!hasType) CollectionType(from2Type.cons, select2.nodeType.asCollectionType.elementType)
else nodeType)
}
}
@@ -416,10 +413,10 @@ final case class Aggregate(sym: TermSymbol, from: Node, select: Node) extends Bi
def generators = Seq((sym, from))
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(sym = gen(0))
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val from2 :@ CollectionType(_, el) = from.infer(scope, typeChildren, retype)
val select2 = select.infer(scope + (sym -> el), typeChildren, retype)
withChildren(Vector(from2, select2)) :@ (if(!hasType || retype) select2.nodeType else nodeType)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val from2 :@ CollectionType(_, el) = from.infer(scope, typeChildren)
val select2 = select.infer(scope + (sym -> el), typeChildren)
withChildren(Vector(from2, select2)) :@ (if(!hasType) select2.nodeType else nodeType)
}
}
@@ -433,10 +430,10 @@ final case class TableExpansion(generator: TermSymbol, table: Node, columns: Nod
def generators = Seq((generator, table))
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(generator = gen(0))
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self = {
val table2 = table.infer(scope, typeChildren, retype)
val columns2 = columns.infer(scope + (generator -> table2.nodeType.asCollectionType.elementType), typeChildren, retype)
withChildren(Vector(table2, columns2)) :@ (if(!hasType || retype) table2.nodeType else nodeType)
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self = {
val table2 = table.infer(scope, typeChildren)
val columns2 = columns.infer(scope + (generator -> table2.nodeType.asCollectionType.elementType), typeChildren)
withChildren(Vector(table2, columns2)) :@ (if(!hasType) table2.nodeType else nodeType)
}
}
@@ -463,8 +460,8 @@ final case class Apply(sym: TermSymbol, children: Seq[Node])(val buildType: Type
/** A reference to a Symbol */
final case class Ref(sym: TermSymbol) extends NullaryNode {
type Self = Ref
def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean): Self =
if(hasType && !retype) this else {
def withInferredType(scope: Type.Scope, typeChildren: Boolean): Self =
if(hasType) this else {
scope.get(sym) match {
case Some(t) => if(t == nodeType) this else copy() :@ t
case _ => throw new SlickException("No type for symbol "+sym+" found for "+this)
@@ -579,12 +576,12 @@ final case class OptionFold(from: Node, ifEmpty: Node, map: Node, gen: TermSymbo
override def childNames = IndexedSeq("from "+gen, "ifEmpty", "map")
protected[this] def rebuild(ch: IndexedSeq[Node]) = copy(ch(0), ch(1), ch(2))
protected[this] def rebuildWithSymbols(gen: IndexedSeq[TermSymbol]) = copy(gen = gen(0))
protected[this] def withInferredType(scope: Type.Scope, typeChildren: Boolean, retype: Boolean) = {
val from2 = from.infer(scope, typeChildren, retype)
val ifEmpty2 = ifEmpty.infer(scope, typeChildren, retype)
protected[this] def withInferredType(scope: Type.Scope, typeChildren: Boolean) = {
val from2 = from.infer(scope, typeChildren)
val ifEmpty2 = ifEmpty.infer(scope, typeChildren)
val genScope = scope + (gen -> from2.nodeType.structural.asOptionType.elementType)
val map2 = map.infer(genScope, typeChildren, retype)
withChildren(IndexedSeq(from2, ifEmpty2, map2)) :@ (if(!hasType || retype) map2.nodeType else nodeType)
val map2 = map.infer(genScope, typeChildren)
withChildren(IndexedSeq(from2, ifEmpty2, map2)) :@ (if(!hasType) map2.nodeType else nodeType)
}
override def getDumpInfo = super.getDumpInfo.copy(mainInfo = "")
}
Oops, something went wrong.

0 comments on commit ffee22c

Please sign in to comment.