Early transformation of monadic joins into applicative joins.

- A new compiler phase `rewriteJoins`, which runs directly after
  `flattenProjections`, converts all monadic joins into applicative
  joins. In its current state it is good enough to pass all tests.
  The implementation is much simpler than the old one in
  `fuseComprehensions` (the AST is still purely functional at this
  point) and should be easy to extend to cover more cases.

- A new compiler phase `verifySymbols` runs after `rewriteJoins` to
  check that all joins have been properly rewritten and that all
  references are reachable. When a join could not be transformed, this
  will fail with a useful error message instead of producing invalid
  SQL code.

- `createResultSetMapping` is now run at the end of the standard phases.
  It uses the original result type stored by the new `removeMappedTypes`
  phase, thus keeping the AST free of the client-side parts until the
  point where they actually match the server side.

- `fuseComprehensions` requires a small change to translate aggregations
  arising from `rewriteJoins`. This is only to make it work for now. The
  whole phase needs to be rewritten from scratch.

- Better code generator for explicit join syntax. The default in
  JdbcStatementBuilderComponent implements the standard SQL syntax.
  MySQL, SQLite and Hsqldb require special handling (in particular
  Hsqldb, which does not support arbitrary nesting on the RHS of a
  join).

common-test-resources/logback.xml

@@ -16,13 +16,15 @@
     <logger name="slick.compiler.AssignUniqueSymbols"       level="${log.qcomp.assignUniqueSymbols:-inherited}" />
     <logger name="slick.compiler.InferTypes"                level="${log.qcomp.inferTypes:-inherited}" />
     <logger name="slick.compiler.ExpandTables"              level="${log.qcomp.expandTables:-inherited}" />
-    <logger name="slick.compiler.CreateResultSetMapping"    level="${log.qcomp.createResultSetMapping:-inherited}" />
     <logger name="slick.compiler.EmulateOuterJoins"         level="${log.qcomp.emulateOuterJoins:-inherited}" />
     <logger name="slick.compiler.ForceOuterBinds"           level="${log.qcomp.forceOuterBinds:-inherited}" />
+    <logger name="slick.compiler.RemoveMappedTypes"         level="${log.qcomp.removeMappedTypes:-inherited}" />
+    <logger name="slick.compiler.CreateResultSetMapping"    level="${log.qcomp.createResultSetMapping:-inherited}" />
     <logger name="slick.compiler.ExpandSums"                level="${log.qcomp.expandSums:-inherited}" />
     <logger name="slick.compiler.ExpandRecords"             level="${log.qcomp.expandRecords:-inherited}" />
     <logger name="slick.compiler.ExpandConditionals"        level="${log.qcomp.expandConditionals:-inherited}" />
     <logger name="slick.compiler.FlattenProjections"        level="${log.qcomp.flattenProjections:-inherited}" />
+    <logger name="slick.compiler.RewriteJoins"              level="${log.qcomp.rewriteJoins:-inherited}" />
     <logger name="slick.compiler.RelabelUnions"             level="${log.qcomp.relabelUnions:-inherited}" />
     <logger name="slick.compiler.PruneFields"               level="${log.qcomp.pruneFields:-inherited}" />
     <logger name="slick.compiler.ResolveZipJoins"           level="${log.qcomp.resolveZipJoins:-inherited}" />

project/Build.scala

@@ -13,8 +13,8 @@ import de.johoop.testngplugin.TestNGPlugin._
 
 object SlickBuild extends Build {
 
-  val slickVersion = "3.0.0-RC3"
-  val binaryCompatSlickVersion = "3.0.0" // Slick base version for binary compatibility checks
+  val slickVersion = "3.1.0-SNAPSHOT"
+  val binaryCompatSlickVersion = "3.1.0" // Slick base version for binary compatibility checks
   val scalaVersions = Seq("2.10.5", "2.11.6")
 
   /** Dependencies for reuse in different parts of the build */

slick-testkit/src/doctest/resources/logback.xml

@@ -16,13 +16,15 @@
     <logger name="slick.compiler.AssignUniqueSymbols"       level="${log.qcomp.assignUniqueSymbols:-inherited}" />
     <logger name="slick.compiler.InferTypes"                level="${log.qcomp.inferTypes:-inherited}" />
     <logger name="slick.compiler.ExpandTables"              level="${log.qcomp.expandTables:-inherited}" />
-    <logger name="slick.compiler.CreateResultSetMapping"    level="${log.qcomp.createResultSetMapping:-inherited}" />
     <logger name="slick.compiler.EmulateOuterJoins"         level="${log.qcomp.emulateOuterJoins:-inherited}" />
     <logger name="slick.compiler.ForceOuterBinds"           level="${log.qcomp.forceOuterBinds:-inherited}" />
+    <logger name="slick.compiler.RemoveMappedTypes"         level="${log.qcomp.removeMappedTypes:-inherited}" />
+    <logger name="slick.compiler.CreateResultSetMapping"    level="${log.qcomp.createResultSetMapping:-inherited}" />
     <logger name="slick.compiler.ExpandSums"                level="${log.qcomp.expandSums:-inherited}" />
     <logger name="slick.compiler.ExpandRecords"             level="${log.qcomp.expandRecords:-inherited}" />
     <logger name="slick.compiler.ExpandConditionals"        level="${log.qcomp.expandConditionals:-inherited}" />
     <logger name="slick.compiler.FlattenProjections"        level="${log.qcomp.flattenProjections:-inherited}" />
+    <logger name="slick.compiler.RewriteJoins"              level="${log.qcomp.rewriteJoins:-inherited}" />
     <logger name="slick.compiler.RelabelUnions"             level="${log.qcomp.relabelUnions:-inherited}" />
     <logger name="slick.compiler.PruneFields"               level="${log.qcomp.pruneFields:-inherited}" />
     <logger name="slick.compiler.ResolveZipJoins"           level="${log.qcomp.resolveZipJoins:-inherited}" />

slick-testkit/src/main/scala/com/typesafe/slick/testkit/tests/AggregateTest.scala

@@ -175,12 +175,12 @@ class AggregateTest extends AsyncTest[RelationalTestDB] {
       def * = id
     }
     val as = TableQuery[A]
-    for {
-      _ <- as.schema.create
-      _ <- as += 1
-      q1 = as.groupBy(_.id).map { case (_, q) => (q.map(_.id).min, q.length) }
-      _ <- q1.result
-    } yield ()
+    val q1 = as.groupBy(_.id).map { case (_, q) => (q.map(_.id).min, q.length) }
+    DBIO.seq(
+      as.schema.create,
+      as += 1,
+      q1.result
+    )
   }
 
   def testGroup3 = {

slick-testkit/src/main/scala/com/typesafe/slick/testkit/tests/CountTest.scala

@@ -69,27 +69,24 @@ class CountTest extends AsyncTest[RelationalTestDB] {
       def * = (aId, data)
     }
     lazy val bs = TableQuery[B]
-    for {
-      _ <- (as.schema ++ bs.schema).create
-      _ <- as ++= Seq(1L, 2L)
-      _ <- bs ++= Seq((1L, "1a"), (1L, "1b"), (2L, "2"))
-      qDirectLength = for {
+    DBIO.seq(
+      (as.schema ++ bs.schema).create,
+      as ++= Seq(1L, 2L),
+      bs ++= Seq((1L, "1a"), (1L, "1b"), (2L, "2")),
+      (for {
         a <- as if a.id === 1L
       } yield (a, (for {
           b <- bs if b.aId === a.id
-        } yield b).length)
-      _ <- qDirectLength.result.map(_ shouldBe Seq((1L, 2)))
-      qJoinLength = for {
+        } yield b).length)).result.named("directLength").map(_ shouldBe Seq((1L, 2))),
+      (for {
         a <- as if a.id === 1L
         l <- Query((for {
           b <- bs if b.aId === a.id
         } yield b).length)
-      } yield (a, l)
-      _ <- qJoinLength.result.map(_ shouldBe Seq((1L, 2)))
-      qOuterJoinLength = (for {
+      } yield (a, l)).result.named("joinLength").map(_ shouldBe Seq((1L, 2))),
+      (for {
         (a, b) <- as joinLeft bs on (_.id === _.aId)
-      } yield (a.id, b.map(_.data))).length
-      _ <- qOuterJoinLength.result.map(_ shouldBe 3)
-    } yield ()
+      } yield (a.id, b.map(_.data))).length.result.named("outerJoinLength").map(_ shouldBe 3)
+    )
   }
 }

slick-testkit/src/main/scala/com/typesafe/slick/testkit/tests/JoinTest.scala

@@ -304,16 +304,16 @@ class JoinTest extends AsyncTest[RelationalTestDB] {
     }
     lazy val cs = TableQuery[C]
 
-    val q1 = for {
+    def q1 = for {
       (a, b) <- as joinLeft bs on (_.id === _.foreignId)
     } yield (a, b)
 
-    val q2 = for {
+    def q2 = for {
       (a, b) <- q1
       c <- cs if c.foreignId === a.id
     } yield (a, c)
 
-    val q3 = for {
+    def q3 = for {
       (a, b) <- as joinLeft bs on (_.id === _.foreignId)
       c <- cs if c.foreignId === a.id
     } yield (a, c)
@@ -323,9 +323,9 @@ class JoinTest extends AsyncTest[RelationalTestDB] {
       as ++= Seq(1,2,3),
       bs ++= Seq(1,2,4,5),
       cs ++= Seq(1,2,4,6),
-      q1.result.map(_.toSet shouldBe Set((1, Some(1)), (2, Some(2)), (3, None))),
-      q2.result.map(_.toSet shouldBe Set((1,1), (2,2))),
-      q3.result.map(_.toSet shouldBe Set((1,1), (2,2)))
+      q1.result.named("q1").map(_.toSet shouldBe Set((1, Some(1)), (2, Some(2)), (3, None))),
+      q2.result.named("q2").map(_.toSet shouldBe Set((1,1), (2,2))),
+      q3.result.named("q3").map(_.toSet shouldBe Set((1,1), (2,2)))
     )
   }
 }

slick-testkit/src/main/scala/com/typesafe/slick/testkit/tests/NestingTest.scala

@@ -104,13 +104,13 @@ class NestingTest extends AsyncTest[RelationalTestDB] {
     val q5t: Query[(Rep[Option[Int]], Rep[Option[String]]), _, Seq] = q5
 
     val t1 = seq(
-      q1.result.map(_ shouldBe r.map(t => Some(t))),
-      q1a2.result.map(_ shouldBe r.map(t => Some(Some(t)))),
-      q2.result.map(_ shouldBe r.map(t => Some(t._1))),
-      q2a2.result.map(_ shouldBe r.map(t => Some(Some(t._1)))),
-      q3.result.map(_ shouldBe r.map(t => t._3)),
-      q4.result.map(_ shouldBe r.map(t => Some(t._3))),
-      q5.result.map(_ shouldBe r.map(t => (t._3, Some(t._2))))
+      q1.result.named("q1").map(_ shouldBe r.map(t => Some(t))),
+      q1a2.result.named("q1a2").map(_ shouldBe r.map(t => Some(Some(t)))),
+      q2.result.named("q2").map(_ shouldBe r.map(t => Some(t._1))),
+      q2a2.result.named("q2a2").map(_ shouldBe r.map(t => Some(Some(t._1)))),
+      q3.result.named("q3").map(_ shouldBe r.map(t => t._3)),
+      q4.result.named("q4").map(_ shouldBe r.map(t => Some(t._3))),
+      q5.result.named("q5").map(_ shouldBe r.map(t => (t._3, Some(t._2))))
     )
 
     // Get plain values out
@@ -124,10 +124,10 @@ class NestingTest extends AsyncTest[RelationalTestDB] {
     val q4bt: Query[Rep[Option[Int]], _, Seq] = q4b
 
     val t2 = seq(
-      q1b.result.map(_ shouldBe r.map(t => Some(t)).map(_.getOrElse((0, "", None: Option[String])))),
-      q2b.result.map(_ shouldBe r.map(t => Some(t._1)).map(_.get)),
-      q3b.result.map(_ shouldBe r.map(t => t._3).filter(_.isDefined).map(_.get)),
-      q4b.result.map(_ shouldBe r.map(t => Some(t._3)).map(_.getOrElse(None: Option[String])))
+      q1b.result.named("q1b").map(_ shouldBe r.map(t => Some(t)).map(_.getOrElse((0, "", None: Option[String])))),
+      q2b.result.named("q2b").map(_ shouldBe r.map(t => Some(t._1)).map(_.get)),
+      q3b.result.named("q3b").map(_ shouldBe r.map(t => t._3).filter(_.isDefined).map(_.get)),
+      q4b.result.named("q4b").map(_ shouldBe r.map(t => Some(t._3)).map(_.getOrElse(None: Option[String])))
     )
 
     // Unpack result types
@@ -142,8 +142,8 @@ class NestingTest extends AsyncTest[RelationalTestDB] {
     val q3c = q3.map(so => so + 10)
 
     val t3 = seq(
-      q2c.result.map(_ shouldBe r.map(t => Some(t._1)).map(_.map(_ + 42))),
-      q3c.result.map(_ shouldBe r.map(t => t._3).map(_.map(_ + 10)))
+      q2c.result.named("q2c").map(_ shouldBe r.map(t => Some(t._1)).map(_.map(_ + 42))),
+      q3c.result.named("q3c").map(_ shouldBe r.map(t => t._3).map(_.map(_ + 10)))
     )
 
     // Use Option.map
@@ -163,11 +163,11 @@ class NestingTest extends AsyncTest[RelationalTestDB] {
     val q4dt: Query[Rep[Option[Int]], _, Seq] = q4d
 
     val t4 = seq(
-      q1d.result.map(_ shouldBe r.map(t => Some(t)).map(_.map(_._1))),
-      q1d2.result.map(_ shouldBe r.map(t => Some(t)).map(_.map(x => (x._1, x._2, x._3)))),
-      q2d.result.map(_ shouldBe r.map(t => Some(t._1)).map(_.map(_ + 1))),
-      q3d.result.map(_ shouldBe r.map(t => t._3).map(_.map(s => (s, s, 1)))),
-      q4d.result.map(_ shouldBe r.map(t => Some(t._3)).map(_.filter(_.isDefined).map(_.get)))
+      q1d.result.named("q1d").map(_ shouldBe r.map(t => Some(t)).map(_.map(_._1))),
+      q1d2.result.named("q1d2").map(_ shouldBe r.map(t => Some(t)).map(_.map(x => (x._1, x._2, x._3)))),
+      q2d.result.named("q2d").map(_ shouldBe r.map(t => Some(t._1)).map(_.map(_ + 1))),
+      q3d.result.named("q3d").map(_ shouldBe r.map(t => t._3).map(_.map(s => (s, s, 1)))),
+      q4d.result.named("q4d").map(_ shouldBe r.map(t => Some(t._3)).map(_.filter(_.isDefined).map(_.get)))
     )
 
     // Use Option.flatMap
@@ -180,10 +180,10 @@ class NestingTest extends AsyncTest[RelationalTestDB] {
     val q2et: Query[Rep[Option[Int]], _, Seq] = q2e
 
     val t5 = seq(
-      q1e1.result.map(_ shouldBe r.map(t => Some(t)).map { to => to.flatMap { t => Some(t._2) }}),
-      q1e2.result.map(_ shouldBe r.map(t => Some(t)).map { to => to.flatMap { t => t._3 }}),
-      q1e3.result.map(_ shouldBe r.map(t => Some(t)).map(to => Some(to)).map(_.flatMap(identity))),
-      q2e.result.map(_ shouldBe r.map(t => Some(t._1)).map { io => io.flatMap { i => Some(i) }})
+      q1e1.result.named("q1e1").map(_ shouldBe r.map(t => Some(t)).map { to => to.flatMap { t => Some(t._2) }}),
+      q1e2.result.named("q1e2").map(_ shouldBe r.map(t => Some(t)).map { to => to.flatMap { t => t._3 }}),
+      q1e3.result.named("q1e3").map(_ shouldBe r.map(t => Some(t)).map(to => Some(to)).map(_.flatMap(identity))),
+      q2e.result.named("q2e").map(_ shouldBe r.map(t => Some(t._1)).map { io => io.flatMap { i => Some(i) }})
     )
 
     // Use Option.flatten
@@ -201,12 +201,12 @@ class NestingTest extends AsyncTest[RelationalTestDB] {
     val q2f3t: Query[Rep[Option[Int]], _, Seq] = q2f3
 
     val t6 = seq(
-      q1f1.result.map(_ shouldBe Vector(Some(Some((1,"1",Some(1)))), Some(Some((2,"2",Some(2)))), Some(Some((3,"3",None))))),
-      q1f2.result.map(_ shouldBe r.map(t => Some(t)).map { to => Some(to).flatten }),
-      q1f3.result.map(_ shouldBe r.map(t => Some(t)).map { to => Some(to) }.map(_.flatten)),
-      q2f1.result.map(_ shouldBe r.map(t => Some(t._1)).map { io => Some(io) }),
-      q2f2.result.map(_ shouldBe r.map(t => Some(t._1)).map { io => Some(io).flatten }),
-      q2f3.result.map(_ shouldBe r.map(t => Some(t._1)).map { io => Some(io) }.map(_.flatten))
+      q1f1.result.named("q1f1").map(_ shouldBe Vector(Some(Some((1,"1",Some(1)))), Some(Some((2,"2",Some(2)))), Some(Some((3,"3",None))))),
+      q1f2.result.named("q1f2").map(_ shouldBe r.map(t => Some(t)).map { to => Some(to).flatten }),
+      q1f3.result.named("q1f3").map(_ shouldBe r.map(t => Some(t)).map { to => Some(to) }.map(_.flatten)),
+      q2f1.result.named("q2f1").map(_ shouldBe r.map(t => Some(t._1)).map { io => Some(io) }),
+      q2f2.result.named("q2f2").map(_ shouldBe r.map(t => Some(t._1)).map { io => Some(io).flatten }),
+      q2f3.result.named("q2f3").map(_ shouldBe r.map(t => Some(t._1)).map { io => Some(io) }.map(_.flatten))
     )
 
     setup >> t1 >> t2 >> t3 >> t4 >> t5 >> t6

slick-testkit/src/main/scala/com/typesafe/slick/testkit/tests/NewQuerySemanticsTest.scala

@@ -112,7 +112,7 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
       (c2, s2) <- q1b_0
     } yield (c.name, s.city, c2.name)
 
-    val a2 = seq(
+    def a2 = seq(
       q0.result.named("Plain table").map(_.toSet).map { r0 =>
         r0 shouldBe Set(
           ("Colombian",         101, 799, 1, 0),
@@ -181,7 +181,7 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
       d <- q4b_0
     } yield (c,d)
 
-    val a3 = seq(
+    def a3 = seq(
       q2.result.named("More elaborate query").map(_.toSet).map { r2 =>
         r2 shouldBe Set(
           ("Colombian","Acme, Inc."),
@@ -229,7 +229,7 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
     // Unused outer query result, unbound TableQuery
     val q6 = coffees.flatMap(c => suppliers)
 
-    val a4 = seq(
+    def a4 = seq(
       q5.result.map(_.toSet).map { r5 =>
         r5 shouldBe Set(
           (("Colombian",101,799,1,0),("Colombian",101,799,1,0)),
@@ -268,7 +268,7 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
       c <- coffees.filter(_.price < 800).map((_, 1))
     } yield (c._1.name, c._1.supID, c._2)
 
-    val a5 = seq(
+    def a5 = seq(
       q7a.result.named("Simple union").map(_.toSet).map { r7a =>
         r7a shouldBe Set(
           ("Colombian",101,0),
@@ -306,7 +306,7 @@ class NewQuerySemanticsTest extends AsyncTest[RelationalTestDB] {
       t <- coffees.sortBy(_.sales).take(1) joinLeft coffees.sortBy(_.sales).take(2) on (_.name === _.name) joinLeft coffees.sortBy(_.sales).take(4) on (_._1.supID === _.supID)
     } yield (t._1, t._2)
 
-    val a6 = seq(
+    def a6 = seq(
       q7b.result.named("Union with filter on the outside").map(_.toSet).map { r7b =>
         r7b shouldBe Set(
           ("French_Roast",49,1),

slick-testkit/src/test/scala/slick/benchmark/UnboxedBenchmark.scala

@@ -3,8 +3,8 @@ package slick.benchmark
 import slick.ast._
 import slick.jdbc._
 import slick.relational._
-import slick.util.TreePrinter
 import com.typesafe.slick.testkit.util.DelegateResultSet
+import slick.util.TreePrinter
 
 @deprecated("Using deprecated .simple API", "3.0")
 object UnboxedBenchmark extends App {

slick/src/main/scala/slick/ast/Node.scala

@@ -84,6 +84,8 @@ trait Node extends Dumpable {
     else nodeTyped(tpe)
   }
 
+  final def :@ (tpe: Type): Self = nodeTypedOrCopy(tpe)
+
   def nodeBuildTypedNode[T >: this.type <: Node](newNode: T, newType: Type): T =
     if(newNode ne this) newNode.nodeTyped(newType)
     else if(newType == _nodeType) this
@@ -514,7 +516,8 @@ final case class Ref(sym: Symbol) extends NullaryNode {
   def nodeRebuild = copy()
 }
 
-/** A constructor/extractor for nested Selects starting at a Ref. */
+/** A constructor/extractor for nested Selects starting at a Ref so that, for example,
+  * `c :: b :: a :: Nil` corresponds to path `a.b.c`. */
 object Path {
   def apply(l: List[Symbol]): Node = l match {
     case s :: Nil => Ref(s)
@@ -532,6 +535,8 @@ object Path {
   }
 }
 
+/** A constructor/extractor for nested Selects starting at a Ref so that, for example,
+  * `a :: b :: c :: Nil` corresponds to path `a.b.c`. */
 object FwdPath {
   def apply(ch: List[Symbol]) = Path(ch.reverse)
   def unapply(n: Node): Option[List[Symbol]] = Path.unapply(n).map(_.reverse)

slick/src/main/scala/slick/ast/Util.scala

@@ -36,7 +36,8 @@ object Scope {
 final class NodeOps(val tree: Node) extends AnyVal {
   import Util._
 
-  @inline def collect[T](pf: PartialFunction[Node, T]): Seq[T] = NodeOps.collect(tree, pf)
+  @inline def collect[T](pf: PartialFunction[Node, T], stopOnMatch: Boolean = false): Seq[T] =
+    NodeOps.collect(tree, pf, stopOnMatch)
 
   def collectAll[T](pf: PartialFunction[Node, Seq[T]]): Seq[T] = collect[Seq[T]](pf).flatten
 
@@ -75,6 +76,16 @@ final class NodeOps(val tree: Node) extends AnyVal {
     case (s: ElementSymbol, ProductNode(ch)) => ch(s.idx-1)
     case (s, n) => Select(n, s)
   }
+
+  def hasRefTo(s: Symbol): Boolean = findNode {
+    case Ref(s2) if s2 == s => true
+    case _ => false
+  }.isDefined
+
+  def hasRefToOneOf(s: Set[Symbol]): Boolean = findNode {
+    case Ref(s2) if s contains s2 => true
+    case _ => false
+  }.isDefined
 }
 
 object NodeOps {
@@ -83,9 +94,15 @@ object NodeOps {
   // These methods should be in the class but 2.10.0-RC1 took away the ability
   // to use closures in value classes
 
-  def collect[T](tree: Node, pf: PartialFunction[Node, T]): Seq[T] = {
+  def collect[T](tree: Node, pf: PartialFunction[Node, T], stopOnMatch: Boolean): Seq[T] = {
     val b = new ArrayBuffer[T]
-    tree.foreach(pf.andThen[Unit]{ case t => b += t }.orElse[Node, Unit]{ case _ => () })
+    def f(n: Node): Unit = pf.andThen[Unit] { case t =>
+      b += t
+      if(!stopOnMatch) n.nodeChildren.foreach(f)
+    }.orElse[Node, Unit]{ case _ =>
+      n.nodeChildren.foreach(f)
+    }.apply(n)
+    f(tree)
     b
   }