Merge pull request #1920 from deusaquilus2/quat_base_subquery_compressed

Change Subquery Expansion to be Quat-based

quill-core-portable/src/main/scala/io/getquill/AstPrinter.scala

@@ -26,8 +26,8 @@ class AstPrinter(traceOpinions: Boolean, traceAstSimple: Boolean) extends pprint
 
   private def printRenameable(r: Renameable) =
     r match {
-      case ByStrategy => Tree.Literal("S")
-      case Fixed      => Tree.Literal("F")
+      case ByStrategy => Tree.Literal("Ren")
+      case Fixed      => Tree.Literal("Fix")
     }
 
   private def printVisibility(v: Visibility) =

quill-core-portable/src/main/scala/io/getquill/ast/Ast.scala

@@ -68,6 +68,9 @@ case class Entity(name: String, properties: List[PropertyAlias], quat: Quat.Prod
   // scala creates companion objects, the apply/unapply wouldn't be able to work correctly.
   def renameable: Renameable = Renameable.neutral
 
+  def copy(name: String = this.name, properties: List[PropertyAlias] = this.properties, quat: Quat.Product = this.quat) =
+    Entity.Opinionated(name, properties, quat, this.renameable)
+
   override def equals(that: Any) =
     that match {
       case e: Entity => this.id == e.id
@@ -194,8 +197,13 @@ case class Ident(name: String, quat: Quat) extends Terminal with Ast {
 
   override def hashCode = id.hashCode()
 
-  override def withQuat(newQuat: Quat) =
+  override def withQuat(newQuat: Quat) = {
     Ident.Opinionated(this.name, newQuat, this.visibility)
+  }
+
+  // need to define a copy which will propogate current value of visibility into the copy
+  def copy(name: String = this.name, quat: Quat = this.quat): Ident =
+    Ident.Opinionated(name, quat, this.visibility)
 }
 
 /**
@@ -246,6 +254,10 @@ case class ExternalIdent(name: String, quat: Quat) extends Ast {
     }
 
   override def hashCode = id.hashCode()
+
+  // need to define a copy which will propogate current value of visibility into the copy
+  def copy(name: String = this.name, quat: Quat = this.quat): ExternalIdent =
+    ExternalIdent.Opinionated(name, quat, this.renameable)
 }
 
 object ExternalIdent {
@@ -314,10 +326,14 @@ case class Property(ast: Ast, name: String) extends Ast {
   def renameable: Renameable = Renameable.neutral
 
   def quat = ast.quat.lookup(name)
+  def prevName = ast.quat.beforeRenamed(name)
 
   // Properties that are 'Hidden' are used for embedded objects whose path should not be expressed
   // during SQL Tokenization.
   def visibility: Visibility = Visibility.Visible
+
+  def copy(ast: Ast = this.ast, name: String = this.name): Property =
+    Property.Opinionated(ast, name, this.renameable, this.visibility)
 }
 
 object Property {

quill-core-portable/src/main/scala/io/getquill/ast/AstOps.scala

@@ -93,3 +93,14 @@ object IfExist {
     case _                                => None
   }
 }
+
+object PropertyOrCore {
+  def unapply(ast: Ast): Boolean =
+    Core.unapply(ast) || ast.isInstanceOf[Property]
+}
+
+/* Things that can be on the inside of a series of nested properties */
+object Core {
+  def unapply(ast: Ast): Boolean =
+    ast.isInstanceOf[Ident] || ast.isInstanceOf[Infix] || ast.isInstanceOf[Constant]
+}

quill-core-portable/src/main/scala/io/getquill/norm/RenameProperties.scala

@@ -46,10 +46,14 @@ object RenameProperties {
 
 object CompleteRenames extends StatelessTransformer {
   // NOTE Leaving renames on Entities so knowledges of what renames have been done remains in the AST. May want to change this in the future.
-  override def applyIdent(e: Ident): Ident = e match {
-    case e: Ident =>
-      e.copy(quat = e.quat.applyRenames)
-  }
+  override def applyIdent(e: Ident): Ident =
+    e.copy(quat = e.quat.applyRenames)
+
+  override def apply(e: Query): Query =
+    e match {
+      case ent: Entity => ent.copy(quat = ent.quat.applyRenames)
+      case _           => super.apply(e)
+    }
 
   override def apply(e: Ast): Ast = e match {
     case e: Ident =>
@@ -146,20 +150,20 @@ object SeedRenames extends StatelessTransformer {
 // Represents a nested property path to an identity i.e. Property(Property(... Ident(), ...))
 object PropertyMatroshka {
 
-  def traverse(initial: Property): Option[(Ident, List[String])] =
+  def traverse(initial: Property): Option[(Ast, List[String])] =
     initial match {
       // If it's a nested-property walk inside and append the name to the result (if something is returned)
       case Property(inner: Property, name) =>
         traverse(inner).map { case (id, list) => (id, list :+ name) }
       // If it's a property with ident in the core, return that
-      case Property(id: Ident, name) =>
-        Some((id, List(name)))
+      case Property(inner, name) if !inner.isInstanceOf[Property] =>
+        Some((inner, List(name)))
       // Otherwise an ident property is not inside so don't return anything
       case _ =>
         None
     }
 
-  def unapply(ast: Ast): Option[(Ident, List[String])] =
+  def unapply(ast: Property): Option[(Ast, List[String])] =
     ast match {
       case p: Property => traverse(p)
       case _           => None

quill-core-portable/src/main/scala/io/getquill/norm/RepropagateQuats.scala

@@ -98,7 +98,7 @@ object RepropagateQuats extends StatelessTransformer {
             case OnConflict.Properties(props) =>
               val propsR = props.map {
                 // Recreate the assignment with new idents but only if we need to repropagate
-                case prop @ PropertyMatroshka(ident, _) =>
+                case prop @ PropertyMatroshka(ident: Ident, _) =>
                   trace"Repropagate OnConflict.Properties Quat ${oca.quat.suppress(msg)} from $oca into:" andReturn
                     BetaReduction(prop, RWR, ident -> ident.withQuat(oca.quat)).asInstanceOf[Property]
                 case other =>

quill-core-portable/src/main/scala/io/getquill/quat/Quat.scala

@@ -63,15 +63,23 @@ sealed trait Quat {
     case Quat.Null    => "N"
   }
 
+  /** What was the value of a given property before it was renamed (i.e. looks up the value of the Renames hash) */
+  def beforeRenamed(path: String): Option[String] = (this, path) match {
+    case (cc: Quat.Product, fieldName) =>
+      // NOTE This is a linear lookup. To improve efficiency store a map going back from rename to the initial property,
+      // if we did that however, we would need to make sure to warn a user of two things are renamed to the same property however,
+      // that kind of warning should probably exist already
+      renames.find(_._2 == fieldName).headOption.map(_._2)
+    case (other, fieldName) =>
+      QuatException(s"The post-rename field '${fieldName}' does not exist in an SQL-level type ${other}")
+  }
+
   def lookup(path: String): Quat = (this, path) match {
     case (cc @ Quat.Product(fields), fieldName) =>
+      // TODO Change to Get
       fields.find(_._1 == fieldName).headOption.map(_._2).getOrElse(QuatException(s"The field ${fieldName} does not exist in the SQL-level ${cc}"))
-    case (Quat.Value, fieldName) =>
-      QuatException(s"The field '${fieldName}' does not exist in an SQL-level leaf-node")
-    case (Quat.Null, fieldName) =>
-      QuatException(s"The field '${fieldName}' cannot be looked up from a SQL-level null node")
-    case (Quat.Generic, fieldName) =>
-      QuatException(s"The field '${fieldName}' cannot be looked up from a SQL-level generic node")
+    case (other, fieldName) =>
+      QuatException(s"The field '${fieldName}' does not exist in an SQL-level type ${other}")
   }
   def lookup(list: List[String]): Quat =
     list match {
@@ -127,23 +135,26 @@ object Quat {
     }
 
     def leastUpperTypeProduct(other: Quat.Product): Option[Quat.Product] = {
-      val newFields =
+      val newFieldsIter =
         fields.zipWith(other.fields) {
           case (key, thisQuat, Some(otherQuat)) => (key, thisQuat.leastUpperType(otherQuat))
         }.collect {
           case (key, Some(value)) => (key, value)
         }
-      Some(Quat.Product(newFields))
+      val newFields = mutable.LinkedHashMap(newFieldsIter.toList: _*)
+      // Note, some extra renames from properties that don't exist could make it here.
+      // Need to make sure to ignore extra ones when they are actually applied.
+      Some(Quat.Product(newFields).withRenames(renames))
     }
 
     def withRenames(renames: List[(String, String)]) =
       Product.WithRenames(fields, renames)
 
     /**
-     * Rename the properties and reset renames to empty.
-     * An interesting idea would be not to clear
-     * the renames at the end but rather keep them around until the SqlQuery phase wherein we could potentially
-     * create SQL aliases not based on the renamed properties by based on the original property name.
+     * Rename the properties based on the renames list. Keep this list
+     * around since it is used in sql sub-query expansion to determine whether
+     * the property is fixed or not (i.e. whether the column naming strategy should
+     * be applied to it).
      */
     override def applyRenames: Quat.Product = {
       val newFields = fields.map {
@@ -153,7 +164,7 @@ object Quat {
           val newValue = q.applyRenames
           (newKey, newValue)
       }
-      Product(newFields)
+      Product.WithRenames(newFields, renames)
     }
   }
   def LeafProduct(list: String*) = Quat.Product(list.map(e => (e, Quat.Value)))

quill-core-portable/src/main/scala/io/getquill/util/Messages.scala

@@ -8,6 +8,8 @@ object Messages {
     Option(System.getProperty(propName)).orElse(sys.env.get(envName)).getOrElse(default)
 
   private[util] def prettyPrint = variable("quill.macro.log.pretty", "quill_macro_log", "false").toBoolean
+  private[getquill] def alwaysAlias = variable("quill.query.alwaysAlias", "quill_query_alwaysAlias", "false").toBoolean
+  private[getquill] def pruneColumns = variable("quill.query.pruneColumns", "quill_query_pruneColumns", "true").toBoolean
   private[util] def debugEnabled = variable("quill.macro.log", "quill_macro_log", "true").toBoolean
   private[util] def traceEnabled = variable("quill.trace.enabled", "quill_trace_enabled", "false").toBoolean
   private[util] def traceColors = variable("quill.trace.color", "quill_trace_color,", "false").toBoolean

quill-core/src/main/scala/io/getquill/quat/QuatMaking.scala

@@ -18,18 +18,25 @@ trait QuatMaking extends QuatMakingBase {
   lazy val u: Uni = c.universe
 
   import u.{ Block => _, Constant => _, Function => _, Ident => _, If => _, _ }
+  import collection.mutable.HashMap;
 
-  def existsEncoderFor(tpe: Type) = {
-    OptionalTypecheck(c)(q"implicitly[${c.prefix}.Encoder[$tpe]]") match {
-      case Some(enc) => true
-      case None      => false
+  val cachedEncoderLookups: HashMap[Type, Boolean] = HashMap();
+  def existsEncoderFor(tpe: Type): Boolean = {
+    cachedEncoderLookups.get(tpe) match {
+      case Some(value) =>
+        value
+      case None =>
+        val lookup =
+          OptionalTypecheck(c)(q"implicitly[${c.prefix}.Encoder[$tpe]]") match {
+            case Some(enc) => true
+            case None      => false
+          }
+        cachedEncoderLookups.put(tpe, lookup)
+        lookup
     }
   }
 
-  import collection.mutable.HashMap;
-
   val cachedQuats: HashMap[Type, Quat] = HashMap();
-
   override def inferQuat(tpe: u.Type): Quat = {
     cachedQuats.get(tpe) match {
       case Some(value) =>
@@ -258,8 +265,10 @@ trait QuatMakingBase {
 
         case _ if (isNone(tpe)) =>
           Quat.Null
-        // For other types of case classes
-        case CaseClassBaseType(name, fields) =>
+
+        // For other types of case classes (and if there does not exist an encoder for it)
+        // the exception to that is a cassandra UDT that we treat like an encodeable entity even if it has a parsed type
+        case CaseClassBaseType(name, fields) if !existsEncoderFor(tpe) || tpe <:< typeOf[Udt] =>
           Quat.Product(fields.map { case (fieldName, fieldType) => (fieldName, parseType(fieldType)) })
 
         // If we are already inside a bounded type, treat an arbitrary type as a interface list

quill-core/src/main/scala/io/getquill/quotation/Parsing.scala

@@ -715,13 +715,13 @@ trait Parsing extends ValueComputation with QuatMaking {
   }
 
   val valueParser: Parser[Ast] = Parser[Ast] {
-    case q"null"                         => NullValue
-    case q"scala.Some.apply[$t]($v)"     => OptionSome(astParser(v))
-    case q"scala.Option.apply[$t]($v)"   => OptionApply(astParser(v))
-    case q"scala.None"                   => OptionNone(Quat.Null)
-    case q"scala.Option.empty[$t]"       => OptionNone(inferQuat(t.tpe))
-    case Literal(c.universe.Constant(v)) => Constant(v)
-    case q"((..$v))" if (v.size > 1)     => Tuple(v.map(astParser(_)))
+    case q"null"                             => NullValue
+    case q"scala.Some.apply[$t]($v)"         => OptionSome(astParser(v))
+    case q"scala.Option.apply[$t]($v)"       => OptionApply(astParser(v))
+    case q"scala.None"                       => OptionNone(Quat.Null)
+    case q"scala.Option.empty[$t]"           => OptionNone(inferQuat(t.tpe))
+    case l @ Literal(c.universe.Constant(v)) => { Constant(v); }
+    case q"((..$v))" if (v.size > 1)         => Tuple(v.map(astParser(_)))
     case q"new $ccTerm(..$v)" if (isCaseClass(c.WeakTypeTag(ccTerm.tpe.erasure))) => {
       val values = v.map(astParser(_))
       val params = firstConstructorParamList(c.WeakTypeTag(ccTerm.tpe.erasure))

quill-core/src/test/scala/io/getquill/Spec.scala

@@ -14,6 +14,7 @@ abstract class Spec extends AnyFreeSpec with Matchers with BeforeAndAfterAll {
   val QV = Quat.Value
   val QEP = Quat.Product.empty
   def QP(fields: String*) = Quat.LeafProduct(fields: _*)
+  val fisdfsd = "sudufnskhdbf"
 
   // Used by various tests to replace temporary idents created by AttachToEntity with 'x'
   val replaceTempIdent = new StatelessTransformer {

quill-core/src/test/scala/io/getquill/UpperCaseNonDefault.scala

@@ -6,3 +6,10 @@ trait UpperCaseNonDefault extends NamingStrategy {
   override def default(s: String) = s
 }
 object UpperCaseNonDefault extends UpperCaseNonDefault
+
+trait UpperCaseEscapeColumn extends NamingStrategy {
+  override def column(s: String): String = s""""${s.toUpperCase}""""
+  override def table(s: String): String = s
+  override def default(s: String) = s
+}
+object UpperCaseEscapeColumn extends UpperCaseEscapeColumn

quill-orientdb/src/main/scala/io/getquill/context/orientdb/OrientDBIdiom.scala

@@ -6,11 +6,12 @@ import io.getquill.ast.{ AggregationOperator, External, _ }
 import io.getquill.context.sql._
 import io.getquill.NamingStrategy
 import io.getquill.context.CannotReturn
-import io.getquill.util.Messages.fail
+import io.getquill.util.Messages.{ fail, trace }
 import io.getquill.idiom._
 import io.getquill.context.sql.norm.SqlNormalize
-import io.getquill.util.Interleave
+import io.getquill.util.{ Interleave, Messages }
 import io.getquill.context.sql.idiom.VerifySqlQuery
+import io.getquill.sql.norm.{ RemoveExtraAlias, RemoveUnusedSelects }
 
 object OrientDBIdiom extends OrientDBIdiom with CannotReturn
 
@@ -29,7 +30,15 @@ trait OrientDBIdiom extends Idiom {
         case q: Query =>
           val sql = SqlQuery(q)
           VerifySqlQuery(sql).map(fail)
-          new ExpandNestedQueries(naming)(sql, List()).token
+          val expanded = ExpandNestedQueries(sql)
+          trace("expanded sql")(expanded)
+          val refined = if (Messages.pruneColumns) RemoveUnusedSelects(expanded) else expanded
+          trace("filtered sql (only used selects)")(refined)
+          val cleaned = if (!Messages.alwaysAlias) RemoveExtraAlias(naming)(refined) else refined
+          trace("cleaned sql")(cleaned)
+          val tokenized = cleaned.token
+          trace("tokenized sql")(tokenized)
+          tokenized
         case other =>
           other.token
       }

quill-spark/src/main/scala/io/getquill/context/spark/SimpleNestedExpansion.scala

@@ -0,0 +1,61 @@
+package io.getquill.context.spark
+
+import io.getquill.ast._
+import io.getquill.context.sql._
+import io.getquill.sql.norm.StatelessQueryTransformer
+
+object TopLevelExpansion {
+
+  implicit class AliasOp(alias: Option[String]) {
+    def concatWith(str: String): Option[String] =
+      alias.orElse(Some("")).map(v => s"${v}${str}")
+  }
+
+  def apply(values: List[SelectValue]): List[SelectValue] =
+    values.flatMap(apply(_))
+
+  private def apply(value: SelectValue): List[SelectValue] = {
+    value match {
+      case SelectValue(Tuple(values), alias, concat) =>
+        values.zipWithIndex.map {
+          case (ast, i) =>
+            SelectValue(ast, alias.concatWith(s"_${i + 1}"), concat)
+        }
+      //      case SelectValue(CaseClass(fields), alias, concat) =>
+      //        fields.flatMap {
+      //          case (name, ast) =>
+      //            apply(SelectValue(ast, alias.concatWith(name), concat))
+      //        }
+      // Direct infix select, etc...
+      case other => List(other)
+    }
+  }
+}
+
+object SimpleNestedExpansion extends StatelessQueryTransformer {
+
+  protected override def apply(q: SqlQuery, isTopLevel: Boolean = false): SqlQuery =
+    q match {
+      case q: FlattenSqlQuery =>
+        expandNested(q.copy(select = TopLevelExpansion(q.select))(q.quat), isTopLevel)
+      case other =>
+        super.apply(q, isTopLevel)
+    }
+
+  protected override def expandNested(q: FlattenSqlQuery, isTopLevel: Boolean): FlattenSqlQuery =
+    q match {
+      case FlattenSqlQuery(from, where, groupBy, orderBy, limit, offset, select, distinct) =>
+        val newFroms = q.from.map(expandContext(_))
+
+        def distinctIfNotTopLevel(values: List[SelectValue]) =
+          if (isTopLevel)
+            values
+          else
+            values.distinct
+
+        val distinctSelects =
+          distinctIfNotTopLevel(select)
+
+        q.copy(select = distinctSelects, from = newFroms)(q.quat)
+    }
+}