Remove uneeded norm phase. Further improve quat computation.

quill-core/src/main/scala/io/getquill/quat/QuatMaking.scala

@@ -252,10 +252,22 @@ trait QuatMakingBase extends MacroUtilUniverse {
         }
     }
 
+    def isPrimitive(tpe: Type): Boolean =
+      tpe <:< typeOf[String] ||
+        tpe <:< typeOf[Int] ||
+        tpe <:< typeOf[Short] ||
+        tpe <:< typeOf[Long] ||
+        tpe <:< typeOf[Boolean] ||
+        tpe <:< typeOf[Byte] ||
+        tpe <:< typeOf[Float] ||
+        tpe <:< typeOf[Double]
+
     object DefiniteValue {
       def unapply(tpe: Type): Option[Type] =
         // UDTs (currently only used by cassandra) are created as tables even though there is an encoder for them.
-        if (tpe <:< typeOf[Udt])
+        if (isPrimitive(tpe))
+          Some(tpe)
+        else if (tpe <:< typeOf[Udt])
           None
         else if (isType[AnyVal](tpe))
           Some(tpe)
@@ -358,7 +370,6 @@ trait QuatMakingBase extends MacroUtilUniverse {
 
         // Otherwise it's a terminal value
         case _ =>
-          println(Messages.qprint(s"Could not infer SQL-type of ${tpe}, assuming it is a Unknown Quat."))
           Messages.trace(s"Could not infer SQL-type of ${tpe}, assuming it is a Unknown Quat.")
           Quat.Unknown
       }

quill-engine/src/main/scala/io/getquill/norm/SheathLeafClauses.scala

@@ -306,6 +306,11 @@ case class SheathLeafClauses(state: Option[String], traceConfig: TraceConfig)
       // Unfortunately however due to the ExpandJoin phase being non-well typed this would cause various kinds of queries to fail. Some
       // examples are in SqlQuerySpec. If ExpandJoin can be rewritten to be well-typed this approach can be re-examined.
       case Join(t, a, b, iA, iB, on) =>
+        if (LeafQuat.unapply(iA).isDefined) {
+          val (a1, sa) = apply(a)
+          val iA1      = Ident(iA.name, a1.quat)
+        }
+
         val (a1, sa)   = apply(a)
         val (b1, sb)   = apply(b)
         val (iA1, iB1) = (Ident(iA.name, a1.quat), Ident(iB.name, b1.quat))
@@ -317,12 +322,13 @@ case class SheathLeafClauses(state: Option[String], traceConfig: TraceConfig)
           (Join(t, a1m, b1m, iA, iB, on), SheathLeafClauses(None, traceConfig))
         }
 
-      case Filter(ent, e, LeafQuat(body)) =>
+      // For filter clauses the body is always a quat:V so we check the ident to see if it's something that needs to be sheathed
+      case Filter(ent, LeafQuat(e: Ident), body) =>
         val (ent1, s) = apply(ent)
         val e1        = Ident(e.name, ent1.quat)
-        // For filter clauses we want to go from: Filter(M(ent,e,e.v),e == 123) to Filter(M(ent,e,CC(v->e.v)),e,e.v == 123)
+        // For filter clauses we want to go from: Filter(M(ent,e,e.v),x == 123) to Filter(M(ent,e,CC(v->e.v)),x,x.v == 123)
         // the body should not be re-sheathed since a body of CC(v->e.v == 123) would make no sense since
-        // that's not even a boolean. Instead we just need to do e.v == 123.
+        // that's not even a boolean. Instead we just need to do x.v == 123.
         val bodyC = elaborateSheath(body)(s.state, e, e1)
         trace"Sheath Filter(qry) with $stateInfo in $qq becomes" andReturn {
           (Filter(ent1, e1, bodyC), s)

quill-engine/src/main/scala/io/getquill/sql/SqlQuery.scala

@@ -228,10 +228,11 @@ class SqlQueryApply(traceConfig: TraceConfig) {
                 }
               val collectedAliases = aliases(ctx).map { case (a, quat) => Ident(a, quat) }
               val select           = Tuple(collectedAliases)
-              FlattenSqlQuery(
+              val o = FlattenSqlQuery(
                 from = ctx :: Nil,
                 select = List(SelectValue(select, None))
               )(q.quat)
+              o
             }
           case q @ (_: Filter | _: Entity) =>
             trace"base| Flattening Filter/Entity $q" andReturn { flatten(sources, q, alias, nestNextMap) }
@@ -349,6 +350,17 @@ class SqlQueryApply(traceConfig: TraceConfig) {
           val agg = b.select.collect { case s @ SelectValue(_: Aggregation, _, _) =>
             s
           }
+          // if it's not distinct and there's no aggregation and there's no applicative join
+          // the we can flatten out the inner context. Note that inner applicative-joins are complicated
+          // because they could have wonky inner alises declared so we need to be aggresive about
+          // nesting them.
+//          if (q.isInstanceOf[Join])
+//            trace"Flattening| Map(Ident) [Join]" andReturn
+//              FlattenSqlQuery(
+//                from = QueryContext(apply(q), q.asInstanceOf[Join].) :: Nil,
+//                select = selectValues(p)
+//              )(quat)
+//          else
           if (!b.distinct.isDistinct && agg.isEmpty)
             trace"Flattening| Map(Ident) [Simple]" andReturn
               b.copy(select = selectValues(p))(quat)

quill-engine/src/main/scala/io/getquill/sql/idiom/SqlIdiom.scala

@@ -15,13 +15,21 @@ import io.getquill.idiom.StatementInterpolator._
 import io.getquill.idiom._
 import io.getquill.norm.ConcatBehavior.AnsiConcat
 import io.getquill.norm.EqualityBehavior.AnsiEquality
-import io.getquill.norm.{ConcatBehavior, EqualityBehavior, ExpandReturning, NormalizeCaching, ProductAggregationToken}
+import io.getquill.norm.{
+  ConcatBehavior,
+  EqualityBehavior,
+  ExpandReturning,
+  NormalizeCaching,
+  ProductAggregationToken,
+  SheathLeafClauses
+}
 import io.getquill.quat.Quat
 import io.getquill.sql.norm.{
   HideTopLevelFilterAlias,
   NormalizeFilteredActionAliases,
   RemoveExtraAlias,
-  RemoveUnusedSelects
+  RemoveUnusedSelects,
+  SheathIdentContexts
 }
 import io.getquill.util.{Interleave, Interpolator, Messages, TraceConfig}
 import io.getquill.util.Messages.{TraceType, fail, trace}
@@ -89,7 +97,9 @@ trait SqlIdiom extends Idiom {
           VerifySqlQuery(sql).map(fail)
           val expanded = ExpandNestedQueries(sql, topLevelQuat)
           trace"Expanded SQL: ${expanded}".andLog()
-          val refined = if (Messages.pruneColumns) RemoveUnusedSelects(expanded) else expanded
+          val sheathed = SheathIdentContexts(expanded, topLevelQuat)
+          trace"Sheathed-Clause SQL: ${sheathed}".andLog()
+          val refined = if (Messages.pruneColumns) RemoveUnusedSelects(sheathed) else expanded
           trace"Filtered SQL (only used selects): ${refined}".andLog()
           val cleaned = if (!Messages.alwaysAlias) RemoveExtraAlias(naming)(refined, topLevelQuat) else refined
           trace"Cleaned SQL: ${cleaned}".andLog()

quill-engine/src/main/scala/io/getquill/sql/norm/SheathIdentContexts.scala

@@ -0,0 +1,67 @@
+package io.getquill.sql.norm
+
+import io.getquill.NamingStrategy
+import io.getquill.ast.Ast.LeafQuat
+import io.getquill.ast.{Ident, Property, Renameable}
+import io.getquill.context.sql._
+import io.getquill.quat.Quat
+
+object SheathIdentContexts extends StatelessQueryTransformer {
+
+  override protected def expandNested(q: FlattenSqlQuery, level: QueryLevel): FlattenSqlQuery = {
+    val from = q.from.map(expandContext(_))
+    val select = q.select.map { selectValue =>
+      selectValue.ast match {
+        case LeafQuat(origId: Ident) =>
+          val selects = findSelectsOfAlias(from, selectValue.alias.getOrElse(origId.name))
+          selects.map(_.alias).distinct match {
+            // if it's a single value of a single subselect eg:
+            // SELECT z FROM (SELECT x.i FROM foo) AS z)
+            // or
+            // SELECT z FROM (SELECT x.i FROM foo) UNION (SELECT y.i FROM bar) AS z)
+            case List(Some(value)) =>
+              // Then make it into:
+              // SELECT z.i FROM (SELECT x.i FROM foo) AS z.i)
+              // or
+              // SELECT z.i FROM (SELECT x.i FROM foo) UNION (SELECT y.i FROM bar) AS z.i)
+              // this z.i will be:
+              // Property(Ident(z, CC(i -> zOrig.quat), "i")
+              val newId = Ident(origId.name, Quat.Product("<single-prop-gen>", value -> origId.quat))
+              SelectValue(Property(newId, value))
+            case _ =>
+              selectValue
+          }
+
+        case _ => selectValue
+      }
+    }
+    q.copy(select = select, from = from)(q.quat)
+  }
+
+  // find selects of a query aliased as X
+  protected def findSelectsOfAlias(contexts: List[FromContext], alias: String): List[SelectValue] = {
+    def handleQuery(q: SqlQuery, queryAlias: String): List[SelectValue] =
+      q match {
+        case SetOperationSqlQuery(a, _, b) => handleQuery(a, queryAlias) ++ handleQuery(b, queryAlias)
+        case UnaryOperationSqlQuery(_, q)  => handleQuery(q, queryAlias)
+        case flatten: FlattenSqlQuery      => flatten.select
+      }
+
+    contexts.flatMap {
+      case QueryContext(q, queryAlias) =>
+        if (queryAlias == alias)
+          handleQuery(q, queryAlias)
+        else
+          List()
+
+      case JoinContext(_, a, b, _) =>
+        findSelectsOfAlias(List(a), alias) ++ findSelectsOfAlias(List(b), alias)
+
+      case FlatJoinContext(_, a, _) =>
+        findSelectsOfAlias(List(a), alias)
+
+      // table or infix would be a single variable, not sure can't do anything in that case
+      case _: TableContext | _: InfixContext => List()
+    }
+  }
+}

quill-engine/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala

@@ -55,16 +55,15 @@ class SqlNormalize(
       .andThen(demarcate("RenameProperties"))
       .andThen(ExpandDistinctPhase.apply _)
       .andThen(demarcate("ExpandDistinct"))
-      .andThen(NormalizePhase.apply _)
-      .andThen(demarcate("Normalize")) // Needed only because ExpandDistinct introduces an alias.
-      .andThen(NormalizePhase.apply _)
+      .andThen(demarcate("Normalize"))
+      .andThen(NormalizePhase.apply _) // Needed only because ExpandDistinct introduces an alias.
       .andThen(demarcate("Normalize"))
       .andThen(ExpandJoinPhase.apply _)
       .andThen(demarcate("ExpandJoin"))
       .andThen(ExpandMappedInfix.apply _)
       .andThen(demarcate("ExpandMappedInfix"))
-      .andThen(SheathLeafClausesPhase.apply _)
-      .andThen(demarcate("SheathLeaves"))
+//      .andThen(SheathLeafClausesPhase.apply _)
+//      .andThen(demarcate("SheathLeaves"))
       .andThen { ast =>
         // In the final stage of normalization, change all temporary aliases into
         // shorter ones of the form x[0-9]+.

quill-sql-test/src/test/scala/io/getquill/context/sql/AggregationSpec.scala

@@ -5,7 +5,7 @@ import io.getquill.norm.{DisablePhase, OptionalPhase}
 import io.getquill.{Literal, MirrorSqlDialect, SqlMirrorContext, TestEntities}
 import io.getquill.norm.ConfigList._
 
-class AggregationSpec extends Spec {
+class AggregationSpec extends Spec { //
   case class Person(id: Int, name: String, age: Int)
   case class PersonOpt(name: Option[String], age: Int)
 

quill-sql-test/src/test/scala/io/getquill/context/sql/SqlQuerySpec.scala

@@ -8,7 +8,7 @@ import io.getquill.base.Spec
 import io.getquill.context.sql.util.StringOps._
 import io.getquill.util.TraceConfig
 
-class SqlQuerySpec extends Spec {
+class SqlQuerySpec extends Spec { //
 
   implicit val naming: Literal = new Literal {}
 
@@ -613,7 +613,7 @@ class SqlQuerySpec extends Spec {
 
       // If you look inside BetaReduction, you will see that tuple values that are the same are collapsed via 'distinct'.
       // In this case, use different values that do not allow this to happen
-      "with map query inside join with non-distinct tuple" in {
+      "with map query inside join with non-distinct tuple" in { //
         val q = quote {
           qr1
             .join(
@@ -622,9 +622,11 @@ class SqlQuerySpec extends Spec {
                 .distinct
             )
             .on((a, b) => a.i == b._1)
-        }
+        }.dynamic //
+        println(testContext.run(q))
         testContext.run(q).string mustEqual
           "SELECT a.s, a.i, a.l, a.o, a.b, q21._1, q21._2 FROM TestEntity a INNER JOIN (SELECT DISTINCT q2.i AS _1, q2.l AS _2 FROM TestEntity2 q2) AS q21 ON a.i = q21._1"
+        // SELECT a.s, a.i, a.l, a.o, a.b, q2._1, q2._2 FROM TestEntity a INNER JOIN (SELECT DISTINCT q2.i AS _1, q2.l AS _2 FROM TestEntity2 q2) AS q21 ON a.i = q21._1
       }
 
       "with map query inside join with non-distinct tuple with operation" in {

quill-sql/src/test/scala/io/getquill/context/sql/idiom/SqlIdiomTestSpec.scala

@@ -0,0 +1,45 @@
+package io.getquill.context.sql.idiom
+
+import io.getquill.ReturnAction.ReturnColumns
+import io.getquill.{Action, Literal, MirrorSqlDialectWithReturnMulti, Ord, PostgresDialect, Query, SqlMirrorContext}
+import io.getquill.context.mirror.Row
+
+object SqlIdiomTestSpec {
+
+  case class TestEntity(s: String, i: Int, l: Long, o: Option[Int], b: Boolean)
+  case class TestEntity2(s: String, i: Int, l: Long, o: Option[Int])
+
+  val ctx = new SqlMirrorContext(PostgresDialect, Literal)
+  import ctx._
+
+  val qr1 = quote {
+    query[TestEntity]
+  }
+  val qr2 = quote {
+    query[TestEntity2]
+  }
+
+  def main(args: Array[String]): Unit = {
+//    System.setProperty("quill.trace.enabled", "true")
+//    System.setProperty("quill.trace.color", "true")
+//    System.setProperty("quill.trace.quat", "full")
+//    System.setProperty("quill.trace.types", "all")
+//    io.getquill.util.Messages.resetCache() //
+
+    // val q = quote {
+    //  for {
+    //    v1 <- qr1.map(x => x.i).distinct
+    //    v2 <- qr2.filter(_.i == v1)
+    //  } yield (v1, v2)
+    // }.dynamic
+    // println(ctx.run(q).string)
+    // "SELECT i._1, x1.s, x1.i, x1.l, x1.o FROM (SELECT DISTINCT i.i AS _1 FROM TestEntity i) AS i, TestEntity2 x1 WHERE x1.i = i._1"
+
+    val q = quote {
+      qr1.map(x => x.i).nested
+    }.dynamic
+
+    println(run(q).string) //
+  }
+
+}