From 48ca20464cc5eb9d58e79fe189f5f210c9c90f8f Mon Sep 17 00:00:00 2001 From: Alexander Ioffe Date: Mon, 6 Nov 2023 01:50:49 -0500 Subject: [PATCH] experimenting --- .../scala/io/getquill/quat/QuatMaking.scala | 15 +++- .../io/getquill/norm/SheathLeafClauses.scala | 12 +++- .../main/scala/io/getquill/sql/SqlQuery.scala | 14 +++- .../io/getquill/sql/idiom/SqlIdiom.scala | 16 ++++- .../sql/norm/SheathIdentContexts.scala | 67 ++++++++++++++++++ .../io/getquill/sql/norm/SqlNormalize.scala | 9 ++- quill-jdbc-test-sqlite/quill_test.db | Bin 61440 -> 61440 bytes .../context/sql/AggregationSpec.scala | 2 +- .../getquill/context/sql/SqlQuerySpec.scala | 8 ++- .../context/sql/idiom/SqlIdiomTestSpec.scala | 45 ++++++++++++ 10 files changed, 170 insertions(+), 18 deletions(-) create mode 100644 quill-engine/src/main/scala/io/getquill/sql/norm/SheathIdentContexts.scala create mode 100644 quill-sql/src/test/scala/io/getquill/context/sql/idiom/SqlIdiomTestSpec.scala diff --git a/quill-core/src/main/scala/io/getquill/quat/QuatMaking.scala b/quill-core/src/main/scala/io/getquill/quat/QuatMaking.scala index ec37286f58..8a39285c6a 100644 --- a/quill-core/src/main/scala/io/getquill/quat/QuatMaking.scala +++ b/quill-core/src/main/scala/io/getquill/quat/QuatMaking.scala @@ -252,10 +252,22 @@ trait QuatMakingBase extends MacroUtilUniverse { } } + def isPrimitive(tpe: Type): Boolean = + tpe <:< typeOf[String] || + tpe <:< typeOf[Int] || + tpe <:< typeOf[Short] || + tpe <:< typeOf[Long] || + tpe <:< typeOf[Boolean] || + tpe <:< typeOf[Byte] || + tpe <:< typeOf[Float] || + tpe <:< typeOf[Double] + object DefiniteValue { def unapply(tpe: Type): Option[Type] = // UDTs (currently only used by cassandra) are created as tables even though there is an encoder for them. - if (tpe <:< typeOf[Udt]) + if (isPrimitive(tpe)) + Some(tpe) + else if (tpe <:< typeOf[Udt]) None else if (isType[AnyVal](tpe)) Some(tpe) @@ -358,7 +370,6 @@ trait QuatMakingBase extends MacroUtilUniverse { // Otherwise it's a terminal value case _ => - println(Messages.qprint(s"Could not infer SQL-type of ${tpe}, assuming it is a Unknown Quat.")) Messages.trace(s"Could not infer SQL-type of ${tpe}, assuming it is a Unknown Quat.") Quat.Unknown } diff --git a/quill-engine/src/main/scala/io/getquill/norm/SheathLeafClauses.scala b/quill-engine/src/main/scala/io/getquill/norm/SheathLeafClauses.scala index 89aaff0516..a76092dad9 100644 --- a/quill-engine/src/main/scala/io/getquill/norm/SheathLeafClauses.scala +++ b/quill-engine/src/main/scala/io/getquill/norm/SheathLeafClauses.scala @@ -306,6 +306,11 @@ case class SheathLeafClauses(state: Option[String], traceConfig: TraceConfig) // Unfortunately however due to the ExpandJoin phase being non-well typed this would cause various kinds of queries to fail. Some // examples are in SqlQuerySpec. If ExpandJoin can be rewritten to be well-typed this approach can be re-examined. case Join(t, a, b, iA, iB, on) => + if (LeafQuat.unapply(iA).isDefined) { + val (a1, sa) = apply(a) + val iA1 = Ident(iA.name, a1.quat) + } + val (a1, sa) = apply(a) val (b1, sb) = apply(b) val (iA1, iB1) = (Ident(iA.name, a1.quat), Ident(iB.name, b1.quat)) @@ -317,12 +322,13 @@ case class SheathLeafClauses(state: Option[String], traceConfig: TraceConfig) (Join(t, a1m, b1m, iA, iB, on), SheathLeafClauses(None, traceConfig)) } - case Filter(ent, e, LeafQuat(body)) => + // For filter clauses the body is always a quat:V so we check the ident to see if it's something that needs to be sheathed + case Filter(ent, LeafQuat(e: Ident), body) => val (ent1, s) = apply(ent) val e1 = Ident(e.name, ent1.quat) - // For filter clauses we want to go from: Filter(M(ent,e,e.v),e == 123) to Filter(M(ent,e,CC(v->e.v)),e,e.v == 123) + // For filter clauses we want to go from: Filter(M(ent,e,e.v),x == 123) to Filter(M(ent,e,CC(v->e.v)),x,x.v == 123) // the body should not be re-sheathed since a body of CC(v->e.v == 123) would make no sense since - // that's not even a boolean. Instead we just need to do e.v == 123. + // that's not even a boolean. Instead we just need to do x.v == 123. val bodyC = elaborateSheath(body)(s.state, e, e1) trace"Sheath Filter(qry) with $stateInfo in $qq becomes" andReturn { (Filter(ent1, e1, bodyC), s) diff --git a/quill-engine/src/main/scala/io/getquill/sql/SqlQuery.scala b/quill-engine/src/main/scala/io/getquill/sql/SqlQuery.scala index 61cfc9b85c..b9ec78aecd 100644 --- a/quill-engine/src/main/scala/io/getquill/sql/SqlQuery.scala +++ b/quill-engine/src/main/scala/io/getquill/sql/SqlQuery.scala @@ -228,10 +228,11 @@ class SqlQueryApply(traceConfig: TraceConfig) { } val collectedAliases = aliases(ctx).map { case (a, quat) => Ident(a, quat) } val select = Tuple(collectedAliases) - FlattenSqlQuery( + val o = FlattenSqlQuery( from = ctx :: Nil, select = List(SelectValue(select, None)) )(q.quat) + o } case q @ (_: Filter | _: Entity) => trace"base| Flattening Filter/Entity $q" andReturn { flatten(sources, q, alias, nestNextMap) } @@ -349,6 +350,17 @@ class SqlQueryApply(traceConfig: TraceConfig) { val agg = b.select.collect { case s @ SelectValue(_: Aggregation, _, _) => s } + // if it's not distinct and there's no aggregation and there's no applicative join + // the we can flatten out the inner context. Note that inner applicative-joins are complicated + // because they could have wonky inner alises declared so we need to be aggresive about + // nesting them. +// if (q.isInstanceOf[Join]) +// trace"Flattening| Map(Ident) [Join]" andReturn +// FlattenSqlQuery( +// from = QueryContext(apply(q), q.asInstanceOf[Join].) :: Nil, +// select = selectValues(p) +// )(quat) +// else if (!b.distinct.isDistinct && agg.isEmpty) trace"Flattening| Map(Ident) [Simple]" andReturn b.copy(select = selectValues(p))(quat) diff --git a/quill-engine/src/main/scala/io/getquill/sql/idiom/SqlIdiom.scala b/quill-engine/src/main/scala/io/getquill/sql/idiom/SqlIdiom.scala index bf5289f279..e4cc57446e 100644 --- a/quill-engine/src/main/scala/io/getquill/sql/idiom/SqlIdiom.scala +++ b/quill-engine/src/main/scala/io/getquill/sql/idiom/SqlIdiom.scala @@ -15,13 +15,21 @@ import io.getquill.idiom.StatementInterpolator._ import io.getquill.idiom._ import io.getquill.norm.ConcatBehavior.AnsiConcat import io.getquill.norm.EqualityBehavior.AnsiEquality -import io.getquill.norm.{ConcatBehavior, EqualityBehavior, ExpandReturning, NormalizeCaching, ProductAggregationToken} +import io.getquill.norm.{ + ConcatBehavior, + EqualityBehavior, + ExpandReturning, + NormalizeCaching, + ProductAggregationToken, + SheathLeafClauses +} import io.getquill.quat.Quat import io.getquill.sql.norm.{ HideTopLevelFilterAlias, NormalizeFilteredActionAliases, RemoveExtraAlias, - RemoveUnusedSelects + RemoveUnusedSelects, + SheathIdentContexts } import io.getquill.util.{Interleave, Interpolator, Messages, TraceConfig} import io.getquill.util.Messages.{TraceType, fail, trace} @@ -89,7 +97,9 @@ trait SqlIdiom extends Idiom { VerifySqlQuery(sql).map(fail) val expanded = ExpandNestedQueries(sql, topLevelQuat) trace"Expanded SQL: ${expanded}".andLog() - val refined = if (Messages.pruneColumns) RemoveUnusedSelects(expanded) else expanded + val sheathed = SheathIdentContexts(expanded, topLevelQuat) + trace"Sheathed-Clause SQL: ${sheathed}".andLog() + val refined = if (Messages.pruneColumns) RemoveUnusedSelects(sheathed) else expanded trace"Filtered SQL (only used selects): ${refined}".andLog() val cleaned = if (!Messages.alwaysAlias) RemoveExtraAlias(naming)(refined, topLevelQuat) else refined trace"Cleaned SQL: ${cleaned}".andLog() diff --git a/quill-engine/src/main/scala/io/getquill/sql/norm/SheathIdentContexts.scala b/quill-engine/src/main/scala/io/getquill/sql/norm/SheathIdentContexts.scala new file mode 100644 index 0000000000..6dc0fb5063 --- /dev/null +++ b/quill-engine/src/main/scala/io/getquill/sql/norm/SheathIdentContexts.scala @@ -0,0 +1,67 @@ +package io.getquill.sql.norm + +import io.getquill.NamingStrategy +import io.getquill.ast.Ast.LeafQuat +import io.getquill.ast.{Ident, Property, Renameable} +import io.getquill.context.sql._ +import io.getquill.quat.Quat + +object SheathIdentContexts extends StatelessQueryTransformer { + + override protected def expandNested(q: FlattenSqlQuery, level: QueryLevel): FlattenSqlQuery = { + val from = q.from.map(expandContext(_)) + val select = q.select.map { selectValue => + selectValue.ast match { + case LeafQuat(origId: Ident) => + val selects = findSelectsOfAlias(from, selectValue.alias.getOrElse(origId.name)) + selects.map(_.alias).distinct match { + // if it's a single value of a single subselect eg: + // SELECT z FROM (SELECT x.i FROM foo) AS z) + // or + // SELECT z FROM (SELECT x.i FROM foo) UNION (SELECT y.i FROM bar) AS z) + case List(Some(value)) => + // Then make it into: + // SELECT z.i FROM (SELECT x.i FROM foo) AS z.i) + // or + // SELECT z.i FROM (SELECT x.i FROM foo) UNION (SELECT y.i FROM bar) AS z.i) + // this z.i will be: + // Property(Ident(z, CC(i -> zOrig.quat), "i") + val newId = Ident(origId.name, Quat.Product("", value -> origId.quat)) + SelectValue(Property(newId, value)) + case _ => + selectValue + } + + case _ => selectValue + } + } + q.copy(select = select, from = from)(q.quat) + } + + // find selects of a query aliased as X + protected def findSelectsOfAlias(contexts: List[FromContext], alias: String): List[SelectValue] = { + def handleQuery(q: SqlQuery, queryAlias: String): List[SelectValue] = + q match { + case SetOperationSqlQuery(a, _, b) => handleQuery(a, queryAlias) ++ handleQuery(b, queryAlias) + case UnaryOperationSqlQuery(_, q) => handleQuery(q, queryAlias) + case flatten: FlattenSqlQuery => flatten.select + } + + contexts.flatMap { + case QueryContext(q, queryAlias) => + if (queryAlias == alias) + handleQuery(q, queryAlias) + else + List() + + case JoinContext(_, a, b, _) => + findSelectsOfAlias(List(a), alias) ++ findSelectsOfAlias(List(b), alias) + + case FlatJoinContext(_, a, _) => + findSelectsOfAlias(List(a), alias) + + // table or infix would be a single variable, not sure can't do anything in that case + case _: TableContext | _: InfixContext => List() + } + } +} diff --git a/quill-engine/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala b/quill-engine/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala index 9d1622f686..3b099da45d 100644 --- a/quill-engine/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala +++ b/quill-engine/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala @@ -55,16 +55,15 @@ class SqlNormalize( .andThen(demarcate("RenameProperties")) .andThen(ExpandDistinctPhase.apply _) .andThen(demarcate("ExpandDistinct")) - .andThen(NormalizePhase.apply _) - .andThen(demarcate("Normalize")) // Needed only because ExpandDistinct introduces an alias. - .andThen(NormalizePhase.apply _) + .andThen(demarcate("Normalize")) + .andThen(NormalizePhase.apply _) // Needed only because ExpandDistinct introduces an alias. .andThen(demarcate("Normalize")) .andThen(ExpandJoinPhase.apply _) .andThen(demarcate("ExpandJoin")) .andThen(ExpandMappedInfix.apply _) .andThen(demarcate("ExpandMappedInfix")) - .andThen(SheathLeafClausesPhase.apply _) - .andThen(demarcate("SheathLeaves")) +// .andThen(SheathLeafClausesPhase.apply _) +// .andThen(demarcate("SheathLeaves")) .andThen { ast => // In the final stage of normalization, change all temporary aliases into // shorter ones of the form x[0-9]+. diff --git a/quill-jdbc-test-sqlite/quill_test.db b/quill-jdbc-test-sqlite/quill_test.db index 89d70db5078775ed3249186d92b00b4a363df242..25e45341e23f95c60c629c22bbd41ab1b73326d5 100644 GIT binary patch delta 1689 zcmZuxO>7%Q6rP#=@j7eI+caIQD7bMDh?aK2>$TT0R5W&z93r48p%N0kti9{T;M!&F zs6j}vTj)LQ0TE6`0;v~7n@SY|dMIjd0i{B%s65=P2Y!I>;PUx??JP_yvY_Kh z-*a~>60;`UilZwcv&Mbz=&&L%Yj4$eMij!VoafgK1v4vK_U3no5kdmoK=2p*3OC>v zxCz%8>o@ScBnv8As&hI{uCmoS=i#g>BhWbkr{f$VTtn~=+=W|kjjg`|djcYf%s6GQ z)+dnYt6t4>{K9OIkfHr^PPtTnCy2?=v|n}S3hM+iEFrkZmao8(xq!Dr&HOQalEVphtPk$=B+>!|DsL zWUjH546I*K4m)wywn47A6%VoAML5jQmC7!#CPW!}u0t1Morv&q*5e`8iU_Y|9r0dZ zxQO5)6MY``Ww9!W692IeFAQ!!-Naud*O+y67oRudmY2=s+nYyPS)7spK`;e`kv^b} zNwO`hv`0t@iw9Un_rP1uG~Hq*k+k!cVHGUXNG1veV>IR2Mln%{r&Dg)@$!XZ7TyMd zAjX&kuqJ_YU=ynCxxd86M`~mQe?FGAtTDDv;~7kF?s&F|YB=}aYulSvI`3ridE1D` z?TnE$Q*mQ7nX-(GU9^jqT^zO2ssE?i%#=#x^N#DJjCekiGLpqY!Wb>OmSMZ5okN^)gqmVbd&%7R_EOtBfTxWN<)?^C7Y;5bGD-I#n$ z a.i == b._1) - } + }.dynamic // + println(testContext.run(q)) testContext.run(q).string mustEqual "SELECT a.s, a.i, a.l, a.o, a.b, q21._1, q21._2 FROM TestEntity a INNER JOIN (SELECT DISTINCT q2.i AS _1, q2.l AS _2 FROM TestEntity2 q2) AS q21 ON a.i = q21._1" + // SELECT a.s, a.i, a.l, a.o, a.b, q2._1, q2._2 FROM TestEntity a INNER JOIN (SELECT DISTINCT q2.i AS _1, q2.l AS _2 FROM TestEntity2 q2) AS q21 ON a.i = q21._1 } "with map query inside join with non-distinct tuple with operation" in { diff --git a/quill-sql/src/test/scala/io/getquill/context/sql/idiom/SqlIdiomTestSpec.scala b/quill-sql/src/test/scala/io/getquill/context/sql/idiom/SqlIdiomTestSpec.scala new file mode 100644 index 0000000000..9941fcacc0 --- /dev/null +++ b/quill-sql/src/test/scala/io/getquill/context/sql/idiom/SqlIdiomTestSpec.scala @@ -0,0 +1,45 @@ +package io.getquill.context.sql.idiom + +import io.getquill.ReturnAction.ReturnColumns +import io.getquill.{Action, Literal, MirrorSqlDialectWithReturnMulti, Ord, PostgresDialect, Query, SqlMirrorContext} +import io.getquill.context.mirror.Row + +object SqlIdiomTestSpec { + + case class TestEntity(s: String, i: Int, l: Long, o: Option[Int], b: Boolean) + case class TestEntity2(s: String, i: Int, l: Long, o: Option[Int]) + + val ctx = new SqlMirrorContext(PostgresDialect, Literal) + import ctx._ + + val qr1 = quote { + query[TestEntity] + } + val qr2 = quote { + query[TestEntity2] + } + + def main(args: Array[String]): Unit = { +// System.setProperty("quill.trace.enabled", "true") +// System.setProperty("quill.trace.color", "true") +// System.setProperty("quill.trace.quat", "full") +// System.setProperty("quill.trace.types", "all") +// io.getquill.util.Messages.resetCache() // + + // val q = quote { + // for { + // v1 <- qr1.map(x => x.i).distinct + // v2 <- qr2.filter(_.i == v1) + // } yield (v1, v2) + // }.dynamic + // println(ctx.run(q).string) + // "SELECT i._1, x1.s, x1.i, x1.l, x1.o FROM (SELECT DISTINCT i.i AS _1 FROM TestEntity i) AS i, TestEntity2 x1 WHERE x1.i = i._1" + + val q = quote { + qr1.map(x => x.i).nested + }.dynamic + + println(run(q).string) // + } + +}