From d873ba5cd75c3f9e38d7bb1b2d4802a0cdc4c1f6 Mon Sep 17 00:00:00 2001 From: Jolse Maginnis Date: Sat, 15 Jun 2019 19:02:34 +1000 Subject: [PATCH] Add in clauses, with streaming --- .gitignore | 3 +- .../scala/io/doolse/simpledba/Columns.scala | 5 +- .../scala/io/doolse/simpledba/Effects.scala | 1 + .../io/doolse/simpledba/fs2/package.scala | 2 + .../doolse/simpledba/jdbc/JDBCQueries.scala | 204 ++++++++++++------ .../io/doolse/simpledba/jdbc/JDBCSQL.scala | 3 +- .../io/doolse/simpledba/jdbc/JDBCTable.scala | 10 +- .../io/doolse/simpledba/jdbc/SQLDialect.scala | 1 + .../doolse/simpledba/jdbc/StdSQLDialect.scala | 5 +- jdbc/src/test/resources/reference.conf | 3 + .../test/jdbc/JDBCExpressionProperties.scala | 188 +++++++++++++++- .../simpledba/test/jdbc/JDBCProperties.scala | 32 ++- .../simpledba/test/jdbc/JDBCTester.scala | 2 +- .../simpledba/test/jdbc/JDBCZIOTester.scala | 2 +- .../doolse/simpledba/ziointerop/package.scala | 3 + 15 files changed, 371 insertions(+), 93 deletions(-) diff --git a/.gitignore b/.gitignore index e673575..98f08ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea/ -target/ \ No newline at end of file +target/ +application.conf diff --git a/core/src/main/scala/io/doolse/simpledba/Columns.scala b/core/src/main/scala/io/doolse/simpledba/Columns.scala index c0f41c4..0a93e4a 100644 --- a/core/src/main/scala/io/doolse/simpledba/Columns.scala +++ b/core/src/main/scala/io/doolse/simpledba/Columns.scala @@ -5,7 +5,7 @@ import cats.instances.function._ import cats.syntax.compose._ import shapeless.labelled.FieldType import shapeless.ops.hlist.{Length, LiftAll, Prepend, Split, ToList, ZipWithKeys} -import shapeless.ops.record.{Keys, SelectAll} +import shapeless.ops.record.{Keys, SelectAll, Selector} import shapeless.{::, HList, HNil, Nat, Witness} import scala.annotation.tailrec @@ -160,6 +160,9 @@ case class Columns[C[_], T, R <: HList](columns: Seq[(String, C[_])], iso: Iso[T } def toSubset: ColumnSubset[C, T, R] = ColumnSubset(columns, iso.to) + + def singleColumn[A](col: Witness)(implicit selector: Selector.Aux[R, col.T, A], ev: col.T <:< Symbol): (String, C[A]) = + columns.find(_._1 == col.value.name).get.asInstanceOf[(String, C[A])] } object Columns { diff --git a/core/src/main/scala/io/doolse/simpledba/Effects.scala b/core/src/main/scala/io/doolse/simpledba/Effects.scala index 5817f44..ae2794f 100644 --- a/core/src/main/scala/io/doolse/simpledba/Effects.scala +++ b/core/src/main/scala/io/doolse/simpledba/Effects.scala @@ -30,6 +30,7 @@ trait Streamable[S[_], F[_]] { } SM.flatMap(s)(loop) } + def maxMapped[A, B](n: Int, s: S[A])(f: Seq[A] => B): S[B] def toVector[A](s: S[A]): F[Vector[A]] def last[A](s: S[A]): S[Option[A]] diff --git a/fs2/src/main/scala/io/doolse/simpledba/fs2/package.scala b/fs2/src/main/scala/io/doolse/simpledba/fs2/package.scala index b539f0f..aa18ed4 100644 --- a/fs2/src/main/scala/io/doolse/simpledba/fs2/package.scala +++ b/fs2/src/main/scala/io/doolse/simpledba/fs2/package.scala @@ -48,5 +48,7 @@ package object fs2 { override def bracket[A](acquire: F[A])(release: A => F[Unit]): Stream[F, A] = Stream.bracket(acquire)(release) + + override def maxMapped[A, B](n: Int, s: Stream[F, A])(f: Seq[A] => B): Stream[F, B] = s.chunkN(n).map(c => f(c.toVector)) } } diff --git a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCQueries.scala b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCQueries.scala index 06f34af..43d7f9c 100644 --- a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCQueries.scala +++ b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCQueries.scala @@ -2,19 +2,19 @@ package io.doolse.simpledba.jdbc import java.sql.{Connection, PreparedStatement} -import cats.data.{Kleisli, State} +import cats.data.{Kleisli, NonEmptyList, State} import io.doolse.simpledba._ import io.doolse.simpledba.jdbc.BinOp.BinOp import io.doolse.simpledba.jdbc.JDBCQueries._ import io.doolse.simpledba.jdbc.JDBCTable.TableRecord import shapeless.labelled._ import shapeless.ops.hlist.{Length, Prepend, Split, Take} -import shapeless.ops.record.{Keys, ToMap} +import shapeless.ops.record.{Keys, Selector, ToMap} import shapeless.{::, DepFn2, HList, HNil, LabelledGeneric, Nat, Witness, _0} import scala.annotation.tailrec -case class JDBCMapper[C[_] <: JDBCColumn[_]](dialect: SQLDialect) { +case class JDBCMapper[C[A] <: JDBCColumn[A]](dialect: SQLDialect) { def record[R <: HList](implicit cr: ColumnRecord[C, Unit, R]): ColumnRecord[C, Unit, R] = cr @@ -23,7 +23,7 @@ case class JDBCMapper[C[_] <: JDBCColumn[_]](dialect: SQLDialect) { def mapped[T] = new RelationBuilder[T, C] - class RelationBuilder[T, C[_] <: JDBCColumn[_]] { + class RelationBuilder[T, C[A] <: JDBCColumn[A]] { def embedded[GR <: HList, R <: HList]( implicit gen: LabelledGeneric.Aux[T, GR], @@ -65,7 +65,7 @@ object BindUpdate extends ColumnCompare[JDBCColumn, String, BoundColumn] { } } -case class JDBCQueries[C[_] <: JDBCColumn[_], S[_], F[_]](E: JDBCEffect[S, F], +case class JDBCQueries[C[A] <: JDBCColumn[A], S[_], F[_]](E: JDBCEffect[S, F], dialect: SQLDialect) { val S = E.S val SM = S.SM @@ -146,15 +146,15 @@ case class JDBCQueries[C[_] <: JDBCColumn[_], S[_], F[_]](E: JDBCEffect[S, F], E, ColumnRecord.empty, identity, - _ => (Seq.empty, Seq.empty), + _ => S.emit((Seq.empty, Seq.empty)), Seq.empty ) def deleteFrom(table: JDBCTable[C]) = - new DeleteBuilder[S, C, F, table.DataRec, HNil](E.S, + new DeleteBuilder[S, F, C, table.DataRec, HNil](E.S, table, dialect, - _ => (Seq.empty, Seq.empty)) + _ => S.emit((Seq.empty, Seq.empty))) def query(table: JDBCTable[C]) = new QueryBuilder[S, F, C, table.DataRec, HNil, table.DataRec, table.Data]( @@ -163,7 +163,7 @@ case class JDBCQueries[C[_] <: JDBCColumn[_], S[_], F[_]](E: JDBCEffect[S, F], E, toProjection(table.allColumns), table.allColumns.iso.from, - _ => (Seq.empty, Seq.empty), + _ => S.emit((Seq.empty, Seq.empty)), Seq.empty ) @@ -213,49 +213,135 @@ object JDBCQueries { } } - case class DeleteBuilder[S[_], C[_] <: JDBCColumn[_], F[_], DataRec <: HList, InRec <: HList]( - S: Streamable[S, F], - table: TableRecord[C, DataRec], - dialect: SQLDialect, - toWhere: InRec => (Seq[JDBCWhereClause], Seq[BoundValue]) - ) { - def where[W2 <: HList, ColNames <: HList](cols: Cols[ColNames], op: BinOp)( - implicit css: ColumnSubsetBuilder.Aux[DataRec, ColNames, W2], - ): DeleteBuilder[S, C, F, DataRec, W2 :: InRec] = ??? + case class DeleteBuilder[S[_], F[_], C[A] <: JDBCColumn[A], DataRec <: HList, InRec <: HList]( + private[jdbc] val S: Streamable[S, F], + private[jdbc] val table: TableRecord[C, DataRec], + private[jdbc] val dialect: SQLDialect, + private[jdbc] val toWhere: InRec => S[(Seq[JDBCWhereClause], Seq[BoundValue])] + ) extends WhereBuilder[S, F, C, DataRec, InRec] { + type WhereOut[NewIn <: HList] = DeleteBuilder[S, F, C, DataRec, NewIn] + def build[W2]( + implicit + c: AutoConvert[W2, InRec]): W2 => S[WriteOp] = w => { + S.SM.map(toWhere(c(w))) { + case (where, values) => + val deleteSQL = dialect.querySQL(JDBCDelete(table.name, where)) + JDBCWriteOp(deleteSQL, bindParameters(values)) + } + } - def where[W2 <: HList](col: Witness, op: BinOp)( - implicit cols: ColumnSubsetBuilder.Aux[DataRec, col.T :: HNil, W2] - ): DeleteBuilder[S, C, F, DataRec, W2 :: InRec] = ??? + override def withToWhere[NewIn <: HList](f: NewIn => S[(Seq[JDBCWhereClause], Seq[BoundValue])]) + : DeleteBuilder[S, F, C, DataRec, NewIn] = copy(toWhere = f) + } - def build[W2]( - c: AutoConvert[W2, InRec] - ): W2 => S[WriteOp] = w => { - val (where, values) = toWhere(c(w)) - S.emit { - val deleteSQL = dialect.querySQL(JDBCDelete(table.name, where)) - JDBCWriteOp(deleteSQL, bindParameters(values)) + trait WhereBuilder[S[_], F[_], C[A] <: JDBCColumn[A], DataRec <: HList, InRec <: HList] { + private[jdbc] def S: Streamable[S, F] + type WhereOut[NewIn <: HList] + + private[jdbc] def dialect: SQLDialect + private[jdbc] def table: TableRecord[C, DataRec] + private[jdbc] def toWhere: InRec => S[(Seq[JDBCWhereClause], Seq[BoundValue])] + + protected def withToWhere[NewIn <: HList]( + f: NewIn => S[(Seq[JDBCWhereClause], Seq[BoundValue])]): WhereOut[NewIn] + + protected def addWhere[NewIn <: HList]( + f: NewIn => (InRec, Seq[JDBCWhereClause], Seq[BoundValue])): WhereOut[NewIn] = withToWhere { + newIn => + val (oldIn, newClause, newBind) = f(newIn) + S.SM.map(this.toWhere(oldIn)) { + case (oldClause, oldBind) => (oldClause ++ newClause, oldBind ++ newBind) + } + } + + def whereInNotEmpty[A, NewIn <: HList, WLen <: Nat, K <: Symbol](whereCol: Witness.Aux[K])( + implicit + select: Selector.Aux[DataRec, K, A], + prepend: Prepend.Aux[InRec, NonEmptyList[A] :: HNil, NewIn], + length: Length.Aux[InRec, WLen], + split: Split.Aux[NewIn, WLen, InRec, NonEmptyList[A] :: HNil] + ): WhereOut[NewIn] = { + val (colName, inCol) = table.allColumns.singleColumn(whereCol) + addWhere { newIn => + val (oldWhere, newWhere) = split(newIn) + val (newClause, bindVals) = mkInClause(colName, inCol, newWhere.head.toList) + (oldWhere, newClause, bindVals) + } + } + + private def mkInClause[A](colName: String, inCol: C[A], vals: Seq[A]) = { + val bindVals = vals.map(inCol.bindValue) + val clause = BinClause(ColumnReference(NamedColumn(colName, inCol.columnType)), + BinOp.IN, + Expressions(bindVals.map(_ => Parameter(inCol.columnType)))) + (Seq(clause), bindVals) + } + + def whereIn[A, NewIn <: HList, WLen <: Nat](whereCol: Witness)( + implicit + select: Selector.Aux[DataRec, whereCol.T, A], + ev: whereCol.T <:< Symbol, + prepend: Prepend.Aux[InRec, S[A] :: HNil, NewIn], + length: Length.Aux[InRec, WLen], + split: Split.Aux[NewIn, WLen, InRec, S[A] :: HNil] + ): WhereOut[NewIn] = { + val (colName, inCol) = table.allColumns.singleColumn(whereCol) + withToWhere { newIn => + val (oldWhere, newWhere) = split(newIn) + S.SM.flatMap(toWhere(oldWhere)) { + case (oldClause, oldBind) => + S.maxMapped(dialect.maxInParamaters, newWhere.head) { vals => + val (newClause, newBind) = mkInClause(colName, inCol, vals) + (oldClause ++ newClause, oldBind ++ newBind) + } + } + } + } + + def where[WhereVals <: HList, NewIn <: HList, WLen <: Nat](whereCol: Witness, binOp: BinOp)( + implicit + csb: ColumnSubsetBuilder.Aux[DataRec, whereCol.T :: HNil, WhereVals], + prepend: Prepend.Aux[InRec, WhereVals, NewIn], + length: Length.Aux[InRec, WLen], + split: Split.Aux[NewIn, WLen, InRec, WhereVals] + ): WhereOut[NewIn] = where(Cols(whereCol), binOp) + + def where[ColNames <: HList, WhereVals <: HList, NewIn <: HList, WLen <: Nat]( + whereCols: Cols[ColNames], + binOp: BinOp)( + implicit + css: ColumnSubsetBuilder.Aux[DataRec, ColNames, WhereVals], + len: Length.Aux[InRec, WLen], + prepend: Prepend.Aux[InRec, WhereVals, NewIn], + split: Split.Aux[NewIn, WLen, InRec, WhereVals] + ): WhereOut[NewIn] = { + val whereCols = table.allColumns.subset(css) + addWhere { newIn => + val (oldWhere, newWhere) = split(newIn) + val opWheres = colsOp(binOp, whereCols).apply(newWhere) + (oldWhere, opWheres.map(_._1), opWheres.map(_._2)) } } } case class QueryBuilder[S[_], F[_], - C[_] <: JDBCColumn[_], + C[A] <: JDBCColumn[A], DataRec <: HList, InRec <: HList, OutRec <: HList, Out]( - table: TableRecord[C, DataRec], - dialect: SQLDialect, - E: JDBCEffect[S, F], - projections: ColumnRecord[C, SQLProjection, OutRec], - mapOut: OutRec => Out, - toWhere: InRec => (Seq[JDBCWhereClause], Seq[BoundValue]), - orderCols: Seq[(NamedColumn, Boolean)] - ) { - - val S = E.S - val SM = S.SM + private[jdbc] val table: TableRecord[C, DataRec], + private[jdbc] val dialect: SQLDialect, + private[jdbc] val E: JDBCEffect[S, F], + private[jdbc] val projections: ColumnRecord[C, SQLProjection, OutRec], + private[jdbc] val mapOut: OutRec => Out, + private[jdbc] val toWhere: InRec => S[(Seq[JDBCWhereClause], Seq[BoundValue])], + private[jdbc] val orderCols: Seq[(NamedColumn, Boolean)] + ) extends WhereBuilder[S, F, C, DataRec, InRec] { + type WhereOut[NewIn <: HList] = QueryBuilder[S, F, C, DataRec, NewIn, OutRec, Out] + private[jdbc] val S = E.S + private[jdbc] val SM = S.SM def count( implicit intCol: C[Int] @@ -300,32 +386,6 @@ object JDBCQueries { copy(orderCols = cols) } - def where[WhereVals <: HList, NewIn <: HList, WLen <: Nat](whereCol: Witness, binOp: BinOp)( - implicit - csb: ColumnSubsetBuilder.Aux[DataRec, whereCol.T :: HNil, WhereVals], - prepend: Prepend.Aux[WhereVals, InRec, NewIn], - length: Length.Aux[WhereVals, WLen], - split: Split.Aux[NewIn, WLen, WhereVals, InRec] - ): QueryBuilder[S, F, C, DataRec, NewIn, OutRec, Out] = where(Cols(whereCol), binOp) - - def where[ColNames <: HList, WhereVals <: HList, NewIn <: HList, WLen <: Nat]( - whereCols: Cols[ColNames], - binOp: BinOp)( - implicit - css: ColumnSubsetBuilder.Aux[DataRec, ColNames, WhereVals], - len: Length.Aux[WhereVals, WLen], - prepend: Prepend.Aux[WhereVals, InRec, NewIn], - split: Split.Aux[NewIn, WLen, WhereVals, InRec] - ): QueryBuilder[S, F, C, DataRec, NewIn, OutRec, Out] = { - - copy(toWhere = newin => { - val (newWhere, oldWhere) = split(newin) - val res = colsOp(binOp, table.allColumns.subset(css)).apply(newWhere) - (res.map(_._1), res.map(_._2)) - }) - - } - def buildAs[In, Out2]( implicit c: AutoConvert[In, InRec], cout: AutoConvert[Out, Out2] @@ -337,10 +397,16 @@ object JDBCQueries { val baseSel = JDBCSelect(table.name, projections.columns.map(_._1), Seq.empty, orderCols, false) w2: In => - val (whereClauses, binds) = toWhere(c(w2)) - val selSQL = dialect.querySQL(baseSel.copy(where = whereClauses)) - SM.map(E.streamForQuery(selSQL, bindParameters(binds), projections))(mapOut) + SM.flatMap(toWhere(c(w2))) { + case (whereClauses, binds) => + val selSQL = dialect.querySQL(baseSel.copy(where = whereClauses)) + SM.map(E.streamForQuery(selSQL, bindParameters(binds), projections))(mapOut) + } } + + override def withToWhere[NewIn <: HList](f: NewIn => S[(Seq[JDBCWhereClause], Seq[BoundValue])]) + : QueryBuilder[S, F, C, DataRec, NewIn, OutRec, Out] = + copy(toWhere = f) } def bindParameters(params: Seq[BoundValue]): (Connection, PreparedStatement) => Seq[Any] = diff --git a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCSQL.scala b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCSQL.scala index bf8d5b4..d007b06 100644 --- a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCSQL.scala +++ b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCSQL.scala @@ -27,7 +27,7 @@ object AggregateOp extends Enumeration { object BinOp extends Enumeration { type BinOp = Value - val EQ, GT, GTE, LT, LTE, LIKE = Value + val EQ, GT, GTE, LT, LTE, LIKE, IN = Value } sealed trait SQLExpression @@ -37,6 +37,7 @@ case class Aggregate(name: AggregateOp, column: Option[NamedColumn]) extends SQL case class FunctionCall(name: String, params: Seq[SQLExpression]) extends SQLExpression case class SQLString(s: String) extends SQLExpression case class Parameter(columnType: ColumnType) extends SQLExpression +case class Expressions(expressions: Seq[SQLExpression]) extends SQLExpression case class ColumnExpression(column: NamedColumn, expression: SQLExpression) case class SQLProjection(columnType: ColumnType, sql: SQLExpression) diff --git a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCTable.scala b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCTable.scala index fe6071e..cacbc1f 100644 --- a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCTable.scala +++ b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/JDBCTable.scala @@ -3,7 +3,7 @@ package io.doolse.simpledba.jdbc import io.doolse.simpledba._ import shapeless.{::, HList, HNil, Witness} -case class JDBCRelation[C[_] <: JDBCColumn[_], T, R <: HList]( +case class JDBCRelation[C[A] <: JDBCColumn[A], T, R <: HList]( name: String, all: Columns[C, T, R] ) { @@ -33,7 +33,7 @@ case class JDBCRelation[C[_] <: JDBCColumn[_], T, R <: HList]( } } -trait JDBCTable[C[_] <: JDBCColumn[_]] { +trait JDBCTable[C[A] <: JDBCColumn[A]] { type Data type DataRec <: HList type KeyList <: HList @@ -66,18 +66,18 @@ trait JDBCTable[C[_] <: JDBCColumn[_]] { } object JDBCTable { - type Aux[C[_] <: JDBCColumn[_], T, R, K, KeyN] = JDBCTable[C] { + type Aux[C[A] <: JDBCColumn[A], T, R, K, KeyN] = JDBCTable[C] { type Data = T type DataRec = R type KeyNames = KeyN type KeyList = K } - type TableRecord[C[_] <: JDBCColumn[_], R] = JDBCTable[C] { + type TableRecord[C[A] <: JDBCColumn[A], R] = JDBCTable[C] { type DataRec = R } - def apply[C[_] <: JDBCColumn[_], T, R <: HList, K <: HList, KeyN <: HList]( + def apply[C[A] <: JDBCColumn[A], T, R <: HList, K <: HList, KeyN <: HList]( tableName: String, all: Columns[C, T, R], keys: ColumnSubsetBuilder.Aux[R, KeyN, K], diff --git a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/SQLDialect.scala b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/SQLDialect.scala index fffd939..3d5c894 100644 --- a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/SQLDialect.scala +++ b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/SQLDialect.scala @@ -10,4 +10,5 @@ trait SQLDialect { def addColumns(t: TableColumns): Seq[String] def truncateTable(t: TableDefinition): String def createIndex(t: TableColumns, named: String): String + def maxInParamaters: Int } diff --git a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/StdSQLDialect.scala b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/StdSQLDialect.scala index 9da4d61..d0d7120 100644 --- a/jdbc/src/main/scala/io/doolse/simpledba/jdbc/StdSQLDialect.scala +++ b/jdbc/src/main/scala/io/doolse/simpledba/jdbc/StdSQLDialect.scala @@ -9,6 +9,7 @@ trait StdSQLDialect extends SQLDialect { import StdSQLDialect._ override def querySQL(query: JDBCPreparedQuery): String = stdQuerySQL(query) + override def maxInParamaters: Int = 100 def expressionSQL(expression: SQLExpression): String = stdExpressionSQL(expression) def typeName(c: ColumnType, keyColumn: Boolean): String = stdTypeName(c, keyColumn) def orderBy(oc: Seq[(NamedColumn, Boolean)]): String = stdOrderBy(oc) @@ -18,10 +19,11 @@ trait StdSQLDialect extends SQLDialect { expr match { case ColumnReference(name) => escapeColumnName(name.name) case FunctionCall(name, params) => - s"$name${params.map(expressionSQL).mkString("(", ",", ")")}" + s"$name${brackets(params.map(expressionSQL))}" case SQLString(s) => s"'$s'" case Parameter(sqlType) => "?" case Aggregate(AggregateOp.Count, None) => "count(*)" + case Expressions(exprs) => brackets(exprs.map(expressionSQL)) } } @@ -60,6 +62,7 @@ trait StdSQLDialect extends SQLDialect { case BinOp.LT => "<" case BinOp.LTE => "<=" case BinOp.LIKE => "LIKE" + case BinOp.IN => "IN" } s"${expressionSQL(left)} $opString ${expressionSQL(right)}" } diff --git a/jdbc/src/test/resources/reference.conf b/jdbc/src/test/resources/reference.conf index 775e90c..83c80ee 100644 --- a/jdbc/src/test/resources/reference.conf +++ b/jdbc/src/test/resources/reference.conf @@ -6,4 +6,7 @@ simpledba { // password = localtest // } } + test { + log = false + } } \ No newline at end of file diff --git a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCExpressionProperties.scala b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCExpressionProperties.scala index 73a4609..6e7838c 100644 --- a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCExpressionProperties.scala +++ b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCExpressionProperties.scala @@ -1,8 +1,192 @@ package io.doolse.simpledba.test.jdbc +import java.util.UUID + +import io.doolse.simpledba.test.SimpleDBAProperties import zio.Task -import zio.stream.ZStream +import zio.stream.{ZSink, ZStream} +import org.scalacheck._ +import org.scalacheck.Prop._ +import Arbitrary._ +import io.doolse.simpledba.jdbc.BinOp +import io.doolse.simpledba.jdbc.BinOp.BinOp + +case class SimpleTable(id: UUID, value1: Int, value2: String) + +object JDBCExpressionProperties + extends SimpleDBAProperties("JDBC Expressions") + with JDBCProperties[ZStream[Any, Throwable, ?], Task] + with ZIOProperties { + + val simpleTable = mapper.mapped[SimpleTable].table("simpleTable").key('id) + + val writer = sqlQueries.writes(simpleTable) + setup(simpleTable) + + val genSimple: Gen[SimpleTable] = for { + id <- arbitrary[UUID] + value1 <- arbitrary[Int] + value2 <- Gen.alphaNumStr + } yield SimpleTable(id, value1, value2) + + val ops: Seq[BinOp] = Seq(BinOp.EQ, BinOp.GT, BinOp.GTE, BinOp.LT, BinOp.LTE) + + def filterOp[A](f: SimpleTable => A, op: BinOp, compare: A)( + implicit o: Ordering[A]): SimpleTable => Boolean = + t => { + val opf = op match { + case BinOp.EQ => o.equiv _ + case BinOp.GT => o.gt _ + case BinOp.GTE => o.gteq _ + case BinOp.LT => o.lt _ + case BinOp.LTE => o.lteq _ + } + opf(f(t), compare) + } + + def uniqueify(rows: Seq[SimpleTable], f: SimpleTable => Any) = + rows.map(t => f(t) -> t).toMap.values.toSeq + + def insertData(rows: Seq[SimpleTable]): Task[Seq[SimpleTable]] = { + val dupesRemoved = uniqueify(rows, _.id) + flushable + .flush(S + .emit(sqlQueries.rawSQL(sqlQueries.dialect.truncateTable(simpleTable.definition))) ++ writer + .insertAll(ZStream(dupesRemoved: _*))) + .runDrain + .map(_ => dupesRemoved) + } + + def compareResults(fromDB: Seq[SimpleTable], fromScala: Seq[SimpleTable]): Prop = { + fromDB.sortBy(_.id).toList ?= fromScala.sortBy(_.id).toList + } + + property("oneComparison") = forAll(Gen.listOfN(10, genSimple), arbitrary[Int], Gen.oneOf(ops)) { + (rawRows, compareInt, op) => + run { + for { + rows <- insertData(rawRows) + fromDB <- sqlQueries + .query(simpleTable) + .where('value1, op) + .build[Int] + .apply(compareInt) + .runCollect + } yield { + compareResults(fromDB, rows.filter(filterOp(_.value1, op, compareInt))).label(op.toString) + } + } + } + + property("twoComparisons") = forAll(Gen.listOfN(10, genSimple), + arbitrary[Int], + arbitrary[String], + Gen.oneOf(ops), + Gen.oneOf(ops)) { + (rawRows, compareInt, compareString, op1, op2) => + run { + for { + rows <- insertData(rawRows) + fromDB <- sqlQueries + .query(simpleTable) + .where('value1, op1) + .where('value2, op2) + .build[(Int, String)] + .apply((compareInt, compareString)) + .runCollect + } yield { + compareResults(fromDB, rows.filter { r => + filterOp(_.value1, op1, compareInt).apply(r) && + filterOp(_.value2, op2, compareString).apply(r) + }).label(s"${op1.toString} and ${op2.toString}") + } + } + } + + property("deleteOneComparison") = + forAll(Gen.listOfN(10, genSimple), arbitrary[Int], Gen.oneOf(ops)) { + (rawRows, compareInt, op) => + run { + for { + rows <- insertData(rawRows) + _ <- flushable + .flush( + sqlQueries + .deleteFrom(simpleTable) + .where('value1, op) + .build[Int] + .apply(compareInt)) + .runDrain + fromDB <- sqlQueries.allRows(simpleTable).runCollect + } yield { + compareResults(fromDB, rows.filterNot(filterOp(_.value1, op, compareInt))) + .label(op.toString) + } + } + } + + property("deleteTwoComparisons") = forAll(Gen.listOfN(10, genSimple), + arbitrary[Int], + arbitrary[String], + Gen.oneOf(ops), + Gen.oneOf(ops)) { + (rawRows, compareInt, compareString, op1, op2) => + run { + for { + rows <- insertData(rawRows) + _ <- flushable + .flush( + sqlQueries + .deleteFrom(simpleTable) + .where('value1, op1) + .where('value2, op2) + .build[(Int, String)] + .apply((compareInt, compareString))) + .runDrain + fromDB <- sqlQueries.allRows(simpleTable).runCollect + } yield { + compareResults(fromDB, rows.filterNot { r => + filterOp(_.value1, op1, compareInt).apply(r) && + filterOp(_.value2, op2, compareString).apply(r) + }).label(s"${op1.toString} and ${op2.toString}") + } + } + } + + property("whereIn") = forAll(Gen.choose(0, 500).flatMap(sz => Gen.listOfN(sz, genSimple))) { + rawRows => + run { + for { + rows <- insertData(uniqueify(rawRows, _.value1)) + fromDB <- sqlQueries + .query(simpleTable) + .whereIn('value1) + .build[ZStream[Any, Throwable, Int]] + .apply(ZStream(rows.map(_.value1): _*)) + .runCollect + } yield { + compareResults(fromDB, rows).label("IN") + } + } + } -object JDBCExpressionProperties extends JDBCProperties[ZStream[Any, Throwable, ?], Task] with ZIOProperties { + property("whereIn and op") = forAll(Gen.choose(0, 500).flatMap(sz => Gen.listOfN(sz, genSimple)), + Gen.oneOf(ops), + arbitrary[Int]) { (rawRows, op, compareInt) => + run { + for { + rows <- insertData(uniqueify(rawRows, _.value2)) + fromDB <- sqlQueries + .query(simpleTable) + .where('value1, op) + .whereIn('value2) + .build[(Int, ZStream[Any, Throwable, String])] + .apply((compareInt, ZStream(rows.map(_.value2): _*))) + .runCollect + } yield { + compareResults(fromDB, rows.filter(filterOp(_.value1, op, compareInt))).label("IN + op") + } + } + } } diff --git a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCProperties.scala b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCProperties.scala index 0a50fda..425cd3f 100644 --- a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCProperties.scala +++ b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCProperties.scala @@ -7,9 +7,17 @@ import io.doolse.simpledba.jdbc.hsql._ import io.doolse.simpledba.syntax._ import cats.syntax.functor._ import cats.syntax.flatMap._ +import com.typesafe.config.ConfigFactory object JDBCProperties { - lazy val connection = connectionFromConfig() + val config = ConfigFactory.load() + lazy val connection = connectionFromConfig(config) + def mkLogger[F[_]: Sync]: JDBCLogger[F] = { + val testConfig = config.getConfig("simpledba.test") + if (testConfig.getBoolean("log")) { + new ConsoleLogger() + } else new NothingLogger() + } } trait JDBCProperties[S[_], F[_]] { @@ -17,13 +25,12 @@ trait JDBCProperties[S[_], F[_]] { implicit def shortCol = HSQLColumn[Short](StdJDBCColumn.shortCol, ColumnType("INTEGER")) - implicit def S : Streamable[S, F] - implicit def JE : JavaEffects[F] - implicit def Sync : Sync[F] + implicit def S: Streamable[S, F] + implicit def JE: JavaEffects[F] + implicit def Sync: Sync[F] - lazy val mapper = hsqldbMapper - def effect = JDBCEffect[S, F]( - S.M.pure(connection), _ => S.M.pure(), new NothingLogger) + lazy val mapper = hsqldbMapper + def effect = JDBCEffect[S, F](S.M.pure(connection), _ => S.M.pure(), mkLogger) lazy val sqlQueries = mapper.queries(effect) implicit def flushable = sqlQueries.flushable @@ -32,10 +39,13 @@ trait JDBCProperties[S[_], F[_]] { def setup(bq: JDBCTable[HSQLColumn]*): Unit = { implicit val SM = S.SM - run( S.drain { for { - t <- S.emits(Seq(bq: _*)).map(_.definition) - _ <- flushable.flush(rawSQLStream(S.emits(Seq(dialect.dropTable(t), dialect.createTable(t))))) - } yield () } ) + run(S.drain { + for { + t <- S.emits(Seq(bq: _*)).map(_.definition) + _ <- flushable.flush( + rawSQLStream(S.emits(Seq(dialect.dropTable(t), dialect.createTable(t))))) + } yield () + }) } def run[A](fa: F[A]): A diff --git a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCTester.scala b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCTester.scala index f6358b8..153166b 100644 --- a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCTester.scala +++ b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCTester.scala @@ -11,7 +11,7 @@ import shapeless.syntax.singleton._ import io.doolse.simpledba.fs2._ import io.doolse.simpledba.test.Test -trait JDBCTester[C[_] <: JDBCColumn[_]] extends StdColumns[C] with Test[fs2.Stream[IO, ?], IO] { +trait JDBCTester[C[A] <: JDBCColumn[A]] extends StdColumns[C] with Test[fs2.Stream[IO, ?], IO] { type F[A] = IO[A] diff --git a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCZIOTester.scala b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCZIOTester.scala index 17584a1..2450bb3 100644 --- a/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCZIOTester.scala +++ b/jdbc/src/test/scala/io/doolse/simpledba/test/jdbc/JDBCZIOTester.scala @@ -12,7 +12,7 @@ import zio.{Task, ZIO} import shapeless.HList import shapeless.syntax.singleton._ -trait JDBCZIOTester[C[_] <: JDBCColumn[_]] extends StdColumns[C] with Test[ZStream[Any, Throwable, ?], Task] { +trait JDBCZIOTester[C[A] <: JDBCColumn[A]] extends StdColumns[C] with Test[ZStream[Any, Throwable, ?], Task] { type F[A] = Task[A] type S[A] = ZStream[Any, Throwable, A] diff --git a/zio/src/main/scala/io/doolse/simpledba/ziointerop/package.scala b/zio/src/main/scala/io/doolse/simpledba/ziointerop/package.scala index 92b09ea..8e352a0 100644 --- a/zio/src/main/scala/io/doolse/simpledba/ziointerop/package.scala +++ b/zio/src/main/scala/io/doolse/simpledba/ziointerop/package.scala @@ -73,5 +73,8 @@ package object ziointerop { override def bracket[A](acquire: ZIO[Any, Throwable, A])( release: A => ZIO[Any, Throwable, Unit]): ZStream[Any, Throwable, A] = ZStream.bracket(acquire)(release.andThen(_.orDie)) + + override def maxMapped[A, B](n: Int, s: ZStream[Any, Throwable, A])(f: Seq[A] => B): ZStream[Any, Throwable, B] + = s.transduce(ZSink.identity[A].collectAllN(n).mapError(_ => throw new Throwable("How?")) ).map(f) } }