From 0ffdd7b477969cf836b20ba5eadf73e33186c70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 3 Sep 2025 13:09:23 +0200 Subject: [PATCH 1/5] add Expression type, replace Int value with Long, fix numeric values defined within elasticsearch queries, add type for sql between --- .../elastic/sql/bridge/ElasticQuery.scala | 4 +- .../elastic/sql/bridge/package.scala | 97 +++++++++++++++---- .../elastic/sql/SQLQuerySpec.scala | 95 +++++++++++++++++- .../elastic/sql/bridge/ElasticQuery.scala | 4 +- .../elastic/sql/bridge/package.scala | 97 +++++++++++++++---- .../elastic/sql/SQLQuerySpec.scala | 95 +++++++++++++++++- .../softnetwork/elastic/sql/SQLParser.scala | 53 +++++++--- .../softnetwork/elastic/sql/SQLWhere.scala | 77 +++++++++------ .../app/softnetwork/elastic/sql/package.scala | 79 +++++++++++---- 9 files changed, 491 insertions(+), 110 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index d49cf08c..ec1d8bd7 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -64,7 +64,9 @@ case class ElasticQuery(filter: ElasticFilter) { case isNull: SQLIsNull => isNull case isNotNull: SQLIsNotNull => isNotNull case in: SQLIn[_, _] => in - case between: SQLBetween => between + case between: SQLBetween[String] => between + case between: SQLBetween[Long] => between + case between: SQLBetween[Double] => between case geoDistance: ElasticGeoDistance => geoDistance case matchExpression: ElasticMatch => matchExpression case other => diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 122cf9c0..d44ff6af 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -105,56 +105,97 @@ package object bridge { ) } + def applyNumericOp[A](n: SQLNumeric[_])( + longOp: Long => A, + doubleOp: Double => A + ): A = n.toEither.fold(longOp, doubleOp) + implicit def expressionToQuery(expression: SQLExpression): Query = { import expression._ value match { - case n: SQLNumeric[Any] @unchecked => + case n: SQLNumeric[_] if !aggregation => operator match { case _: Ge.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) lt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lt l, + d => rangeQuery(identifier.name) lt d + ) case _ => - rangeQuery(identifier.name) gte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gte l, + d => rangeQuery(identifier.name) gte d + ) } case _: Gt.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) lte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lte l, + d => rangeQuery(identifier.name) lte d + ) case _ => - rangeQuery(identifier.name) gt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gt l, + d => rangeQuery(identifier.name) gt d + ) } case _: Le.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) gt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gt l, + d => rangeQuery(identifier.name) gt d + ) case _ => - rangeQuery(identifier.name) lte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lte l, + d => rangeQuery(identifier.name) lte d + ) } case _: Lt.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) gte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gte l, + d => rangeQuery(identifier.name) gte d + ) case _ => - rangeQuery(identifier.name) lt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lt l, + d => rangeQuery(identifier.name) lt d + ) } case _: Eq.type => maybeNot match { case Some(_) => - not(termQuery(identifier.name, n.sql)) + applyNumericOp(n)( + l => not(termQuery(identifier.name, l)), + d => not(termQuery(identifier.name, d)) + ) case _ => - termQuery(identifier.name, n.sql) + applyNumericOp(n)( + l => termQuery(identifier.name, l), + d => termQuery(identifier.name, d) + ) } case _: Ne.type => maybeNot match { case Some(_) => - termQuery(identifier.name, n.sql) + applyNumericOp(n)( + l => termQuery(identifier.name, l), + d => termQuery(identifier.name, d) + ) case _ => - not(termQuery(identifier.name, n.sql)) + applyNumericOp(n)( + l => not(termQuery(identifier.name, l)), + d => not(termQuery(identifier.name, d)) + ) } case _ => matchAllQuery() } - case l: SQLLiteral => + case l: SQLLiteral if !aggregation => operator match { case _: Like.type => maybeNot match { @@ -207,7 +248,7 @@ package object bridge { } case _ => matchAllQuery() } - case b: SQLBoolean => + case b: SQLBoolean if !aggregation => operator match { case _: Eq.type => maybeNot match { @@ -263,10 +304,32 @@ package object bridge { } implicit def betweenToQuery( - between: SQLBetween + between: SQLBetween[String] + ): Query = { + import between._ + val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value + maybeNot match { + case Some(_) => not(r) + case _ => r + } + } + + implicit def betweenLongsToQuery( + between: SQLBetween[Long] + ): Query = { + import between._ + val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value + maybeNot match { + case Some(_) => not(r) + case _ => r + } + } + + implicit def betweenDoublesToQuery( + between: SQLBetween[Double] ): Query = { import between._ - val r = rangeQuery(identifier.name) gte from.value lte to.value + val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value maybeNot match { case Some(_) => not(r) case _ => r diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index d8f7467b..ae831fae 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -456,7 +456,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | { | "range": { | "profiles.birthYear": { - | "lte": "2000" + | "lte": 2000 | } | } | } @@ -516,6 +516,93 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { |}""".stripMargin.replaceAll("\\s+", "") } + it should "perform query with group by and having" in { + val select: ElasticSearchRequest = + SQLQuery(groupByWithHaving) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "size": 0, + | "sort": [ + | { + | "customerid": { + | "order": "desc" + | } + | }, + | { + | "country": { + | "order": "asc" + | } + | } + | ], + | "_source": true, + | "aggs": { + | "filtered_agg": { + | "filter": { + | "bool": { + | "filter": [ + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "country": { + | "value": "usa" + | } + | } + | } + | ] + | } + | }, + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "city": { + | "value": "berlin" + | } + | } + | } + | ] + | } + | }, + | { + | "match_all": {} + | } + | ] + | } + | }, + | "aggs": { + | "country": { + | "terms": { + | "field": "country.keyword" + | }, + | "aggs": { + | "city": { + | "terms": { + | "field": "city.keyword" + | }, + | "aggs": { + | "cnt": { + | "value_count": { + | "field": "customerid" + | } + | } + | } + | } + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + it should "perform complex query" in { val select: ElasticSearchRequest = SQLQuery( @@ -581,14 +668,14 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | { | "range": { | "preparationTime": { - | "lte": "120" + | "lte": 120 | } | } | }, | { | "term": { | "deliveryPeriods.dayOfWeek": { - | "value": "6" + | "value": 6 | } | } | }, @@ -690,7 +777,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | { | "range": { | "products.stock": { - | "gt": "0" + | "gt": 0 | } | } | }, diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 91506b63..746d4af8 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -64,7 +64,9 @@ case class ElasticQuery(filter: ElasticFilter) { case isNull: SQLIsNull => isNull case isNotNull: SQLIsNotNull => isNotNull case in: SQLIn[_, _] => in - case between: SQLBetween => between + case between: SQLBetween[String] => between + case between: SQLBetween[Long] => between + case between: SQLBetween[Double] => between case geoDistance: ElasticGeoDistance => geoDistance case matchExpression: ElasticMatch => matchExpression case other => diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 1ae79170..2557ed95 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -106,56 +106,97 @@ package object bridge { ) } + def applyNumericOp[A](n: SQLNumeric[_])( + longOp: Long => A, + doubleOp: Double => A + ): A = n.toEither.fold(longOp, doubleOp) + implicit def expressionToQuery(expression: SQLExpression): Query = { import expression._ value match { - case n: SQLNumeric[Any] @unchecked => + case n: SQLNumeric[_] if !aggregation => operator match { case _: Ge.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) lt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lt l, + d => rangeQuery(identifier.name) lt d + ) case _ => - rangeQuery(identifier.name) gte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gte l, + d => rangeQuery(identifier.name) gte d + ) } case _: Gt.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) lte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lte l, + d => rangeQuery(identifier.name) lte d + ) case _ => - rangeQuery(identifier.name) gt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gt l, + d => rangeQuery(identifier.name) gt d + ) } case _: Le.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) gt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gt l, + d => rangeQuery(identifier.name) gt d + ) case _ => - rangeQuery(identifier.name) lte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lte l, + d => rangeQuery(identifier.name) lte d + ) } case _: Lt.type => maybeNot match { case Some(_) => - rangeQuery(identifier.name) gte n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) gte l, + d => rangeQuery(identifier.name) gte d + ) case _ => - rangeQuery(identifier.name) lt n.sql + applyNumericOp(n)( + l => rangeQuery(identifier.name) lt l, + d => rangeQuery(identifier.name) lt d + ) } case _: Eq.type => maybeNot match { case Some(_) => - not(termQuery(identifier.name, n.sql)) + applyNumericOp(n)( + l => not(termQuery(identifier.name, l)), + d => not(termQuery(identifier.name, d)) + ) case _ => - termQuery(identifier.name, n.sql) + applyNumericOp(n)( + l => termQuery(identifier.name, l), + d => termQuery(identifier.name, d) + ) } case _: Ne.type => maybeNot match { case Some(_) => - termQuery(identifier.name, n.sql) + applyNumericOp(n)( + l => termQuery(identifier.name, l), + d => termQuery(identifier.name, d) + ) case _ => - not(termQuery(identifier.name, n.sql)) + applyNumericOp(n)( + l => not(termQuery(identifier.name, l)), + d => not(termQuery(identifier.name, d)) + ) } case _ => matchAllQuery() } - case l: SQLLiteral => + case l: SQLLiteral if !aggregation => operator match { case _: Like.type => maybeNot match { @@ -208,7 +249,7 @@ package object bridge { } case _ => matchAllQuery() } - case b: SQLBoolean => + case b: SQLBoolean if !aggregation => operator match { case _: Eq.type => maybeNot match { @@ -264,10 +305,32 @@ package object bridge { } implicit def betweenToQuery( - between: SQLBetween + between: SQLBetween[String] ): Query = { import between._ - val r = rangeQuery(identifier.name) gte from.value lte to.value + val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value + maybeNot match { + case Some(_) => not(r) + case _ => r + } + } + + implicit def betweenLongsToQuery( + between: SQLBetween[Long] + ): Query = { + import between._ + val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value + maybeNot match { + case Some(_) => not(r) + case _ => r + } + } + + implicit def betweenDoublesToQuery( + between: SQLBetween[Double] + ): Query = { + import between._ + val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value maybeNot match { case Some(_) => not(r) case _ => r diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index d8f7467b..d4582ec6 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -456,7 +456,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | { | "range": { | "profiles.birthYear": { - | "lte": "2000" + | "lte": 2000 | } | } | } @@ -516,6 +516,93 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { |}""".stripMargin.replaceAll("\\s+", "") } + it should "perform query with group by and having" in { + val select: ElasticSearchRequest = + SQLQuery(groupByWithHaving) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "size": 0, + | "sort": [ + | { + | "customerid": { + | "order": "desc" + | } + | }, + | { + | "country": { + | "order": "asc" + | } + | } + | ], + | "_source": true, + | "aggs": { + | "filtered_agg": { + | "filter": { + | "bool": { + | "filter": [ + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "country": { + | "value": "usa" + | } + | } + | } + | ] + | } + | }, + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "city": { + | "value": "berlin" + | } + | } + | } + | ] + | } + | }, + | { + | "match_all": {} + | } + | ] + | } + | }, + | "aggs": { + | "country": { + | "terms": { + | "field": "country.keyword" + | }, + | "aggs": { + | "city": { + | "terms": { + | "field": "city.keyword" + | }, + | "aggs": { + | "cnt": { + | "value_count": { + | "field": "customerid" + | } + | } + | } + | } + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + it should "perform complex query" in { val select: ElasticSearchRequest = SQLQuery( @@ -581,14 +668,14 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | { | "range": { | "preparationTime": { - | "lte": "120" + | "lte": 120 | } | } | }, | { | "term": { | "deliveryPeriods.dayOfWeek": { - | "value": "6" + | "value": 6 | } | } | }, @@ -690,7 +777,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | { | "range": { | "products.stock": { - | "gt": "0" + | "gt": 0 | } | } | }, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala index 41bcc3b6..00922443 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala @@ -62,7 +62,7 @@ trait SQLParser extends RegexParsers with PackratParsers { def literal: PackratParser[SQLLiteral] = """"[^"]*"|'[^']*'""".r ^^ (str => SQLLiteral(str.substring(1, str.length - 1))) - def int: PackratParser[SQLInt] = """(-)?(0|[1-9]\d*)""".r ^^ (str => SQLInt(str.toInt)) + def long: PackratParser[SQLLong] = """(-)?(0|[1-9]\d*)""".r ^^ (str => SQLLong(str.toLong)) def double: PackratParser[SQLDouble] = """(-)?(\d+\.\d+)""".r ^^ (str => SQLDouble(str.toDouble)) @@ -167,7 +167,7 @@ trait SQLWhereParser { private def ne: PackratParser[SQLExpressionOperator] = Ne.sql ^^ (_ => Ne) private def equality: PackratParser[SQLExpression] = - not.? ~ (identifierWithFunction | identifier) ~ (eq | ne) ~ (boolean | literal | double | int) ^^ { + not.? ~ (identifierWithFunction | identifier) ~ (eq | ne) ~ (boolean | literal | double | long) ^^ { case n ~ i ~ o ~ v => SQLExpression(i, o, v, n) } @@ -185,40 +185,59 @@ trait SQLWhereParser { def lt: PackratParser[SQLExpressionOperator] = Lt.sql ^^ (_ => Lt) private def comparison: PackratParser[SQLExpression] = - not.? ~ (identifierWithFunction | identifier) ~ (ge | gt | le | lt) ~ (double | int | literal) ^^ { + not.? ~ (identifierWithFunction | identifier) ~ (ge | gt | le | lt) ~ (double | long | literal) ^^ { case n ~ i ~ o ~ v => SQLExpression(i, o, v, n) } def in: PackratParser[SQLExpressionOperator] = In.regex ^^ (_ => In) private def inLiteral: PackratParser[SQLCriteria] = - identifier ~ not.? ~ in ~ start ~ rep1(literal ~ separator.?) ~ end ^^ { + identifier ~ not.? ~ in ~ start ~ rep1sep(literal, separator) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => SQLIn( i, - SQLLiteralValues(v map { - _._1 - }), + SQLLiteralValues(v), n ) } - private def inNumerical: PackratParser[SQLCriteria] = - (identifierWithFunction | identifier) ~ not.? ~ in ~ start ~ rep1( - (double | int) ~ separator.? + private def inDoubles: PackratParser[SQLCriteria] = + (identifierWithFunction | identifier) ~ not.? ~ in ~ start ~ rep1sep( + double, + separator + ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => + SQLIn( + i, + SQLDoubleValues(v), + n + ) + } + + private def inLongs: PackratParser[SQLCriteria] = + (identifierWithFunction | identifier) ~ not.? ~ in ~ start ~ rep1sep( + long, + separator ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => SQLIn( i, - SQLNumericValues(v map { - _._1 - }), + SQLLongValues(v), n ) } def between: PackratParser[SQLCriteria] = (identifierWithFunction | identifier) ~ not.? ~ Between.regex ~ literal ~ and ~ literal ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, from, to, n) + case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, SQLLiteralFromTo(from, to), n) + } + + def betweenLongs: PackratParser[SQLCriteria] = + (identifierWithFunction | identifier) ~ not.? ~ Between.regex ~ long ~ and ~ long ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, SQLLongFromTo(from, to), n) + } + + def betweenDoubles: PackratParser[SQLCriteria] = + (identifierWithFunction | identifier) ~ not.? ~ Between.regex ~ double ~ and ~ double ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, SQLDoubleFromTo(from, to), n) } def distance: PackratParser[SQLCriteria] = @@ -241,7 +260,7 @@ trait SQLWhereParser { def not: PackratParser[Not.type] = Not.regex ^^ (_ => Not) def criteria: PackratParser[SQLCriteria] = - (equality | like | comparison | inLiteral | inNumerical | between | isNotNull | isNull | distance | matchCriteria) ^^ ( + (equality | like | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | distance | matchCriteria) ^^ ( c => c ) @@ -498,6 +517,8 @@ trait SQLOrderByParser { trait SQLLimitParser { self: SQLParser => - def limit: PackratParser[SQLLimit] = Limit.regex ~ int ^^ { case _ ~ i => SQLLimit(i.value) } + def limit: PackratParser[SQLLimit] = Limit.regex ~ long ^^ { case _ ~ i => + SQLLimit(i.value.toInt) + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala index 63ad2e77..076ca868 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala @@ -151,15 +151,22 @@ case class ElasticBoolQuery( } +trait Expression extends SQLCriteriaWithIdentifier with ElasticFilter { + def maybeValue: Option[SQLToken] + def maybeNot: Option[Not.type] + def notAsString: String = maybeNot.map(v => s"$v ").getOrElse("") + def valueAsString: String = maybeValue.map(v => s" $v").getOrElse("") + override def sql = s"$identifier $notAsString$operator$valueAsString" +} + case class SQLExpression( identifier: SQLIdentifier, operator: SQLExpressionOperator, value: SQLToken, maybeNot: Option[Not.type] = None -) extends SQLCriteriaWithIdentifier - with ElasticFilter { - override def sql = - s"$identifier ${maybeNot.map(_ => "not ").getOrElse("")}$operator $value" +) extends Expression { + override def maybeValue: Option[SQLToken] = Option(value) + override def update(request: SQLSearchRequest): SQLCriteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -171,11 +178,13 @@ case class SQLExpression( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class SQLIsNull(identifier: SQLIdentifier) - extends SQLCriteriaWithIdentifier - with ElasticFilter { +case class SQLIsNull(identifier: SQLIdentifier) extends Expression { override val operator: SQLOperator = IsNull - override def sql = s"$identifier $operator" + + override def maybeValue: Option[SQLToken] = None + + override def maybeNot: Option[Not.type] = None + override def update(request: SQLSearchRequest): SQLCriteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -187,11 +196,13 @@ case class SQLIsNull(identifier: SQLIdentifier) override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class SQLIsNotNull(identifier: SQLIdentifier) - extends SQLCriteriaWithIdentifier - with ElasticFilter { +case class SQLIsNotNull(identifier: SQLIdentifier) extends Expression { override val operator: SQLOperator = IsNotNull - override def sql = s"$identifier $operator" + + override def maybeValue: Option[SQLToken] = None + + override def maybeNot: Option[Not.type] = None + override def update(request: SQLSearchRequest): SQLCriteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -207,15 +218,13 @@ case class SQLIn[R, +T <: SQLValue[R]]( identifier: SQLIdentifier, values: SQLValues[R, T], maybeNot: Option[Not.type] = None -) extends SQLCriteriaWithIdentifier - with SQLTokenWithFunction - with ElasticFilter { this: SQLIn[R, T] => +) extends Expression { this: SQLIn[R, T] => private[this] lazy val id = function match { case Some(f) => s"$f($identifier)" case _ => s"$identifier" } override def sql = - s"$id ${maybeNot.map(_ => "not ").getOrElse("")}$operator $values" + s"$id $notAsString$operator $values" override def operator: SQLOperator = In override def update(request: SQLSearchRequest): SQLCriteria = { val updated = this.copy(identifier = identifier.update(request)) @@ -225,23 +234,22 @@ case class SQLIn[R, +T <: SQLValue[R]]( updated } + override def maybeValue: Option[SQLToken] = Some(values) + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class SQLBetween( +case class SQLBetween[+T]( identifier: SQLIdentifier, - from: SQLLiteral, - to: SQLLiteral, + fromTo: SQLFromTo[T], maybeNot: Option[Not.type] -) extends SQLCriteriaWithIdentifier - with SQLTokenWithFunction - with ElasticFilter { +) extends Expression { private[this] lazy val id = function match { case Some(f) => s"$f($identifier)" case _ => s"$identifier" } override def sql = - s"$id ${maybeNot.map(_ => "not ").getOrElse("")}$operator $from and $to" + s"$id $notAsString$operator $fromTo" override def operator: SQLOperator = Between override def update(request: SQLSearchRequest): SQLCriteria = { val updated = this.copy(identifier = identifier.update(request)) @@ -251,6 +259,8 @@ case class SQLBetween( updated } + override def maybeValue: Option[SQLToken] = Some(fromTo) + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } @@ -259,13 +269,17 @@ case class ElasticGeoDistance( distance: SQLLiteral, lat: SQLDouble, lon: SQLDouble -) extends SQLCriteriaWithIdentifier - with ElasticFilter { - override def sql = s"$operator($identifier,($lat,$lon)) <= $distance" - override def operator: SQLOperator = Distance +) extends Expression { + override def sql = s"$Distance($identifier,($lat,$lon)) $operator $distance" + override val function: Option[SQLFunction] = Some(Distance) + override def operator: SQLOperator = Le override def update(request: SQLSearchRequest): ElasticGeoDistance = this.copy(identifier = identifier.update(request)) + override def maybeValue: Option[SQLToken] = Some(distance) + + override def maybeNot: Option[Not.type] = None + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } @@ -307,14 +321,17 @@ case class ElasticMatch( identifier: SQLIdentifier, value: SQLLiteral, options: Option[String] -) extends SQLCriteriaWithIdentifier - with ElasticFilter { +) extends Expression { override def sql: String = s"$operator($identifier,$value${options.map(o => s""","$o"""").getOrElse("")})" override def operator: SQLOperator = Match override def update(request: SQLSearchRequest): SQLCriteria = this.copy(identifier = identifier.update(request)) + override def maybeValue: Option[SQLToken] = Some(value) + + override def maybeNot: Option[Not.type] = None + override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this override def matchCriteria: Boolean = true @@ -327,7 +344,7 @@ sealed abstract class ElasticRelation(val criteria: SQLCriteria, val operator: E private[this] def rtype(criteria: SQLCriteria): Option[String] = criteria match { case SQLPredicate(left, _, right, _, _) => rtype(left).orElse(rtype(right)) - case c: SQLCriteriaWithIdentifier => + case c: Expression => c.identifier.nestedType.orElse(c.identifier.name.split('.').headOption) case relation: ElasticRelation => relation.relationType case _ => None diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index d532a721..3c10f274 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -61,8 +61,8 @@ package object sql { } } - case class SQLBoolean(value: Boolean) extends SQLToken { - override def sql: String = s"$value" + case class SQLBoolean(override val value: Boolean) extends SQLValue[Boolean](value) { + override def sql: String = value.toString } case class SQLLiteral(override val value: String) extends SQLValue[String](value) { @@ -93,9 +93,10 @@ package object sql { } } - abstract class SQLNumeric[+T](override val value: T)(implicit ev$1: T => Ordered[T]) - extends SQLValue[T](value) { - override def sql: String = s"$value" + sealed abstract class SQLNumeric[T: Numeric](override val value: T)(implicit + ev$1: T => Ordered[T] + ) extends SQLValue[T](value) { + override def sql: String = value.toString override def choose[R >: T]( values: Seq[R], operator: Option[SQLExpressionOperator], @@ -106,27 +107,60 @@ package object sql { case _ => super.choose(values, operator, separator) } } - } - - case class SQLInt(override val value: Int) extends SQLNumeric[Int](value) { - def max: Seq[Int] => Int = x => Try(x.max).getOrElse(0) - def min: Seq[Int] => Int = x => Try(x.min).getOrElse(0) - def eq: Seq[Int] => Boolean = { + private[this] val num: Numeric[T] = implicitly[Numeric[T]] + def toDouble: Double = num.toDouble(value) + def toEither: Either[Long, Double] = value match { + case l: Long => Left(l) + case i: Int => Left(i.toLong) + case d: Double => Right(d) + case f: Float => Right(f.toDouble) + case _ => Right(toDouble) + } + def max: Seq[T] => T = x => Try(x.max).getOrElse(num.zero) + def min: Seq[T] => T = x => Try(x.min).getOrElse(num.zero) + def eq: Seq[T] => Boolean = { _.exists { _ == value } } - def ne: Seq[Int] => Boolean = { + def ne: Seq[T] => Boolean = { _.forall { _ != value } } } - case class SQLDouble(override val value: Double) extends SQLNumeric[Double](value) { - def max: Seq[Double] => Double = x => Try(x.max).getOrElse(0) - def min: Seq[Double] => Double = x => Try(x.min).getOrElse(0) - def eq: Seq[Double] => Boolean = { - _.exists { _ == value } + case class SQLLong(override val value: Long) extends SQLNumeric[Long](value) + + case class SQLDouble(override val value: Double) extends SQLNumeric[Double](value) + + sealed abstract class SQLFromTo[+T](val from: SQLValue[T], val to: SQLValue[T]) extends SQLToken { + override def sql = s"${from.sql} and ${to.sql}" + } + + case class SQLLiteralFromTo(override val from: SQLLiteral, override val to: SQLLiteral) + extends SQLFromTo[String](from, to) { + def between: Seq[String] => Boolean = { + _.exists { s => s >= from.value && s <= to.value } } - def ne: Seq[Double] => Boolean = { - _.forall { _ != value } + def notBetween: Seq[String] => Boolean = { + _.forall { s => s < from.value || s > to.value } + } + } + + case class SQLLongFromTo(override val from: SQLLong, override val to: SQLLong) + extends SQLFromTo[Long](from, to) { + def between: Seq[Long] => Boolean = { + _.exists { n => n >= from.value && n <= to.value } + } + def notBetween: Seq[Long] => Boolean = { + _.forall { n => n < from.value || n > to.value } + } + } + + case class SQLDoubleFromTo(override val from: SQLDouble, override val to: SQLDouble) + extends SQLFromTo[Double](from, to) { + def between: Seq[Double] => Boolean = { + _.exists { n => n >= from.value && n <= to.value } + } + def notBetween: Seq[Double] => Boolean = { + _.forall { n => n < from.value || n > to.value } } } @@ -146,7 +180,7 @@ package object sql { } } - case class SQLNumericValues[R: TypeTag](override val values: Seq[SQLNumeric[R]]) + class SQLNumericValues[R: TypeTag](override val values: Seq[SQLNumeric[R]]) extends SQLValues[R, SQLNumeric[R]](values) { def eq: Seq[R] => Boolean = { _.exists { n => innerValues.contains(n) } @@ -156,6 +190,11 @@ package object sql { } } + case class SQLLongValues(override val values: Seq[SQLLong]) extends SQLNumericValues[Long](values) + + case class SQLDoubleValues(override val values: Seq[SQLDouble]) + extends SQLNumericValues[Double](values) + def choose[T]( values: Seq[T], criteria: Option[SQLCriteria], From 1ef0c2355cab28ea1611112bb3f18a831bcee935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 3 Sep 2025 20:49:15 +0200 Subject: [PATCH 2/5] finalizing implementation of having and order by with aggregates and buckets --- .../sql/bridge/ElasticAggregation.scala | 93 ++++++++---- .../elastic/sql/bridge/package.scala | 43 ++++-- .../elastic/sql/SQLQuerySpec.scala | 122 +++++++++------ .../sql/bridge/ElasticAggregation.scala | 111 +++++++++----- .../elastic/sql/bridge/package.scala | 43 ++++-- .../elastic/sql/SQLQuerySpec.scala | 121 +++++++++------ .../softnetwork/elastic/sql/SQLGroupBy.scala | 143 ++++++++++++++++++ .../softnetwork/elastic/sql/SQLOrderBy.scala | 4 +- .../elastic/sql/SQLSearchRequest.scala | 3 + .../app/softnetwork/elastic/sql/package.scala | 18 ++- 10 files changed, 501 insertions(+), 200 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index c7e7f5c3..bb6fbcb2 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -2,7 +2,9 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.{ AggregateFunction, + Asc, Avg, + BucketSelectorScript, Count, ElasticBoolQuery, Max, @@ -10,10 +12,12 @@ import app.softnetwork.elastic.sql.{ SQLBucket, SQLCriteria, SQLField, + SortOrder, Sum } import com.sksamuel.elastic4s.ElasticApi.{ avgAgg, + bucketSelectorAggregation, cardinalityAgg, filterAgg, maxAgg, @@ -23,11 +27,13 @@ import com.sksamuel.elastic4s.ElasticApi.{ termsAgg, valueCountAgg } +import com.sksamuel.elastic4s.script.Script import com.sksamuel.elastic4s.searches.aggs.{ Aggregation, FilterAggregation, NestedAggregation, - TermsAggregation + TermsAggregation, + TermsOrder } import scala.language.implicitConversions @@ -42,17 +48,24 @@ case class ElasticAggregation( nestedAgg: Option[NestedAggregation] = None, filteredAgg: Option[FilterAggregation] = None, aggType: AggregateFunction, - agg: Aggregation + agg: Aggregation, + direction: Option[SortOrder] = None ) { val nested: Boolean = nestedAgg.nonEmpty val filtered: Boolean = filteredAgg.nonEmpty } object ElasticAggregation { - def apply(sqlAgg: SQLField, filter: Option[SQLCriteria]): ElasticAggregation = { + def apply( + sqlAgg: SQLField, + having: Option[SQLCriteria], + bucketsDirection: Map[String, SortOrder] + ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.name + val direction = bucketsDirection.get(identifier.identifierName) + val field = fieldAlias match { case Some(alias) => alias.alias case _ => sourceField @@ -92,7 +105,7 @@ object ElasticAggregation { val filteredAggName = "filtered_agg" val filteredAgg: Option[FilterAggregation] = - filter match { + having match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) Some( @@ -135,44 +148,60 @@ object ElasticAggregation { nestedAgg = nestedAgg, filteredAgg = filteredAgg, aggType = aggType, - agg = _agg + agg = _agg, + direction = direction ) } - /* - def apply( + def buildBuckets( buckets: Seq[SQLBucket], + bucketsDirection: Map[String, SortOrder], aggregations: Seq[Aggregation], - current: Option[TermsAggregation] + aggregationsDirection: Map[String, SortOrder], + having: Option[SQLCriteria] ): Option[TermsAggregation] = { - buckets match { - case Nil => - current.map(_.copy(subaggs = aggregations)) - case bucket +: tail => - val agg = termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") - current match { - case Some(a) => - apply(tail, aggregations, Some(agg)) match { - case Some(subAgg) => - Some(a.copy(subaggs = a.subaggs :+ subAgg)) - case _ => Some(a) - } + Console.println(bucketsDirection) + buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => + val agg = { + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") + .order(Seq(direction match { + case Asc => TermsOrder(bucket.name, asc = true) + case _ => TermsOrder(bucket.name, asc = false) + })) case None => - apply(tail, aggregations, Some(agg)) + termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") } - } - } - */ - - def buildBuckets( - buckets: Seq[SQLBucket], - aggregations: Seq[Aggregation] - ): Option[TermsAggregation] = { - buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => - val agg = termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") + } current match { case Some(subAgg) => Some(agg.copy(subaggs = Seq(subAgg))) - case None => Some(agg.copy(subaggs = aggregations)) + case None => + val aggregationsWithOrder: Seq[TermsOrder] = aggregationsDirection.toSeq.map { kv => + kv._2 match { + case Asc => TermsOrder(kv._1, asc = true) + case _ => TermsOrder(kv._1, asc = false) + } + } + val withAggregationOrders = + if (aggregationsWithOrder.nonEmpty) + agg.order(aggregationsWithOrder) + else + agg + val withHaving = having match { + case Some(criteria) => + import BucketSelectorScript._ + val script = toPainless(criteria) + val bucketsPath = extractBucketsPath(criteria) + + val bucketSelector = + bucketSelectorAggregation("having_filter", Script(script), bucketsPath) + + withAggregationOrders.copy(subaggs = aggregations :+ bucketSelector) + + case None => withAggregationOrders.copy(subaggs = aggregations) + } + Some(withHaving) } } } diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index d44ff6af..b1167f7c 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -21,14 +21,17 @@ package object bridge { request.limit.map(_.limit), request, request.buckets, - request.aggregates.map(ElasticAggregation(_, request.having.flatMap(_.criteria))) + request.aggregates.map( + ElasticAggregation(_, request.having.flatMap(_.criteria), request.sorts) + ) ).minScore(request.score) implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { import request._ val notNestedBuckets = buckets.filterNot(_.identifier.nested) val nestedBuckets = buckets.filter(_.identifier.nested).groupBy(_.nestedBucket.getOrElse("")) - val aggregations = aggregates.map(ElasticAggregation(_, request.having.flatMap(_.criteria))) + val aggregations = + aggregates.map(ElasticAggregation(_, request.having.flatMap(_.criteria), request.sorts)) val notNestedAggregations = aggregations.filterNot(_.nested) val nestedAggregations = aggregations.filter(_.nested).groupBy(_.nestedAgg.map(_.name).getOrElse("")) @@ -41,13 +44,22 @@ package object bridge { nestedAggregations.map { case (nested, aggs) => val first = aggs.head val aggregations = aggs.map(_.agg) - val buckets = ElasticAggregation.buildBuckets( - nestedBuckets.getOrElse(nested, Seq.empty), - aggregations - ) match { - case Some(b) => Seq(b) - case _ => aggregations - } + val aggregationDirections: Map[String, SortOrder] = + aggs + .filter(_.direction.isDefined) + .map(agg => agg.agg.name -> agg.direction.getOrElse(Asc)) + .toMap + val buckets = + ElasticAggregation.buildBuckets( + nestedBuckets.getOrElse(nested, Seq.empty), + request.sorts -- aggregationDirections.keys, + aggregations, + aggregationDirections, + request.having.flatMap(_.criteria) + ) match { + case Some(b) => Seq(b) + case _ => aggregations + } val filtered: Option[Aggregation] = first.filteredAgg.map(filtered => filtered.subAggregations(buckets)) first.nestedAgg.get.subAggregations(filtered.map(Seq(_)).getOrElse(buckets)) @@ -62,10 +74,17 @@ package object bridge { case _ => _search aggregations { val first = notNestedAggregations.head + val aggregationDirections: Map[String, SortOrder] = notNestedAggregations + .filter(_.direction.isDefined) + .map(agg => agg.agg.name -> agg.direction.get) + .toMap val aggregations = notNestedAggregations.map(_.agg) val buckets = ElasticAggregation.buildBuckets( notNestedBuckets, - aggregations + request.sorts -- aggregationDirections.keys, + aggregations, + aggregationDirections, + request.having.flatMap(_.criteria) ) match { case Some(b) => Seq(b) case _ => aggregations @@ -77,7 +96,7 @@ package object bridge { } _search = orderBy match { - case Some(o) => + case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map(sort => sort.order match { case Some(Desc) => FieldSort(sort.field).desc() @@ -372,7 +391,7 @@ package object bridge { .map { case Left(l) => l.aggregates - .map(ElasticAggregation(_, l.having.flatMap(_.criteria))) + .map(ElasticAggregation(_, l.having.flatMap(_.criteria), l.sorts)) .map(aggregation => { val queryFiltered = l.where diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index ae831fae..58f76b96 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -527,18 +527,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "sort": [ - | { - | "customerid": { - | "order": "desc" - | } - | }, - | { - | "country": { - | "order": "asc" - | } - | } - | ], | "_source": true, | "aggs": { | "filtered_agg": { @@ -580,18 +568,34 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "country": { | "terms": { - | "field": "country.keyword" + | "field": "country.keyword", + | "order": { + | "country": "asc" + | } | }, | "aggs": { | "city": { | "terms": { - | "field": "city.keyword" + | "field": "city.keyword", + | "order": { + | "cnt": "desc" + | } | }, | "aggs": { | "cnt": { | "value_count": { | "field": "customerid" | } + | }, + | "having_filter": { + | "bucket_selector": { + | "buckets_path": { + | "cnt": "cnt" + | }, + | "script": { + | "source": "1 == 1 && 1 == 1 && params.cnt > 1" + | } + | } | } | } | } @@ -600,15 +604,18 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("==", " == ") + .replaceAll("&&", " && ") + .replaceAll(">", " > ") } it should "perform complex query" in { val select: ElasticSearchRequest = SQLQuery( s"""SELECT - | inner_products.category as category, - | inner_products.name as productName, + | inner_products.category as cat, | min(inner_products.price) as min_price, | max(inner_products.price) as max_price |FROM @@ -629,15 +636,15 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | ) | ) |GROUP BY - | inner_products.category, - | inner_products.name + | inner_products.category |HAVING inner_products.deleted=false AND | inner_products.upForSale=true AND | inner_products.stock > 0 AND | match (inner_products.name) against ("lasagnes") AND - | match (inner_products.description, inner_products.ingredients) against ("lasagnes") - |ORDER BY preparationTime ASC, nbOrders DESC - |LIMIT 100""".stripMargin + | match (inner_products.description, inner_products.ingredients) against ("lasagnes") AND + | min(inner_products.price) > 5.0 AND + | max(inner_products.price) < 50.0 AND + | inner_products.category <> "coffee"""".stripMargin ).minScore(1.0) val query = select.query println(query) @@ -737,18 +744,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | }, | "size": 0, | "min_score": 1.0, - | "sort": [ - | { - | "preparationTime": { - | "order": "asc" - | } - | }, - | { - | "nbOrders": { - | "order": "desc" - | } - | } - | ], | "_source": true, | "aggs": { | "nested_products": { @@ -807,30 +802,53 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | ] | } + | }, + | { + | "match_all": {} + | }, + | { + | "match_all": {} + | }, + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "products.category": { + | "value": "coffee" + | } + | } + | } + | ] + | } | } | ] | } | }, | "aggs": { - | "category": { + | "cat": { | "terms": { | "field": "products.category.keyword" | }, | "aggs": { - | "productName": { - | "terms": { - | "field": "products.name.keyword" - | }, - | "aggs": { - | "min_price": { - | "min": { - | "field": "products.price" - | } + | "min_price": { + | "min": { + | "field": "products.price" + | } + | }, + | "max_price": { + | "max": { + | "field": "products.price" + | } + | }, + | "having_filter": { + | "bucket_selector": { + | "buckets_path": { + | "min_price": "min_price", + | "max_price": "max_price" | }, - | "max_price": { - | "max": { - | "field": "products.price" - | } + | "script": { + | "source": "1 == 1 && 1 == 1 && 1 == 1 && 1 == 1 && 1 == 1 && params.min_price > 5.0 && params.max_price < 50.0 && 1 == 1" | } | } | } @@ -841,7 +859,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("==", " == ") + .replaceAll("&&", " && ") + .replaceAll("<", " < ") + .replaceAll(">", " > ") + } } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 5d40d1db..cf365736 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -2,7 +2,9 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.{ AggregateFunction, + Asc, Avg, + BucketSelectorScript, Count, ElasticBoolQuery, Max, @@ -10,10 +12,12 @@ import app.softnetwork.elastic.sql.{ SQLBucket, SQLCriteria, SQLField, + SortOrder, Sum } import com.sksamuel.elastic4s.ElasticApi.{ avgAgg, + bucketSelectorAggregation, cardinalityAgg, filterAgg, maxAgg, @@ -23,35 +27,44 @@ import com.sksamuel.elastic4s.ElasticApi.{ termsAgg, valueCountAgg } +import com.sksamuel.elastic4s.requests.script.Script import com.sksamuel.elastic4s.requests.searches.aggs.{ Aggregation, FilterAggregation, NestedAggregation, - TermsAggregation + TermsAggregation, + TermsOrder, } import scala.language.implicitConversions case class ElasticAggregation( - aggName: String, - field: String, - sourceField: String, - sources: Seq[String] = Seq.empty, - query: Option[String] = None, - distinct: Boolean = false, - nestedAgg: Option[NestedAggregation] = None, - filteredAgg: Option[FilterAggregation] = None, - aggType: AggregateFunction, - agg: Aggregation) { + aggName: String, + field: String, + sourceField: String, + sources: Seq[String] = Seq.empty, + query: Option[String] = None, + distinct: Boolean = false, + nestedAgg: Option[NestedAggregation] = None, + filteredAgg: Option[FilterAggregation] = None, + aggType: AggregateFunction, + agg: Aggregation, + direction: Option[SortOrder] = None) { val nested: Boolean = nestedAgg.nonEmpty val filtered: Boolean = filteredAgg.nonEmpty } object ElasticAggregation { - def apply(sqlAgg: SQLField, filter: Option[SQLCriteria]): ElasticAggregation = { + def apply( + sqlAgg: SQLField, + having: Option[SQLCriteria], + bucketsDirection: Map[String, SortOrder] + ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.name + val direction = bucketsDirection.get(identifier.identifierName) + val field = fieldAlias match { case Some(alias) => alias.alias case _ => sourceField @@ -91,7 +104,7 @@ object ElasticAggregation { val filteredAggName = "filtered_agg" val filteredAgg: Option[FilterAggregation] = - filter match { + having match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) Some( @@ -134,44 +147,60 @@ object ElasticAggregation { nestedAgg = nestedAgg, filteredAgg = filteredAgg, aggType = aggType, - agg = _agg + agg = _agg, + direction = direction ) } - /* - def apply( + def buildBuckets( buckets: Seq[SQLBucket], + bucketsDirection: Map[String, SortOrder], aggregations: Seq[Aggregation], - current: Option[TermsAggregation] + aggregationsDirection: Map[String, SortOrder], + having: Option[SQLCriteria] ): Option[TermsAggregation] = { - buckets match { - case Nil => - current.map(_.copy(subaggs = aggregations)) - case bucket +: tail => - val agg = termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") - current match { - case Some(a) => - apply(tail, aggregations, Some(agg)) match { - case Some(subAgg) => - Some(a.copy(subaggs = a.subaggs :+ subAgg)) - case _ => Some(a) - } + Console.println(bucketsDirection) + buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => + val agg = { + bucketsDirection.get(bucket.identifier.identifierName) match { + case Some(direction) => + termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") + .order(Seq(direction match { + case Asc => TermsOrder(bucket.name, asc = true) + case _ => TermsOrder(bucket.name, asc = false) + })) case None => - apply(tail, aggregations, Some(agg)) + termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") } - } - } - */ - - def buildBuckets( - buckets: Seq[SQLBucket], - aggregations: Seq[Aggregation] - ): Option[TermsAggregation] = { - buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => - val agg = termsAgg(bucket.name, s"${bucket.identifier.name}.keyword") + } current match { case Some(subAgg) => Some(agg.copy(subaggs = Seq(subAgg))) - case None => Some(agg.copy(subaggs = aggregations)) + case None => + val aggregationsWithOrder: Seq[TermsOrder] = aggregationsDirection.toSeq.map { kv => + kv._2 match { + case Asc => TermsOrder(kv._1, asc = true) + case _ => TermsOrder(kv._1, asc = false) + } + } + val withAggregationOrders = + if (aggregationsWithOrder.nonEmpty) + agg.order(aggregationsWithOrder) + else + agg + val withHaving = having match { + case Some(criteria) => + import BucketSelectorScript._ + val script = toPainless(criteria) + val bucketsPath = extractBucketsPath(criteria) + + val bucketSelector = + bucketSelectorAggregation("having_filter", Script(script), bucketsPath) + + withAggregationOrders.copy(subaggs = aggregations :+ bucketSelector) + + case None => withAggregationOrders.copy(subaggs = aggregations) + } + Some(withHaving) } } } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 2557ed95..e051841d 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -23,14 +23,17 @@ package object bridge { request.limit.map(_.limit), request, request.buckets, - request.aggregates.map(ElasticAggregation(_, request.having.flatMap(_.criteria))) + request.aggregates.map( + ElasticAggregation(_, request.having.flatMap(_.criteria), request.sorts) + ) ).minScore(request.score) implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { import request._ val notNestedBuckets = buckets.filterNot(_.identifier.nested) val nestedBuckets = buckets.filter(_.identifier.nested).groupBy(_.nestedBucket.getOrElse("")) - val aggregations = aggregates.map(ElasticAggregation(_, request.having.flatMap(_.criteria))) + val aggregations = + aggregates.map(ElasticAggregation(_, request.having.flatMap(_.criteria), request.sorts)) val notNestedAggregations = aggregations.filterNot(_.nested) val nestedAggregations = aggregations.filter(_.nested).groupBy(_.nestedAgg.map(_.name).getOrElse("")) @@ -43,13 +46,22 @@ package object bridge { nestedAggregations.map { case (nested, aggs) => val first = aggs.head val aggregations = aggs.map(_.agg) - val buckets = ElasticAggregation.buildBuckets( - nestedBuckets.getOrElse(nested, Seq.empty), - aggregations - ) match { - case Some(b) => Seq(b) - case _ => aggregations - } + val aggregationDirections: Map[String, SortOrder] = + aggs + .filter(_.direction.isDefined) + .map(agg => agg.agg.name -> agg.direction.getOrElse(Asc)) + .toMap + val buckets = + ElasticAggregation.buildBuckets( + nestedBuckets.getOrElse(nested, Seq.empty), + request.sorts -- aggregationDirections.keys, + aggregations, + aggregationDirections, + request.having.flatMap(_.criteria) + ) match { + case Some(b) => Seq(b) + case _ => aggregations + } val filtered: Option[Aggregation] = first.filteredAgg.map(filtered => filtered.subAggregations(buckets)) first.nestedAgg.get.subAggregations(filtered.map(Seq(_)).getOrElse(buckets)) @@ -63,10 +75,17 @@ package object bridge { case Nil => _search case _ => _search aggregations { val first = notNestedAggregations.head + val aggregationDirections: Map[String, SortOrder] = notNestedAggregations + .filter(_.direction.isDefined) + .map(agg => agg.agg.name -> agg.direction.get) + .toMap val aggregations = notNestedAggregations.map(_.agg) val buckets = ElasticAggregation.buildBuckets( notNestedBuckets, - aggregations + request.sorts -- aggregationDirections.keys, + aggregations, + aggregationDirections, + request.having.flatMap(_.criteria) ) match { case Some(b) => Seq(b) case _ => aggregations @@ -78,7 +97,7 @@ package object bridge { } _search = orderBy match { - case Some(o) => + case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map(sort => sort.order match { case Some(Desc) => FieldSort(sort.field).desc() @@ -373,7 +392,7 @@ package object bridge { .map { case Left(l) => l.aggregates - .map(ElasticAggregation(_, l.having.flatMap(_.criteria))) + .map(ElasticAggregation(_, l.having.flatMap(_.criteria), l.sorts)) .map(aggregation => { val queryFiltered = l.where diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index d4582ec6..ce96ddf5 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -527,18 +527,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "size": 0, - | "sort": [ - | { - | "customerid": { - | "order": "desc" - | } - | }, - | { - | "country": { - | "order": "asc" - | } - | } - | ], | "_source": true, | "aggs": { | "filtered_agg": { @@ -580,18 +568,34 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "aggs": { | "country": { | "terms": { - | "field": "country.keyword" + | "field": "country.keyword", + | "order": { + | "country": "asc" + | } | }, | "aggs": { | "city": { | "terms": { - | "field": "city.keyword" + | "field": "city.keyword", + | "order": { + | "cnt": "desc" + | } | }, | "aggs": { | "cnt": { | "value_count": { | "field": "customerid" | } + | }, + | "having_filter": { + | "bucket_selector": { + | "buckets_path": { + | "cnt": "cnt" + | }, + | "script": { + | "source": "1 == 1 && 1 == 1 && params.cnt > 1" + | } + | } | } | } | } @@ -600,15 +604,18 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } | } - |}""".stripMargin.replaceAll("\\s+", "") + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("==", " == ") + .replaceAll("&&", " && ") + .replaceAll(">", " > ") } it should "perform complex query" in { val select: ElasticSearchRequest = SQLQuery( s"""SELECT - | inner_products.category as category, - | inner_products.name as productName, + | inner_products.category as cat, | min(inner_products.price) as min_price, | max(inner_products.price) as max_price |FROM @@ -629,20 +636,20 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | ) | ) |GROUP BY - | inner_products.category, - | inner_products.name + | inner_products.category |HAVING inner_products.deleted=false AND | inner_products.upForSale=true AND | inner_products.stock > 0 AND | match (inner_products.name) against ("lasagnes") AND - | match (inner_products.description, inner_products.ingredients) against ("lasagnes") - |ORDER BY preparationTime ASC, nbOrders DESC - |LIMIT 100""".stripMargin + | match (inner_products.description, inner_products.ingredients) against ("lasagnes") AND + | min(inner_products.price) > 5.0 AND + | max(inner_products.price) < 50.0 AND + | inner_products.category <> "coffee"""".stripMargin ).minScore(1.0) val query = select.query println(query) query shouldBe - """ + """ |{ | "query": { | "bool": { @@ -737,18 +744,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | }, | "size": 0, | "min_score": 1.0, - | "sort": [ - | { - | "preparationTime": { - | "order": "asc" - | } - | }, - | { - | "nbOrders": { - | "order": "desc" - | } - | } - | ], | "_source": true, | "aggs": { | "nested_products": { @@ -807,30 +802,53 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | ] | } + | }, + | { + | "match_all": {} + | }, + | { + | "match_all": {} + | }, + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "products.category": { + | "value": "coffee" + | } + | } + | } + | ] + | } | } | ] | } | }, | "aggs": { - | "category": { + | "cat": { | "terms": { | "field": "products.category.keyword" | }, | "aggs": { - | "productName": { - | "terms": { - | "field": "products.name.keyword" - | }, - | "aggs": { - | "min_price": { - | "min": { - | "field": "products.price" - | } + | "min_price": { + | "min": { + | "field": "products.price" + | } + | }, + | "max_price": { + | "max": { + | "field": "products.price" + | } + | }, + | "having_filter": { + | "bucket_selector": { + | "buckets_path": { + | "min_price": "min_price", + | "max_price": "max_price" | }, - | "max_price": { - | "max": { - | "field": "products.price" - | } + | "script": { + | "source": "1 == 1 && 1 == 1 && 1 == 1 && 1 == 1 && 1 == 1 && params.min_price > 5.0 && params.max_price < 50.0 && 1 == 1" | } | } | } @@ -842,6 +860,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | } |}""".stripMargin.replaceAll("\\s+", "") + .replaceAll("==", " == ") + .replaceAll("&&", " && ") + .replaceAll("<", " < ") + .replaceAll(">", " > ") + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala index 6ab8726e..4903c3ac 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala @@ -6,6 +6,9 @@ case class SQLGroupBy(buckets: Seq[SQLBucket]) extends Updateable { override def sql: String = s" $GroupBy ${buckets.mkString(",")}" def update(request: SQLSearchRequest): SQLGroupBy = this.copy(buckets = buckets.map(_.update(request))) + lazy val bucketNames: Map[String, SQLBucket] = buckets.map { b => + b.identifier.identifierName -> b + }.toMap } case class SQLBucket( @@ -27,3 +30,143 @@ case class SQLBucket( lazy val name: String = identifier.fieldAlias.getOrElse(sourceBucket.replace(".", "_")) } + +object BucketSelectorScript { + + private[this] def painlessIn(param: String, values: Seq[SQLValue[_]], not: Boolean): String = { + val ret = s"[${values.map { _.painlessValue }.mkString(", ")}].contains($param)" + if (not) s"!$ret" else ret + } + + private[this] def painlessBetween( + param: String, + lower: SQLValue[_], + upper: SQLValue[_], + not: Boolean + ): String = { + val ret = s"($param >= ${lower.painlessValue} && $param <= ${upper.painlessValue})" + if (not) s"!$ret" else ret + } + + private[this] def toPainless( + param: String, + operator: SQLOperator, + value: SQLToken, + not: Boolean + ): String = { + operator match { + case _: SQLComparisonOperator => + val valueStr = + value match { + case v: SQLBoolean => v.painlessValue + case v: SQLDouble => v.painlessValue + case v: SQLLiteral => v.painlessValue + case v: SQLLong => v.painlessValue + case _ => + throw new IllegalArgumentException( + s"Unsupported value type in bucket_selector: $value" + ) + } + if (not) { + operator match { + case Eq => s"$param != $valueStr" + case Ne => s"$param == $valueStr" + case Gt => s"$param <= $valueStr" + case Ge => s"$param < $valueStr" + case Lt => s"$param >= $valueStr" + case Le => s"$param > $valueStr" + case _ => + throw new IllegalArgumentException( + s"Unsupported comparison operator in bucket_selector: $operator" + ) + } + } else + operator match { + case Eq => s"$param == $valueStr" + case Ne => s"$param != $valueStr" + case Gt => s"$param > $valueStr" + case Ge => s"$param >= $valueStr" + case Lt => s"$param < $valueStr" + case Le => s"$param <= $valueStr" + case _ => + throw new IllegalArgumentException( + s"Unsupported comparison operator in bucket_selector: $operator" + ) + } + case In => + value match { + case SQLDoubleValues(vals) => painlessIn(param, vals, not) + case SQLLiteralValues(vals) => painlessIn(param, vals, not) + case SQLLongValues(vals) => painlessIn(param, vals, not) + case _ => throw new IllegalArgumentException("IN requires a list") + } + case Between => + value match { + case SQLDoubleFromTo(lower, upper) => painlessBetween(param, lower, upper, not) + case SQLLiteralFromTo(lower, upper) => painlessBetween(param, lower, upper, not) + case SQLLongFromTo(lower, upper) => painlessBetween(param, lower, upper, not) + case _ => throw new IllegalArgumentException("BETWEEN requires two values") + } + case _ => + throw new IllegalArgumentException(s"Unsupported operator in bucket_selector: $operator") + } + } + + def extractBucketsPath(criteria: SQLCriteria): Map[String, String] = criteria match { + case SQLPredicate(left, _, right, _, _) => + extractBucketsPath(left) ++ extractBucketsPath(right) + case relation: ElasticRelation => extractBucketsPath(relation.criteria) + case _: SQLMatch => Map.empty //MATCH is not supported in bucket_selector + case e: Expression => + import e._ + val name = identifier.fieldAlias.getOrElse(identifier.name) + if (e.aggregation) { + Map(name -> name) + } /*else if (e.identifier.bucket.isDefined) { + Map(name -> "_key") + }*/ + else { + Map.empty // for performance, we only allow aggregation here + } + } + + def toPainless(expr: SQLCriteria): String = expr match { + case SQLPredicate(left, op, right, maybeNot, group) => + val leftStr = toPainless(left) + val rightStr = toPainless(right) + val opStr = op match { + case And => "&&" + case Or => "||" + case _ => throw new IllegalArgumentException(s"Unsupported logical operator: $op") + } + val not = maybeNot.nonEmpty + if (group || not) + s"${maybeNot.map(_ => "!").getOrElse("")}($leftStr) $opStr ($rightStr)" + else + s"$leftStr $opStr $rightStr" + + case relation: ElasticRelation => toPainless(relation.criteria) + + case _: SQLMatch => "1 == 1" //MATCH is not supported in bucket_selector + + case e: Expression => + if (e.aggregation /*|| e.identifier.bucket.isDefined*/ ) { // for performance, we only allow aggregation here + val param = + s"params.${e.identifier.fieldAlias.getOrElse(e.identifier.name)}" + e.maybeValue match { + case Some(v) => toPainless(param, e.operator, v, e.maybeNot.nonEmpty) + case None => + e.operator match { + case IsNull => s"$param == null" + case IsNotNull => s"$param != null" + case _ => + throw new IllegalArgumentException(s"Operator ${e.operator} requires a value") + } + } + } else { + "1 == 1" + } + + case _ => throw new IllegalArgumentException(s"Unsupported SQLCriteria type: $expr") + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala index 4924c57b..74f3110a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala @@ -17,7 +17,9 @@ case class SQLFieldSort( case Some(f) => s"$f($field)" case _ => field } - override def sql: String = s"$fieldWithFunction ${order.getOrElse(Asc)}" + lazy val direction: SortOrder = order.getOrElse(Asc) + lazy val name: String = fieldWithFunction + override def sql: String = s"$name $direction" } case class SQLOrderBy(sorts: Seq[SQLFieldSort]) extends SQLToken { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala index f4f14e09..fb415157 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala @@ -16,6 +16,9 @@ case class SQLSearchRequest( lazy val fieldAliases: Map[String, String] = select.fieldAliases lazy val tableAliases: Map[String, String] = from.tableAliases lazy val unnests: Seq[(String, String, Option[SQLLimit])] = from.unnests + lazy val bucketNames: Map[String, SQLBucket] = groupBy.map(_.bucketNames).getOrElse(Map.empty) + lazy val sorts: Map[String, SortOrder] = + orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap def update(): SQLSearchRequest = { val updated = this.copy(from = from.update(this)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 3c10f274..e7316baa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -59,6 +59,12 @@ package object sql { case _ => values.headOption } } + def painlessValue: String = value match { + case s: String => s""""$s"""" + case b: Boolean => b.toString + case n: Number => n.toString + case _ => value.toString + } } case class SQLBoolean(override val value: Boolean) extends SQLValue[Boolean](value) { @@ -249,7 +255,8 @@ package object sql { nested: Boolean = false, limit: Option[SQLLimit] = None, function: Option[SQLFunction] = None, - fieldAlias: Option[String] = None + fieldAlias: Option[String] = None, + bucket: Option[SQLBucket] = None ) extends SQLExpr({ var parts: Seq[String] = name.split("\\.").toSeq tableAlias match { @@ -294,18 +301,21 @@ package object sql { name = s"${tuple._2}.${parts.tail.mkString(".")}", nested = true, limit = tuple._3, - fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias) + fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), + bucket = request.bucketNames.get(identifierName).orElse(bucket) ) case _ => this.copy( tableAlias = Some(parts.head), name = parts.tail.mkString("."), - fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias) + fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), + bucket = request.bucketNames.get(identifierName).orElse(bucket) ) } } else { this.copy( - fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias) + fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), + bucket = request.bucketNames.get(identifierName).orElse(bucket) ) } } From 3e929582b67ebf4f2197bafd9040c01bbe816c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 3 Sep 2025 20:51:22 +0200 Subject: [PATCH 3/5] update version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 9bebae57..6f0014dc 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.3.0" +ThisBuild / version := "0.4.0" ThisBuild / scalaVersion := scala213 From 0f9b9d0741c88e80b4baa288cd29308209b02d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 3 Sep 2025 20:59:21 +0200 Subject: [PATCH 4/5] to fix criteria specifications --- .../elastic/sql/SQLCriteriaSpec.scala | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 93220c1f..30cf8204 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -31,7 +31,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"term" : { | "identifier" : { - | "value" : "1.0" + | "value" : 1.0 | } | } | } @@ -47,7 +47,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term":{ | "identifier":{ - | "value":"1" + | "value":1 | } | } | } @@ -63,7 +63,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "lt" : "1" + | "lt" : 1 | } | } | } @@ -76,7 +76,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "lte" : "1" + | "lte" : 1 | } | } | } @@ -89,7 +89,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "gt" : "1" + | "gt" : 1 | } | } | } @@ -102,7 +102,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "gte" : "1" + | "gte" : 1 | } | } | } @@ -193,14 +193,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -219,14 +219,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -248,14 +248,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -265,7 +265,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | } @@ -284,7 +284,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, @@ -294,14 +294,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | }, | { | "term" : { | "identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | } @@ -326,14 +326,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -346,14 +346,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | }, | { | "term" : { | "identifier4" : { - | "value" : "4" + | "value" : 4 | } | } | } @@ -420,7 +420,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, @@ -433,14 +433,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range" : { | "nested.identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | }, | { | "term" : { | "nested.identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | } @@ -465,7 +465,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, @@ -475,7 +475,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | "query" : { | "term" : { | "nested.identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | }, @@ -498,7 +498,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -515,14 +515,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range": { | "child.identifier2": { - | "gt": "2" + | "gt": 2 | } | } | }, | { | "term": { | "child.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -550,7 +550,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -564,7 +564,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "child.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -589,7 +589,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -605,14 +605,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range": { | "parent.identifier2": { - | "gt": "2" + | "gt": 2 | } | } | }, | { | "term": { | "parent.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -640,7 +640,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -653,7 +653,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "parent.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -688,7 +688,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "ciblage.statutComportement" : { - | "value" : "1" + | "value" : 1 | } | } | } @@ -835,7 +835,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier": { - | "value": "1" + | "value": 1 | } | } | } @@ -862,7 +862,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range": { | "identifier2": { - | "gt": "2" + | "gt": 2 | } | } | } @@ -872,7 +872,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier3": { - | "value": "3" + | "value": 3 | } | } | } From 353e0fcabad7503f6df38afcd2b61b48f058ff98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 3 Sep 2025 21:12:33 +0200 Subject: [PATCH 5/5] to fix criteria specifications --- .../elastic/sql/SQLCriteriaSpec.scala | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index c4b3d720..d1f088f6 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -30,7 +30,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"term" : { | "identifier" : { - | "value" : "1.0" + | "value" : 1.0 | } | } | } @@ -46,7 +46,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term":{ | "identifier":{ - | "value":"1" + | "value":1 | } | } | } @@ -62,7 +62,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "lt" : "1" + | "lt" : 1 | } | } | } @@ -75,7 +75,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "lte" : "1" + | "lte" : 1 | } | } | } @@ -88,7 +88,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "gt" : "1" + | "gt" : 1 | } | } | } @@ -101,7 +101,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"range" : { | "identifier" : { - | "gte" : "1" + | "gte" : 1 | } | } | } @@ -192,14 +192,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -218,14 +218,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -247,14 +247,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -264,7 +264,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | } @@ -283,7 +283,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, @@ -293,14 +293,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | }, | { | "term" : { | "identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | } @@ -325,14 +325,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, | { | "range" : { | "identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | } @@ -345,14 +345,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | }, | { | "term" : { | "identifier4" : { - | "value" : "4" + | "value" : 4 | } | } | } @@ -419,7 +419,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, @@ -432,14 +432,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range" : { | "nested.identifier2" : { - | "gt" : "2" + | "gt" : 2 | } | } | }, | { | "term" : { | "nested.identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | } @@ -464,7 +464,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "identifier1" : { - | "value" : "1" + | "value" : 1 | } | } | }, @@ -474,7 +474,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | "query" : { | "term" : { | "nested.identifier3" : { - | "value" : "3" + | "value" : 3 | } | } | }, @@ -497,7 +497,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -514,14 +514,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range": { | "child.identifier2": { - | "gt": "2" + | "gt": 2 | } | } | }, | { | "term": { | "child.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -549,7 +549,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -563,7 +563,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "child.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -588,7 +588,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -604,14 +604,14 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range": { | "parent.identifier2": { - | "gt": "2" + | "gt": 2 | } | } | }, | { | "term": { | "parent.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -639,7 +639,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier1": { - | "value": "1" + | "value": 1 | } | } | }, @@ -652,7 +652,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "parent.identifier3": { - | "value": "3" + | "value": 3 | } | } | } @@ -687,7 +687,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term" : { | "ciblage.statutComportement" : { - | "value" : "1" + | "value" : 1 | } | } | } @@ -834,7 +834,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier": { - | "value": "1" + | "value": 1 | } | } | } @@ -861,7 +861,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range": { | "identifier2": { - | "gt": "2" + | "gt": 2 | } | } | } @@ -871,7 +871,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "term": { | "identifier3": { - | "value": "3" + | "value": 3 | } | } | }