From 7d10e7c58372dbb2c0b2ea11e45a0a075aa516fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 20 Sep 2025 13:57:19 +0200 Subject: [PATCH 01/48] major refactoring adding new packages --- .../elastic/client/AggregateResult.scala | 2 +- .../elastic/client/jest/JestClientApi.scala | 10 +- .../client/rest/RestHighLevelClientApi.scala | 10 +- .../sql/bridge/ElasticAggregation.scala | 23 +- .../elastic/sql/bridge/ElasticCriteria.scala | 4 +- .../elastic/sql/bridge/ElasticQuery.scala | 32 +- .../sql/bridge/ElasticSearchRequest.scala | 8 +- .../elastic/sql/bridge/package.scala | 35 +- ...LCriteriaSpec.scala => CriteriaSpec.scala} | 4 +- .../client/rest/RestHighLevelClientApi.scala | 10 +- .../client/java/ElasticsearchClientApi.scala | 10 +- .../client/java/ElasticsearchClientApi.scala | 10 +- .../sql/bridge/ElasticAggregation.scala | 33 +- .../elastic/sql/bridge/ElasticCriteria.scala | 4 +- .../elastic/sql/bridge/ElasticQuery.scala | 32 +- .../sql/bridge/ElasticSearchRequest.scala | 18 +- .../elastic/sql/bridge/package.scala | 35 +- .../elastic/sql/SQLCriteriaSpec.scala | 4 +- .../softnetwork/elastic/sql/Delimiter.scala | 16 + .../app/softnetwork/elastic/sql/From.scala | 38 + .../sql/{SQLGroupBy.scala => GroupBy.scala} | 37 +- .../sql/{SQLHaving.scala => Having.scala} | 6 +- .../app/softnetwork/elastic/sql/Limit.scala | 5 + .../app/softnetwork/elastic/sql/OrderBy.scala | 25 + .../elastic/sql/SQLDelimiter.scala | 14 - .../app/softnetwork/elastic/sql/SQLFrom.scala | 38 - .../softnetwork/elastic/sql/SQLFunction.scala | 1122 ----------------- .../elastic/sql/SQLImplicits.scala | 4 +- .../softnetwork/elastic/sql/SQLLimit.scala | 5 - .../elastic/sql/SQLMultiSearchRequest.scala | 2 +- .../softnetwork/elastic/sql/SQLOperator.scala | 148 --- .../softnetwork/elastic/sql/SQLOrderBy.scala | 23 - .../elastic/sql/SQLSearchRequest.scala | 24 +- .../sql/{SQLSelect.scala => Select.scala} | 40 +- .../{SQLValidator.scala => Validator.scala} | 13 +- .../sql/{SQLWhere.scala => Where.scala} | 253 ++-- .../sql/function/aggregate/package.scala | 19 + .../elastic/sql/function/cond/package.scala | 255 ++++ .../sql/function/convert/package.scala | 29 + .../elastic/sql/function/geo/package.scala | 10 + .../elastic/sql/function/math/package.scala | 97 ++ .../elastic/sql/function/package.scala | 185 +++ .../elastic/sql/function/string/package.scala | 119 ++ .../elastic/sql/function/time/package.scala | 389 ++++++ .../operator/math/ArithmeticExpression.scala | 83 ++ .../elastic/sql/operator/math/package.scala | 17 + .../elastic/sql/operator/package.scala | 67 + .../elastic/sql/operator/time/package.scala | 24 + .../app/softnetwork/elastic/sql/package.scala | 145 +-- .../elastic/sql/{ => parser}/SQLParser.scala | 484 +++---- .../elastic/sql/time/package.scala | 108 ++ .../elastic/sql/{ => type}/SQLType.scala | 2 +- .../elastic/sql/{ => type}/SQLTypeUtils.scala | 6 +- .../elastic/sql/{ => type}/SQLTypes.scala | 2 +- .../sql/SQLDateTimeFunctionSuite.scala | 36 +- .../elastic/sql/SQLParserSpec.scala | 2 + ...gValueSpec.scala => StringValueSpec.scala} | 4 +- 57 files changed, 2175 insertions(+), 2005 deletions(-) rename es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/{SQLCriteriaSpec.scala => CriteriaSpec.scala} (99%) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/From.scala rename sql/src/main/scala/app/softnetwork/elastic/sql/{SQLGroupBy.scala => GroupBy.scala} (69%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{SQLHaving.scala => Having.scala} (62%) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/SQLDelimiter.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/SQLFrom.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/SQLFunction.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/SQLLimit.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/SQLOperator.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala rename sql/src/main/scala/app/softnetwork/elastic/sql/{SQLSelect.scala => Select.scala} (66%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{SQLValidator.scala => Validator.scala} (66%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{SQLWhere.scala => Where.scala} (66%) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => parser}/SQLParser.scala (67%) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => type}/SQLType.scala (94%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => type}/SQLTypeUtils.scala (96%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => type}/SQLTypes.scala (97%) rename sql/src/test/scala/app/softnetwork/elastic/sql/{SQLStringValueSpec.scala => StringValueSpec.scala} (81%) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala index 10e67c52..f5544489 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.client -import app.softnetwork.elastic.sql.AggregateFunction +import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction sealed trait AggregateResult { def field: String diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 6d0a9826..653c1e4b 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -386,7 +386,7 @@ trait JestSingleValueAggregateApi extends SingleValueAggregateApi with JestCount field, aggType, aggType match { - case sql.Count => + case sql.function.aggregate.Count => if (aggregation.distinct) NumericValue( root.getCardinalityAggregation(agg).getCardinality.doubleValue() @@ -396,13 +396,13 @@ trait JestSingleValueAggregateApi extends SingleValueAggregateApi with JestCount root.getValueCountAggregation(agg).getValueCount.doubleValue() ) } - case sql.Sum => + case sql.function.aggregate.Sum => NumericValue(root.getSumAggregation(agg).getSum) - case sql.Avg => + case sql.function.aggregate.Avg => NumericValue(root.getAvgAggregation(agg).getAvg) - case sql.Min => + case sql.function.aggregate.Min => NumericValue(root.getMinAggregation(agg).getMin) - case sql.Max => + case sql.function.aggregate.Max => NumericValue(root.getMaxAggregation(agg).getMax) case _ => EmptyValue }, diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 5eb1d663..510040f6 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -428,19 +428,19 @@ trait RestHighLevelClientSingleValueAggregateApi field, aggType, aggType match { - case sql.Count => + case sql.function.aggregate.Count => if (aggregation.distinct) { NumericValue(root.get(agg).asInstanceOf[Cardinality].value()) } else { NumericValue(root.get(agg).asInstanceOf[ValueCount].value()) } - case sql.Sum => + case sql.function.aggregate.Sum => NumericValue(root.get(agg).asInstanceOf[Sum].value()) - case sql.Avg => + case sql.function.aggregate.Avg => NumericValue(root.get(agg).asInstanceOf[Avg].value()) - case sql.Min => + case sql.function.aggregate.Min => NumericValue(root.get(agg).asInstanceOf[Min].value()) - case sql.Max => + case sql.function.aggregate.Max => NumericValue(root.get(agg).asInstanceOf[Max].value()) case _ => EmptyValue }, diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 86aeec7c..3ecac205 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -1,21 +1,16 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.{ - AggregateFunction, Asc, - Avg, + Bucket, BucketSelectorScript, - Count, + Criteria, ElasticBoolQuery, Field, - Max, - Min, - SQLBucket, - SQLCriteria, - SQLFunctionUtils, - SortOrder, - Sum + SortOrder } +import app.softnetwork.elastic.sql.function._ +import app.softnetwork.elastic.sql.function.aggregate._ import com.sksamuel.elastic4s.ElasticApi.{ avgAgg, bucketSelectorAggregation, @@ -59,7 +54,7 @@ case class ElasticAggregation( object ElasticAggregation { def apply( sqlAgg: Field, - having: Option[SQLCriteria], + having: Option[Criteria], bucketsDirection: Map[String, SortOrder] ): ElasticAggregation = { import sqlAgg._ @@ -89,7 +84,7 @@ object ElasticAggregation { var aggPath = Seq[String]() - val (aggFuncs, transformFuncs) = SQLFunctionUtils.aggregateAndTransformFunctions(identifier) + val (aggFuncs, transformFuncs) = FunctionUtils.aggregateAndTransformFunctions(identifier) require(aggFuncs.size == 1, s"Multiple aggregate functions not supported: $aggFuncs") @@ -172,11 +167,11 @@ object ElasticAggregation { } def buildBuckets( - buckets: Seq[SQLBucket], + buckets: Seq[Bucket], bucketsDirection: Map[String, SortOrder], aggregations: Seq[Aggregation], aggregationsDirection: Map[String, SortOrder], - having: Option[SQLCriteria] + having: Option[Criteria] ): Option[TermsAggregation] = { Console.println(bucketsDirection) buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index bf6ebe38..33428bbe 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -1,9 +1,9 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.SQLCriteria +import app.softnetwork.elastic.sql.Criteria import com.sksamuel.elastic4s.searches.queries.Query -case class ElasticCriteria(criteria: SQLCriteria) { +case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { val query = criteria.boolQuery.copy(group = group) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 61fd88f1..1792cdac 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -1,6 +1,7 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.{ + BetweenExpr, ElasticBoolQuery, ElasticChild, ElasticFilter, @@ -8,13 +9,12 @@ import app.softnetwork.elastic.sql.{ ElasticMatch, ElasticNested, ElasticParent, - SQLBetween, - SQLExpression, - SQLIn, - SQLIsNotNull, - SQLIsNotNullCriteria, - SQLIsNull, - SQLIsNullCriteria + GenericExpression, + InExpr, + IsNotNullCriteria, + IsNotNullExpr, + IsNullCriteria, + IsNullExpr } import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.searches.queries.Query @@ -62,17 +62,17 @@ case class ElasticQuery(filter: ElasticFilter) { criteria.asQuery(group = group, innerHitsNames = innerHitsNames), score = false ) - case expression: SQLExpression => expression - case isNull: SQLIsNull => isNull - case isNotNull: SQLIsNotNull => isNotNull - case in: SQLIn[_, _] => in - case between: SQLBetween[String] => between - case between: SQLBetween[Long] => between - case between: SQLBetween[Double] => between + case expression: GenericExpression => expression + case isNull: IsNullExpr => isNull + case isNotNull: IsNotNullExpr => isNotNull + case in: InExpr[_, _] => in + case between: BetweenExpr[String] => between + case between: BetweenExpr[Long] => between + case between: BetweenExpr[Double] => between case geoDistance: ElasticGeoDistance => geoDistance case matchExpression: ElasticMatch => matchExpression - case isNull: SQLIsNullCriteria => isNull - case isNotNull: SQLIsNotNullCriteria => isNotNull + case isNull: IsNullCriteria => isNull + case isNotNull: IsNotNullCriteria => isNotNull case other => throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") } diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 3c451a43..60177e0d 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -1,17 +1,17 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{Field, SQLBucket, SQLCriteria, SQLExcept} +import app.softnetwork.elastic.sql.{Bucket, Criteria, Except, Field} import com.sksamuel.elastic4s.searches.SearchRequest import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn case class ElasticSearchRequest( fields: Seq[Field], - except: Option[SQLExcept], + except: Option[Except], sources: Seq[String], - criteria: Option[SQLCriteria], + criteria: Option[Criteria], limit: Option[Int], search: SearchRequest, - buckets: Seq[SQLBucket] = Seq.empty, + buckets: Seq[Bucket] = Seq.empty, aggregations: Seq[ElasticAggregation] = Seq.empty ) { def minScore(score: Option[Double]): ElasticSearchRequest = { diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 36efad37..e2683871 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,5 +1,8 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.function.aggregate.Count +import app.softnetwork.elastic.sql.operator._ + import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp @@ -136,12 +139,12 @@ package object bridge { ) } - def applyNumericOp[A](n: SQLNumericValue[_])( + def applyNumericOp[A](n: NumericValue[_])( longOp: Long => A, doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: SQLExpression): Query = { + implicit def expressionToQuery(expression: GenericExpression): Query = { import expression._ if (aggregation) return matchAllQuery() @@ -149,7 +152,7 @@ package object bridge { return scriptQuery(Script(script = painless).lang("painless").scriptType("source")) } value match { - case n: SQLNumericValue[_] => + case n: NumericValue[_] => operator match { case Ge => maybeNot match { @@ -231,7 +234,7 @@ package object bridge { } case _ => matchAllQuery() } - case l: SQLStringValue => + case l: StringValue => operator match { case Like => maybeNot match { @@ -284,7 +287,7 @@ package object bridge { } case _ => matchAllQuery() } - case b: SQLBoolean => + case b: BooleanValue => operator match { case Eq => maybeNot match { @@ -302,9 +305,9 @@ package object bridge { } case _ => matchAllQuery() } - case i: SQLIdentifier => + case i: GenericIdentifier => operator match { - case op: SQLComparisonOperator => + case op: ComparisonOperator => i.toScript match { case Some(script) => val o = if (maybeNot.isDefined) op.not else op @@ -327,34 +330,34 @@ package object bridge { } implicit def isNullToQuery( - isNull: SQLIsNull + isNull: IsNullExpr ): Query = { import isNull._ not(existsQuery(identifier.name)) } implicit def isNotNullToQuery( - isNotNull: SQLIsNotNull + isNotNull: IsNotNullExpr ): Query = { import isNotNull._ existsQuery(identifier.name) } implicit def isNullCriteriaToQuery( - isNull: SQLIsNullCriteria + isNull: IsNullCriteria ): Query = { import isNull._ not(existsQuery(identifier.name)) } implicit def isNotNullCriteriaToQuery( - isNotNull: SQLIsNotNullCriteria + isNotNull: IsNotNullCriteria ): Query = { import isNotNull._ existsQuery(identifier.name) } - implicit def inToQuery[R, T <: SQLValue[R]](in: SQLIn[R, T]): Query = { + implicit def inToQuery[R, T <: Value[R]](in: InExpr[R, T]): Query = { import in._ val _values: Seq[Any] = values.innerValues val t = @@ -374,7 +377,7 @@ package object bridge { } implicit def betweenToQuery( - between: SQLBetween[String] + between: BetweenExpr[String] ): Query = { import between._ val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value @@ -385,7 +388,7 @@ package object bridge { } implicit def betweenLongsToQuery( - between: SQLBetween[Long] + between: BetweenExpr[Long] ): Query = { import between._ val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value @@ -396,7 +399,7 @@ package object bridge { } implicit def betweenDoublesToQuery( - between: SQLBetween[Double] + between: BetweenExpr[Double] ): Query = { import between._ val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value @@ -421,7 +424,7 @@ package object bridge { } implicit def criteriaToElasticCriteria( - criteria: SQLCriteria + criteria: Criteria ): ElasticCriteria = { ElasticCriteria( criteria diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/CriteriaSpec.scala similarity index 99% rename from es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala rename to es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/CriteriaSpec.scala index 30cf8204..3e959641 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/CriteriaSpec.scala @@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers /** Created by smanciot on 13/04/17. */ -class SQLCriteriaSpec extends AnyFlatSpec with Matchers { +class CriteriaSpec extends AnyFlatSpec with Matchers { import Queries._ @@ -17,7 +17,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { def asQuery(sql: String): String = { import SQLImplicits._ - val criteria: Option[SQLCriteria] = sql + val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) ).string diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index d1cc380f..82889d09 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -423,19 +423,19 @@ trait RestHighLevelClientSingleValueAggregateApi field, aggType, aggType match { - case sql.Count => + case sql.function.aggregate.Count => if (aggregation.distinct) { NumericValue(root.get(agg).asInstanceOf[Cardinality].value()) } else { NumericValue(root.get(agg).asInstanceOf[ValueCount].value()) } - case sql.Sum => + case sql.function.aggregate.Sum => NumericValue(root.get(agg).asInstanceOf[Sum].value()) - case sql.Avg => + case sql.function.aggregate.Avg => NumericValue(root.get(agg).asInstanceOf[Avg].value()) - case sql.Min => + case sql.function.aggregate.Min => NumericValue(root.get(agg).asInstanceOf[Min].value()) - case sql.Max => + case sql.function.aggregate.Max => NumericValue(root.get(agg).asInstanceOf[Max].value()) case _ => EmptyValue }, diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 7938b1a8..52e1607f 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -401,7 +401,7 @@ trait ElasticsearchClientSingleValueAggregateApi field, aggType, aggType match { - case sql.Count => + case sql.function.aggregate.Count => NumericValue( if (aggregation.distinct) { root.get(agg).cardinality().value().toDouble @@ -409,15 +409,15 @@ trait ElasticsearchClientSingleValueAggregateApi root.get(agg).valueCount().value() } ) - case sql.Sum => + case sql.function.aggregate.Sum => NumericValue(root.get(agg).sum().value()) - case sql.Avg => + case sql.function.aggregate.Avg => val avgAgg = root.get(agg).avg() aggregateValue(avgAgg.value(), avgAgg.valueAsString()) - case sql.Min => + case sql.function.aggregate.Min => val minAgg = root.get(agg).min() aggregateValue(minAgg.value(), minAgg.valueAsString()) - case sql.Max => + case sql.function.aggregate.Max => val maxAgg = root.get(agg).max() aggregateValue(maxAgg.value(), maxAgg.valueAsString()) case _ => EmptyValue diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index a6cfb3a0..96cf2c3c 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -396,7 +396,7 @@ trait ElasticsearchClientSingleValueAggregateApi field, aggType, aggType match { - case sql.Count => + case sql.function.aggregate.Count => NumericValue( if (aggregation.distinct) { root.get(agg).cardinality().value().toDouble @@ -404,15 +404,15 @@ trait ElasticsearchClientSingleValueAggregateApi root.get(agg).valueCount().value() } ) - case sql.Sum => + case sql.function.aggregate.Sum => NumericValue(root.get(agg).sum().value()) - case sql.Avg => + case sql.function.aggregate.Avg => val avgAgg = root.get(agg).avg() aggregateValue(avgAgg.value(), avgAgg.valueAsString()) - case sql.Min => + case sql.function.aggregate.Min => val minAgg = root.get(agg).min() aggregateValue(minAgg.value(), minAgg.valueAsString()) - case sql.Max => + case sql.function.aggregate.Max => val maxAgg = root.get(agg).max() aggregateValue(maxAgg.value(), maxAgg.valueAsString()) case _ => EmptyValue diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 1bedbaa4..2835af2b 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -1,21 +1,16 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.{ - AggregateFunction, Asc, - Avg, BucketSelectorScript, - Count, ElasticBoolQuery, Field, - Max, - Min, - SQLBucket, - SQLCriteria, - SQLFunctionUtils, - SortOrder, - Sum + Bucket, + Criteria, + SortOrder } +import app.softnetwork.elastic.sql.function._ +import app.softnetwork.elastic.sql.function.aggregate._ import com.sksamuel.elastic4s.ElasticApi.{ avgAgg, bucketSelectorAggregation, @@ -57,9 +52,9 @@ case class ElasticAggregation( object ElasticAggregation { def apply( - sqlAgg: Field, - having: Option[SQLCriteria], - bucketsDirection: Map[String, SortOrder] + sqlAgg: Field, + having: Option[Criteria], + bucketsDirection: Map[String, SortOrder] ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.name @@ -88,7 +83,7 @@ object ElasticAggregation { var aggPath = Seq[String]() - val (aggFuncs, transformFuncs) = SQLFunctionUtils.aggregateAndTransformFunctions(identifier) + val (aggFuncs, transformFuncs) = FunctionUtils.aggregateAndTransformFunctions(identifier) require(aggFuncs.size == 1, s"Multiple aggregate functions not supported: $aggFuncs") @@ -171,11 +166,11 @@ object ElasticAggregation { } def buildBuckets( - buckets: Seq[SQLBucket], - bucketsDirection: Map[String, SortOrder], - aggregations: Seq[Aggregation], - aggregationsDirection: Map[String, SortOrder], - having: Option[SQLCriteria] + buckets: Seq[Bucket], + bucketsDirection: Map[String, SortOrder], + aggregations: Seq[Aggregation], + aggregationsDirection: Map[String, SortOrder], + having: Option[Criteria] ): Option[TermsAggregation] = { Console.println(bucketsDirection) buckets.reverse.foldLeft(Option.empty[TermsAggregation]) { (current, bucket) => diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index d6542758..6abfb525 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -1,9 +1,9 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.SQLCriteria +import app.softnetwork.elastic.sql.Criteria import com.sksamuel.elastic4s.requests.searches.queries.Query -case class ElasticCriteria(criteria: SQLCriteria) { +case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { val query = criteria.boolQuery.copy(group = group) diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 3a532263..b485973d 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -8,13 +8,13 @@ import app.softnetwork.elastic.sql.{ ElasticMatch, ElasticNested, ElasticParent, - SQLBetween, - SQLExpression, - SQLIn, - SQLIsNotNull, - SQLIsNotNullCriteria, - SQLIsNull, - SQLIsNullCriteria + BetweenExpr, + GenericExpression, + InExpr, + IsNotNullExpr, + IsNotNullCriteria, + IsNullExpr, + IsNullCriteria } import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.requests.searches.queries.Query @@ -62,17 +62,17 @@ case class ElasticQuery(filter: ElasticFilter) { criteria.asQuery(group = group, innerHitsNames = innerHitsNames), score = false ) - case expression: SQLExpression => expression - case isNull: SQLIsNull => isNull - case isNotNull: SQLIsNotNull => isNotNull - case in: SQLIn[_, _] => in - case between: SQLBetween[String] => between - case between: SQLBetween[Long] => between - case between: SQLBetween[Double] => between + case expression: GenericExpression => expression + case isNull: IsNullExpr => isNull + case isNotNull: IsNotNullExpr => isNotNull + case in: InExpr[_, _] => in + case between: BetweenExpr[String] => between + case between: BetweenExpr[Long] => between + case between: BetweenExpr[Double] => between case geoDistance: ElasticGeoDistance => geoDistance case matchExpression: ElasticMatch => matchExpression - case isNull: SQLIsNullCriteria => isNull - case isNotNull: SQLIsNotNullCriteria => isNotNull + case isNull: IsNullCriteria => isNull + case isNotNull: IsNotNullCriteria => isNotNull case other => throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index adcf87e1..fac950c9 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -1,17 +1,17 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{SQLBucket, SQLCriteria, SQLExcept, Field} +import app.softnetwork.elastic.sql.{Bucket, Criteria, Except, Field} import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequest} case class ElasticSearchRequest( - fields: Seq[Field], - except: Option[SQLExcept], - sources: Seq[String], - criteria: Option[SQLCriteria], - limit: Option[Int], - search: SearchRequest, - buckets: Seq[SQLBucket] = Seq.empty, - aggregations: Seq[ElasticAggregation] = Seq.empty + fields: Seq[Field], + except: Option[Except], + sources: Seq[String], + criteria: Option[Criteria], + limit: Option[Int], + search: SearchRequest, + buckets: Seq[Bucket] = Seq.empty, + aggregations: Seq[ElasticAggregation] = Seq.empty ) { def minScore(score: Option[Double]): ElasticSearchRequest = { score match { diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 84d5b845..a04d34cf 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,5 +1,8 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.function.aggregate.Count +import app.softnetwork.elastic.sql.operator._ + import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.requests.script.Script @@ -137,12 +140,12 @@ package object bridge { ) } - def applyNumericOp[A](n: SQLNumericValue[_])( + def applyNumericOp[A](n: NumericValue[_])( longOp: Long => A, doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: SQLExpression): Query = { + implicit def expressionToQuery(expression: GenericExpression): Query = { import expression._ if (aggregation) return matchAllQuery() @@ -150,7 +153,7 @@ package object bridge { return scriptQuery(Script(script = painless).lang("painless").scriptType("source")) } value match { - case n: SQLNumericValue[_] => + case n: NumericValue[_] => operator match { case Ge => maybeNot match { @@ -232,7 +235,7 @@ package object bridge { } case _ => matchAllQuery() } - case l: SQLStringValue => + case l: StringValue => operator match { case Like => maybeNot match { @@ -285,7 +288,7 @@ package object bridge { } case _ => matchAllQuery() } - case b: SQLBoolean => + case b: BooleanValue => operator match { case Eq => maybeNot match { @@ -303,9 +306,9 @@ package object bridge { } case _ => matchAllQuery() } - case i: SQLIdentifier => + case i: GenericIdentifier => operator match { - case op: SQLComparisonOperator => + case op: ComparisonOperator => i.toScript match { case Some(script) => val o = if (maybeNot.isDefined) op.not else op @@ -328,34 +331,34 @@ package object bridge { } implicit def isNullToQuery( - isNull: SQLIsNull + isNull: IsNullExpr ): Query = { import isNull._ not(existsQuery(identifier.name)) } implicit def isNotNullToQuery( - isNotNull: SQLIsNotNull + isNotNull: IsNotNullExpr ): Query = { import isNotNull._ existsQuery(identifier.name) } implicit def isNullCriteriaToQuery( - isNull: SQLIsNullCriteria + isNull: IsNullCriteria ): Query = { import isNull._ not(existsQuery(identifier.name)) } implicit def isNotNullCriteriaToQuery( - isNotNull: SQLIsNotNullCriteria + isNotNull: IsNotNullCriteria ): Query = { import isNotNull._ existsQuery(identifier.name) } - implicit def inToQuery[R, T <: SQLValue[R]](in: SQLIn[R, T]): Query = { + implicit def inToQuery[R, T <: Value[R]](in: InExpr[R, T]): Query = { import in._ val _values: Seq[Any] = values.innerValues val t = @@ -375,7 +378,7 @@ package object bridge { } implicit def betweenToQuery( - between: SQLBetween[String] + between: BetweenExpr[String] ): Query = { import between._ val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value @@ -386,7 +389,7 @@ package object bridge { } implicit def betweenLongsToQuery( - between: SQLBetween[Long] + between: BetweenExpr[Long] ): Query = { import between._ val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value @@ -397,7 +400,7 @@ package object bridge { } implicit def betweenDoublesToQuery( - between: SQLBetween[Double] + between: BetweenExpr[Double] ): Query = { import between._ val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value @@ -422,7 +425,7 @@ package object bridge { } implicit def criteriaToElasticCriteria( - criteria: SQLCriteria + criteria: Criteria ): ElasticCriteria = { ElasticCriteria( criteria diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index d1f088f6..d8c85e3c 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -16,9 +16,9 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { def asQuery(sql: String): String = { import SQLImplicits._ - val criteria: Option[SQLCriteria] = sql + val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( - SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) + SQLSearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) ).string println(result) result diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala new file mode 100644 index 00000000..f3195f83 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala @@ -0,0 +1,16 @@ +package app.softnetwork.elastic.sql + +sealed trait Delimiter extends Token + +sealed trait StartDelimiter extends Delimiter + +case object StartPredicate extends Expr("(") with StartDelimiter +case object StartCase extends Expr("case") with StartDelimiter +case object WhenCase extends Expr("when") with StartDelimiter + +sealed trait EndDelimiter extends Delimiter + +case object EndPredicate extends Expr(")") with EndDelimiter +case object Separator extends Expr(",") with EndDelimiter +case object EndCase extends Expr("end") with EndDelimiter +case object ThenCase extends Expr("then") with EndDelimiter diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/From.scala new file mode 100644 index 00000000..3b4efdd6 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/From.scala @@ -0,0 +1,38 @@ +package app.softnetwork.elastic.sql + +case object From extends Expr("from") with TokenRegex + +case object Unnest extends Expr("unnest") with TokenRegex + +case class Unnest(identifier: GenericIdentifier, limit: Option[Limit]) extends Source { + override def sql: String = s"$Unnest($identifier${asString(limit)})" + def update(request: SQLSearchRequest): Unnest = + this.copy(identifier = identifier.update(request)) + override val name: String = identifier.name +} + +case class Table(source: Source, tableAlias: Option[Alias] = None) extends Updateable { + override def sql: String = s"$source${asString(tableAlias)}" + def update(request: SQLSearchRequest): Table = this.copy(source = source.update(request)) +} + +case class From(tables: Seq[Table]) extends Updateable { + override def sql: String = s" $From ${tables.map(_.sql).mkString(",")}" + lazy val tableAliases: Map[String, String] = tables + .flatMap((table: Table) => table.tableAlias.map(alias => table.source.name -> alias.alias)) + .toMap + lazy val unnests: Seq[(String, String, Option[Limit])] = tables.collect { + case Table(u: Unnest, a) => + (a.map(_.alias).getOrElse(u.identifier.name), u.identifier.name, u.limit) + } + def update(request: SQLSearchRequest): From = + this.copy(tables = tables.map(_.update(request))) + + override def validate(): Either[String, Unit] = { + if (tables.isEmpty) { + Left("At least one table is required in FROM clause") + } else { + Right(()) + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/GroupBy.scala similarity index 69% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/GroupBy.scala index 39220f68..ee73dc88 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLGroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/GroupBy.scala @@ -1,12 +1,15 @@ package app.softnetwork.elastic.sql -case object GroupBy extends SQLExpr("group by") with SQLRegex +import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.`type`.SQLTypes -case class SQLGroupBy(buckets: Seq[SQLBucket]) extends Updateable { +case object GroupBy extends Expr("group by") with TokenRegex + +case class GroupBy(buckets: Seq[Bucket]) extends Updateable { override def sql: String = s" $GroupBy ${buckets.mkString(", ")}" - def update(request: SQLSearchRequest): SQLGroupBy = + def update(request: SQLSearchRequest): GroupBy = this.copy(buckets = buckets.map(_.update(request))) - lazy val bucketNames: Map[String, SQLBucket] = buckets.map { b => + lazy val bucketNames: Map[String, Bucket] = buckets.map { b => b.identifier.identifierName -> b }.toMap @@ -19,11 +22,11 @@ case class SQLGroupBy(buckets: Seq[SQLBucket]) extends Updateable { } } -case class SQLBucket( - identifier: SQLIdentifier +case class Bucket( + identifier: GenericIdentifier ) extends Updateable { override def sql: String = s"$identifier" - def update(request: SQLSearchRequest): SQLBucket = + def update(request: SQLSearchRequest): Bucket = this.copy(identifier = identifier.update(request)) lazy val sourceBucket: String = if (identifier.nested) { @@ -41,23 +44,23 @@ case class SQLBucket( object BucketSelectorScript { - def extractBucketsPath(criteria: SQLCriteria): Map[String, String] = criteria match { - case SQLPredicate(left, _, right, _, _) => + def extractBucketsPath(criteria: Criteria): Map[String, String] = criteria match { + case Predicate(left, _, right, _, _) => extractBucketsPath(left) ++ extractBucketsPath(right) case relation: ElasticRelation => extractBucketsPath(relation.criteria) - case _: SQLMatch => Map.empty //MATCH is not supported in bucket_selector + case _: MatchCriteria => Map.empty //MATCH is not supported in bucket_selector case e: Expression if e.aggregation => import e._ maybeValue match { - case Some(v: SQLIdentifier) if v.aggregation => + case Some(v: GenericIdentifier) if v.aggregation => Map(identifier.aliasOrName -> identifier.aliasOrName, v.aliasOrName -> v.aliasOrName) case _ => Map(identifier.aliasOrName -> identifier.aliasOrName) } case _ => Map.empty } - def toPainless(expr: SQLCriteria): String = expr match { - case SQLPredicate(left, op, right, maybeNot, group) => + def toPainless(expr: Criteria): String = expr match { + case Predicate(left, op, right, maybeNot, group) => val leftStr = toPainless(left) val rightStr = toPainless(right) val opStr = op match { @@ -72,17 +75,17 @@ object BucketSelectorScript { case relation: ElasticRelation => toPainless(relation.criteria) - case _: SQLMatch => "1 == 1" //MATCH is not supported in bucket_selector + case _: MatchCriteria => "1 == 1" //MATCH is not supported in bucket_selector case e: Expression if e.aggregation => val paramName = e.identifier.paramName e.out match { - case SQLTypes.Date if e.operator.isInstanceOf[SQLComparisonOperator] => + case SQLTypes.Date if e.operator.isInstanceOf[ComparisonOperator] => // protect against null params and compare epoch millis s"($paramName != null) && (${e.painless}.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli())" - case SQLTypes.Time if e.operator.isInstanceOf[SQLComparisonOperator] => + case SQLTypes.Time if e.operator.isInstanceOf[ComparisonOperator] => s"($paramName != null) && (${e.painless}.truncatedTo(ChronoUnit.SECONDS).toInstant().toEpochMilli())" - case SQLTypes.DateTime if e.operator.isInstanceOf[SQLComparisonOperator] => + case SQLTypes.DateTime if e.operator.isInstanceOf[ComparisonOperator] => s"($paramName != null) && (${e.painless}.toInstant().toEpochMilli())" case _ => e.painless diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLHaving.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/Having.scala similarity index 62% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLHaving.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/Having.scala index 97ed5dc9..0c7bda43 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLHaving.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/Having.scala @@ -1,13 +1,13 @@ package app.softnetwork.elastic.sql -case object Having extends SQLExpr("having") with SQLRegex +case object Having extends Expr("having") with TokenRegex -case class SQLHaving(criteria: Option[SQLCriteria]) extends Updateable { +case class Having(criteria: Option[Criteria]) extends Updateable { override def sql: String = criteria match { case Some(c) => s" $Having $c" case _ => "" } - def update(request: SQLSearchRequest): SQLHaving = + def update(request: SQLSearchRequest): Having = this.copy(criteria = criteria.map(_.update(request))) override def validate(): Either[String, Unit] = criteria.map(_.validate()).getOrElse(Right(())) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala new file mode 100644 index 00000000..65ae276b --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala @@ -0,0 +1,5 @@ +package app.softnetwork.elastic.sql + +case object Limit extends Expr("limit") with TokenRegex + +case class Limit(limit: Int) extends Expr(s" limit $limit") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala new file mode 100644 index 00000000..c41efa44 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala @@ -0,0 +1,25 @@ +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.function.{Function, FunctionChain} + +case object OrderBy extends Expr("order by") with TokenRegex + +sealed trait SortOrder extends TokenRegex + +case object Desc extends Expr("desc") with SortOrder + +case object Asc extends Expr("asc") with SortOrder + +case class FieldSort( + field: String, + order: Option[SortOrder], + functions: List[Function] = List.empty +) extends FunctionChain { + lazy val direction: SortOrder = order.getOrElse(Asc) + lazy val name: String = toSQL(field) + override def sql: String = s"$name $direction" +} + +case class OrderBy(sorts: Seq[FieldSort]) extends Token { + override def sql: String = s" $OrderBy ${sorts.mkString(", ")}" +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLDelimiter.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLDelimiter.scala deleted file mode 100644 index 1b0b791b..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLDelimiter.scala +++ /dev/null @@ -1,14 +0,0 @@ -package app.softnetwork.elastic.sql - -sealed trait SQLDelimiter extends SQLToken - -sealed trait StartDelimiter extends SQLDelimiter -case object StartPredicate extends SQLExpr("(") with StartDelimiter -case object StartCase extends SQLExpr("case") with StartDelimiter -case object WhenCase extends SQLExpr("when") with StartDelimiter - -sealed trait EndDelimiter extends SQLDelimiter -case object EndPredicate extends SQLExpr(")") with EndDelimiter -case object Separator extends SQLExpr(",") with EndDelimiter -case object EndCase extends SQLExpr("end") with EndDelimiter -case object ThenCase extends SQLExpr("then") with EndDelimiter diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLFrom.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLFrom.scala deleted file mode 100644 index dc7e65a9..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLFrom.scala +++ /dev/null @@ -1,38 +0,0 @@ -package app.softnetwork.elastic.sql - -case object From extends SQLExpr("from") with SQLRegex - -case object Unnest extends SQLExpr("unnest") with SQLRegex - -case class SQLUnnest(identifier: SQLIdentifier, limit: Option[SQLLimit]) extends SQLSource { - override def sql: String = s"$Unnest($identifier${asString(limit)})" - def update(request: SQLSearchRequest): SQLUnnest = - this.copy(identifier = identifier.update(request)) - override val name: String = identifier.name -} - -case class SQLTable(source: SQLSource, tableAlias: Option[SQLAlias] = None) extends Updateable { - override def sql: String = s"$source${asString(tableAlias)}" - def update(request: SQLSearchRequest): SQLTable = this.copy(source = source.update(request)) -} - -case class SQLFrom(tables: Seq[SQLTable]) extends Updateable { - override def sql: String = s" $From ${tables.map(_.sql).mkString(",")}" - lazy val tableAliases: Map[String, String] = tables - .flatMap((table: SQLTable) => table.tableAlias.map(alias => table.source.name -> alias.alias)) - .toMap - lazy val unnests: Seq[(String, String, Option[SQLLimit])] = tables.collect { - case SQLTable(u: SQLUnnest, a) => - (a.map(_.alias).getOrElse(u.identifier.name), u.identifier.name, u.limit) - } - def update(request: SQLSearchRequest): SQLFrom = - this.copy(tables = tables.map(_.update(request))) - - override def validate(): Either[String, Unit] = { - if (tables.isEmpty) { - Left("At least one table is required in FROM clause") - } else { - Right(()) - } - } -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLFunction.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLFunction.scala deleted file mode 100644 index b101a66f..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLFunction.scala +++ /dev/null @@ -1,1122 +0,0 @@ -package app.softnetwork.elastic.sql - -import scala.util.matching.Regex - -sealed trait SQLFunction extends SQLRegex { - def toSQL(base: String): String = if (base.nonEmpty) s"$sql($base)" else sql - def applyType(in: SQLType): SQLType = out - private[this] var _expr: SQLToken = SQLNull - def expr_=(e: SQLToken): Unit = { - _expr = e - } - def expr: SQLToken = _expr - override def nullable: Boolean = expr.nullable -} - -sealed trait SQLFunctionWithIdentifier extends SQLFunction { - def identifier: SQLIdentifier //= SQLIdentifier("", functions = this :: Nil) -} - -trait SQLFunctionWithValue[+T] extends SQLFunction { - def value: T -} - -object SQLFunctionUtils { - def aggregateAndTransformFunctions( - chain: SQLFunctionChain - ): (List[SQLFunction], List[SQLFunction]) = { - chain.functions.partition { - case _: AggregateFunction => true - case _ => false - } - } - - def transformFunctions(chain: SQLFunctionChain): List[SQLFunction] = { - aggregateAndTransformFunctions(chain)._2 - } - -} - -trait SQLFunctionChain extends SQLFunction { - def functions: List[SQLFunction] - - override def validate(): Either[String, Unit] = { - if (aggregations.size > 1) { - Left("Only one aggregation function is allowed in a function chain") - } else if (aggregations.size == 1 && !functions.head.isInstanceOf[AggregateFunction]) { - Left("Aggregation function must be the first function in the chain") - } else { - SQLValidator.validateChain(functions) - } - } - - override def toSQL(base: String): String = - functions.reverse.foldLeft(base)((expr, fun) => { - fun.toSQL(expr) - }) - - def toScript: Option[String] = { - val orderedFunctions = SQLFunctionUtils.transformFunctions(this).reverse - orderedFunctions.foldLeft(Option("")) { - case (expr, f: MathScript) if expr.isDefined => Option(s"${expr.get}${f.script}") - case (_, _) => None // ignore non math scripts - } match { - case Some(s) if s.nonEmpty => - out match { - case SQLTypes.Date => Some(s"$s/d") - case _ => Some(s) - } - case _ => None - } - } - - override def system: Boolean = functions.lastOption.exists(_.system) - - def applyTo(expr: SQLToken): Unit = { - this.expr = expr - functions.reverse.foldLeft(expr) { (currentExpr, fun) => - fun.expr = currentExpr - fun - } - } - - private[this] lazy val aggregations = functions.collect { case af: AggregateFunction => - af - } - - lazy val aggregateFunction: Option[AggregateFunction] = aggregations.headOption - - lazy val aggregation: Boolean = aggregateFunction.isDefined - - override def in: SQLType = functions.lastOption.map(_.in).getOrElse(super.in) - - override def out: SQLType = { - val baseType = functions.lastOption.map(_.in).getOrElse(super.baseType) - functions.reverse.foldLeft(baseType) { (currentType, fun) => - fun.applyType(currentType) - } - } - - def arithmetic: Boolean = functions.nonEmpty && functions.forall { - case _: SQLArithmeticExpression => true - case _ => false - } -} - -sealed trait SQLFunctionN[In <: SQLType, Out <: SQLType] extends SQLFunction with PainlessScript { - def fun: Option[PainlessScript] = None - - def args: List[PainlessScript] - def argsSeparator: String = ", " - - def inputType: In - def outputType: Out - - override def in: SQLType = inputType - override def out: SQLType = outputType - - override def applyType(in: SQLType): SQLType = outputType - - override def sql: String = - s"${fun.map(_.sql).getOrElse("")}(${args.map(_.sql).mkString(argsSeparator)})" - - override def toSQL(base: String): String = s"$base$sql" - - override def painless: String = { - val nullCheck = - args.filter(_.nullable).zipWithIndex.map { case (_, i) => s"arg$i == null" }.mkString(" || ") - - val assignments = - args - .filter(_.nullable) - .zipWithIndex - .map { case (a, i) => s"def arg$i = ${a.painless};" } - .mkString(" ") - - val callArgs = args.zipWithIndex - .map { case (a, i) => - if (a.nullable) - s"arg$i" - else - a.painless - } - - if (args.exists(_.nullable)) - s"($assignments ($nullCheck) ? null : ${toPainlessCall(callArgs)})" - else - s"${toPainlessCall(callArgs)}" - } - - def toPainlessCall(callArgs: List[String]): String = - if (callArgs.nonEmpty) - s"${fun.map(_.painless).getOrElse("")}(${callArgs.mkString(argsSeparator)})" - else - fun.map(_.painless).getOrElse("") -} - -sealed trait SQLBinaryFunction[In1 <: SQLType, In2 <: SQLType, Out <: SQLType] - extends SQLFunctionN[In2, Out] { self: SQLFunction => - - def left: PainlessScript - def right: PainlessScript - - override def args: List[PainlessScript] = List(left, right) - - override def nullable: Boolean = left.nullable || right.nullable -} - -sealed trait SQLTransformFunction[In <: SQLType, Out <: SQLType] extends SQLFunctionN[In, Out] { - def toPainless(base: String, idx: Int): String = { - if (nullable && base.nonEmpty) - s"(def e$idx = $base; e$idx != null ? e$idx$painless : null)" - else - s"$base$painless" - } -} - -sealed trait AggregateFunction extends SQLFunction -case object Count extends SQLExpr("count") with AggregateFunction -case object Min extends SQLExpr("min") with AggregateFunction -case object Max extends SQLExpr("max") with AggregateFunction -case object Avg extends SQLExpr("avg") with AggregateFunction -case object Sum extends SQLExpr("sum") with AggregateFunction - -case object Distance extends SQLExpr("distance") with SQLFunction with SQLOperator - -sealed trait TimeUnit extends PainlessScript with MathScript { - lazy val regex: Regex = s"\\b(?i)$sql(s)?\\b".r - - override def painless: String = s"ChronoUnit.${sql.toUpperCase()}S" - - override def nullable: Boolean = false -} - -sealed trait CalendarUnit extends TimeUnit -sealed trait FixedUnit extends TimeUnit - -object TimeUnit { - case object Year extends SQLExpr("year") with CalendarUnit { - override def script: String = "y" - } - case object Month extends SQLExpr("month") with CalendarUnit { - override def script: String = "M" - } - case object Quarter extends SQLExpr("quarter") with CalendarUnit { - override def script: String = throw new IllegalArgumentException( - "Quarter must be converted to months (value * 3) before creating date-math" - ) - } - case object Week extends SQLExpr("week") with CalendarUnit { - override def script: String = "w" - } - - case object Day extends SQLExpr("day") with CalendarUnit with FixedUnit { - override def script: String = "d" - } - - case object Hour extends SQLExpr("hour") with FixedUnit { - override def script: String = "H" - } - case object Minute extends SQLExpr("minute") with FixedUnit { - override def script: String = "m" - } - case object Second extends SQLExpr("second") with FixedUnit { - override def script: String = "s" - } - -} - -case object Interval extends SQLExpr("interval") with SQLFunction with SQLRegex - -sealed trait TimeInterval extends PainlessScript with MathScript { - def value: Int - def unit: TimeUnit - override def sql: String = s"$Interval $value ${unit.sql}" - - override def painless: String = s"$value, ${unit.painless}" - - override def script: String = TimeInterval.script(this) - - def checkType(in: SQLType): Either[String, SQLType] = { - import TimeUnit._ - in match { - case SQLTypes.Date => - unit match { - case Year | Month | Day => Right(SQLTypes.Date) - case Hour | Minute | Second => Right(SQLTypes.Timestamp) - case _ => Left(s"Invalid interval unit $unit for DATE") - } - case SQLTypes.Time => - unit match { - case Hour | Minute | Second => Right(SQLTypes.Time) - case _ => Left(s"Invalid interval unit $unit for TIME") - } - case SQLTypes.DateTime => - Right(SQLTypes.Timestamp) - case SQLTypes.Timestamp => - Right(SQLTypes.Timestamp) - case SQLTypes.Temporal => - Right(SQLTypes.Timestamp) - case _ => - Left(s"Intervals not supported for type $in") - } - } - - override def nullable: Boolean = false -} - -import TimeUnit._ - -case class CalendarInterval(value: Int, unit: CalendarUnit) extends TimeInterval -case class FixedInterval(value: Int, unit: FixedUnit) extends TimeInterval - -object TimeInterval { - def apply(value: Int, unit: TimeUnit): TimeInterval = unit match { - case cu: CalendarUnit => CalendarInterval(value, cu) - case fu: FixedUnit => FixedInterval(value, fu) - } - def script(interval: TimeInterval): String = interval match { - case CalendarInterval(v, Quarter) => s"${v * 3}M" - case CalendarInterval(v, u) => s"$v${u.script}" - case FixedInterval(v, u) => s"$v${u.script}" - } -} - -sealed trait SQLIntervalFunction[IO <: SQLTemporal] - extends SQLTransformFunction[IO, IO] - with MathScript { - def operator: IntervalOperator - - override def fun: Option[IntervalOperator] = Some(operator) - - def interval: TimeInterval - - override def args: List[PainlessScript] = List(interval) - - override def argsSeparator: String = " " - override def sql: String = s"$operator${args.map(_.sql).mkString(argsSeparator)}" - - override def script: String = s"${operator.script}${interval.script}" - - private[this] var _out: SQLType = outputType - - override def out: SQLType = _out - - override def applyType(in: SQLType): SQLType = { - _out = interval.checkType(in).getOrElse(out) - _out - } - - override def validate(): Either[String, Unit] = interval.checkType(out) match { - case Left(err) => Left(err) - case Right(_) => Right(()) - } - - override def toPainless(base: String, idx: Int): String = - if (nullable) - s"(def e$idx = $base; e$idx != null ? ${SQLTypeUtils.coerce(s"e$idx", expr.out, out, nullable = false)}$painless : null)" - else - s"${SQLTypeUtils.coerce(base, expr.out, out, nullable = expr.nullable)}$painless" -} - -sealed trait AddInterval[IO <: SQLTemporal] extends SQLIntervalFunction[IO] { - override def operator: IntervalOperator = Plus -} - -sealed trait SubtractInterval[IO <: SQLTemporal] extends SQLIntervalFunction[IO] { - override def operator: IntervalOperator = Minus -} - -case class SQLAddInterval(interval: TimeInterval) extends AddInterval[SQLTemporal] { - override def inputType: SQLTemporal = SQLTypes.Temporal - override def outputType: SQLTemporal = SQLTypes.Temporal -} - -case class SQLSubtractInterval(interval: TimeInterval) extends SubtractInterval[SQLTemporal] { - override def inputType: SQLTemporal = SQLTypes.Temporal - override def outputType: SQLTemporal = SQLTypes.Temporal -} - -sealed trait DateTimeFunction extends SQLFunction { - def now: String = "ZonedDateTime.now(ZoneId.of('Z'))" - override def out: SQLType = SQLTypes.DateTime -} - -sealed trait DateFunction extends DateTimeFunction { - override def out: SQLType = SQLTypes.Date -} - -sealed trait TimeFunction extends DateTimeFunction { - override def out: SQLType = SQLTypes.Time -} - -sealed trait SystemFunction extends SQLFunction { - override def system: Boolean = true -} - -sealed trait CurrentFunction extends SystemFunction with PainlessScript - -sealed trait CurrentDateTimeFunction extends DateTimeFunction with CurrentFunction with MathScript { - override def painless: String = now - override def script: String = "now" -} - -sealed trait CurrentDateFunction extends DateFunction with CurrentFunction with MathScript { - override def painless: String = s"$now.toLocalDate()" - override def script: String = "now" -} - -sealed trait CurrentTimeFunction extends TimeFunction with CurrentFunction { - override def painless: String = s"$now.toLocalTime()" -} - -case object CurrentDate extends SQLExpr("current_date") with CurrentDateFunction - -case object CurentDateWithParens extends SQLExpr("current_date()") with CurrentDateFunction - -case object CurrentTime extends SQLExpr("current_time") with CurrentTimeFunction - -case object CurrentTimeWithParens extends SQLExpr("current_time()") with CurrentTimeFunction - -case object CurrentTimestamp extends SQLExpr("current_timestamp") with CurrentDateTimeFunction - -case object CurrentTimestampWithParens - extends SQLExpr("current_timestamp()") - with CurrentDateTimeFunction - -case object Now extends SQLExpr("now") with CurrentDateTimeFunction - -case object NowWithParens extends SQLExpr("now()") with CurrentDateTimeFunction - -case object DateTrunc extends SQLExpr("date_trunc") with SQLRegex with PainlessScript { - override def painless: String = ".truncatedTo" -} - -case class DateTrunc(identifier: SQLIdentifier, unit: TimeUnit) - extends DateTimeFunction - with SQLTransformFunction[SQLTemporal, SQLTemporal] - with SQLFunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(DateTrunc) - - override def args: List[PainlessScript] = List(unit) - - override def inputType: SQLTemporal = SQLTypes.Temporal // par défaut - override def outputType: SQLTemporal = SQLTypes.Temporal // idem - - override def sql: String = DateTrunc.sql - override def toSQL(base: String): String = { - s"$sql($base, ${unit.sql})" - } -} - -case object Extract extends SQLExpr("extract") with SQLRegex with PainlessScript { - override def painless: String = ".get" -} - -case class Extract(unit: TimeUnit, override val sql: String = "extract") - extends DateTimeFunction - with SQLTransformFunction[SQLTemporal, SQLNumeric] { - override def fun: Option[PainlessScript] = Some(Extract) - - override def args: List[PainlessScript] = List(unit) - - override def inputType: SQLTemporal = SQLTypes.Temporal - override def outputType: SQLNumeric = SQLTypes.Numeric - - override def toSQL(base: String): String = s"$sql(${unit.sql} from $base)" - -} - -object YEAR extends Extract(Year, Year.sql) { - override def toSQL(base: String): String = s"$sql($base)" -} - -object MONTH extends Extract(Month, Month.sql) { - override def toSQL(base: String): String = s"$sql($base)" -} - -object DAY extends Extract(Day, Day.sql) { - override def toSQL(base: String): String = s"$sql($base)" -} - -object HOUR extends Extract(Hour, Hour.sql) { - override def toSQL(base: String): String = s"$sql($base)" -} - -object MINUTE extends Extract(Minute, Minute.sql) { - override def toSQL(base: String): String = s"$sql($base)" -} - -object SECOND extends Extract(Second, Second.sql) { - override def toSQL(base: String): String = s"$sql($base)" -} - -case object DateDiff extends SQLExpr("date_diff") with SQLRegex with PainlessScript { - override def painless: String = ".between" -} - -case class DateDiff(end: PainlessScript, start: PainlessScript, unit: TimeUnit) - extends DateTimeFunction - with SQLBinaryFunction[SQLDateTime, SQLDateTime, SQLNumeric] - with PainlessScript { - override def fun: Option[PainlessScript] = Some(DateDiff) - - override def inputType: SQLDateTime = SQLTypes.DateTime - override def outputType: SQLNumeric = SQLTypes.Numeric - - override def left: PainlessScript = start - override def right: PainlessScript = end - - override def sql: String = DateDiff.sql - - override def toSQL(base: String): String = s"$sql(${end.sql}, ${start.sql}, ${unit.sql})" - - override def toPainlessCall(callArgs: List[String]): String = - s"${unit.painless}${DateDiff.painless}(${callArgs.mkString(", ")})" -} - -case object DateAdd extends SQLExpr("date_add") with SQLRegex - -case class DateAdd(identifier: SQLIdentifier, interval: TimeInterval) - extends DateFunction - with AddInterval[SQLDate] - with SQLTransformFunction[SQLDate, SQLDate] - with SQLFunctionWithIdentifier { - override def inputType: SQLDate = SQLTypes.Date - override def outputType: SQLDate = SQLTypes.Date - override def sql: String = DateAdd.sql - override def toSQL(base: String): String = { - s"$sql($base, ${interval.sql})" - } -} - -case object DateSub extends SQLExpr("date_sub") with SQLRegex - -case class DateSub(identifier: SQLIdentifier, interval: TimeInterval) - extends DateFunction - with SubtractInterval[SQLDate] - with SQLTransformFunction[SQLDate, SQLDate] - with SQLFunctionWithIdentifier { - override def inputType: SQLDate = SQLTypes.Date - override def outputType: SQLDate = SQLTypes.Date - override def sql: String = DateSub.sql - override def toSQL(base: String): String = { - s"$sql($base, ${interval.sql})" - } -} - -case object ParseDate extends SQLExpr("parse_date") with SQLRegex with PainlessScript { - override def painless: String = ".parse" -} - -case class ParseDate(identifier: SQLIdentifier, format: String) - extends DateFunction - with SQLTransformFunction[SQLVarchar, SQLDate] - with SQLFunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(ParseDate) - - override def args: List[PainlessScript] = List.empty - - override def inputType: SQLVarchar = SQLTypes.Varchar - override def outputType: SQLDate = SQLTypes.Date - - override def sql: String = ParseDate.sql - override def toSQL(base: String): String = { - s"$sql($base, '$format')" - } - - override def painless: String = throw new NotImplementedError("Use toPainless instead") - override def toPainless(base: String, idx: Int): String = - if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').parse(e$idx, LocalDate::from) : null)" - else - s"DateTimeFormatter.ofPattern('$format').parse($base, LocalDate::from)" -} - -case object FormatDate extends SQLExpr("format_date") with SQLRegex with PainlessScript { - override def painless: String = ".format" -} - -case class FormatDate(identifier: SQLIdentifier, format: String) - extends DateFunction - with SQLTransformFunction[SQLDate, SQLVarchar] - with SQLFunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(FormatDate) - - override def args: List[PainlessScript] = List.empty - - override def inputType: SQLDate = SQLTypes.Date - override def outputType: SQLVarchar = SQLTypes.Varchar - - override def sql: String = FormatDate.sql - override def toSQL(base: String): String = { - s"$sql($base, '$format')" - } - - override def painless: String = throw new NotImplementedError("Use toPainless instead") - override def toPainless(base: String, idx: Int): String = - if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').format(e$idx) : null)" - else - s"DateTimeFormatter.ofPattern('$format').format($base)" -} - -case object DateTimeAdd extends SQLExpr("datetime_add") with SQLRegex - -case class DateTimeAdd(identifier: SQLIdentifier, interval: TimeInterval) - extends DateTimeFunction - with AddInterval[SQLDateTime] - with SQLTransformFunction[SQLDateTime, SQLDateTime] - with SQLFunctionWithIdentifier { - override def inputType: SQLDateTime = SQLTypes.DateTime - override def outputType: SQLDateTime = SQLTypes.DateTime - override def sql: String = DateTimeAdd.sql - override def toSQL(base: String): String = { - s"$sql($base, ${interval.sql})" - } -} - -case object DateTimeSub extends SQLExpr("datetime_sub") with SQLRegex - -case class DateTimeSub(identifier: SQLIdentifier, interval: TimeInterval) - extends DateTimeFunction - with SubtractInterval[SQLDateTime] - with SQLTransformFunction[SQLDateTime, SQLDateTime] - with SQLFunctionWithIdentifier { - override def inputType: SQLDateTime = SQLTypes.DateTime - override def outputType: SQLDateTime = SQLTypes.DateTime - override def sql: String = DateTimeSub.sql - override def toSQL(base: String): String = { - s"$sql($base, ${interval.sql})" - } -} - -case object ParseDateTime extends SQLExpr("parse_datetime") with SQLRegex with PainlessScript { - override def painless: String = ".parse" -} - -case class ParseDateTime(identifier: SQLIdentifier, format: String) - extends DateTimeFunction - with SQLTransformFunction[SQLVarchar, SQLDateTime] - with SQLFunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(ParseDateTime) - - override def args: List[PainlessScript] = List.empty - - override def inputType: SQLVarchar = SQLTypes.Varchar - override def outputType: SQLDateTime = SQLTypes.DateTime - - override def sql: String = ParseDateTime.sql - override def toSQL(base: String): String = { - s"$sql($base, '$format')" - } - - override def painless: String = throw new NotImplementedError("Use toPainless instead") - override def toPainless(base: String, idx: Int): String = - if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').parse(e$idx, ZonedDateTime::from) : null)" - else - s"DateTimeFormatter.ofPattern('$format').parse($base, ZonedDateTime::from)" -} - -case object FormatDateTime extends SQLExpr("format_datetime") with SQLRegex with PainlessScript { - override def painless: String = ".format" -} - -case class FormatDateTime(identifier: SQLIdentifier, format: String) - extends DateTimeFunction - with SQLTransformFunction[SQLDateTime, SQLVarchar] - with SQLFunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(FormatDateTime) - - override def args: List[PainlessScript] = List.empty - - override def inputType: SQLDateTime = SQLTypes.DateTime - override def outputType: SQLVarchar = SQLTypes.Varchar - - override def sql: String = FormatDateTime.sql - override def toSQL(base: String): String = { - s"$sql($base, '$format')" - } - - override def painless: String = throw new NotImplementedError("Use toPainless instead") - override def toPainless(base: String, idx: Int): String = - if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').format(e$idx) : null)" - else - s"DateTimeFormatter.ofPattern('$format').format($base)" -} - -sealed trait SQLConditionalFunction[In <: SQLType] - extends SQLTransformFunction[In, SQLBool] - with SQLFunctionWithIdentifier { - def operator: SQLConditionalOperator - - override def fun: Option[PainlessScript] = Some(operator) - - override def outputType: SQLBool = SQLTypes.Boolean - - override def toPainless(base: String, idx: Int): String = s"($base$painless)" -} - -case class SQLIsNullFunction(identifier: SQLIdentifier) extends SQLConditionalFunction[SQLAny] { - override def operator: SQLConditionalOperator = IsNullFunction - - override def args: List[PainlessScript] = List(identifier) - - override def inputType: SQLAny = SQLTypes.Any - - override def toSQL(base: String): String = sql - - override def painless: String = s" == null" - override def toPainless(base: String, idx: Int): String = { - if (nullable) - s"(def e$idx = $base; e$idx$painless)" - else - s"$base$painless" - } -} - -case class SQLIsNotNullFunction(identifier: SQLIdentifier) extends SQLConditionalFunction[SQLAny] { - override def operator: SQLConditionalOperator = IsNotNullFunction - - override def args: List[PainlessScript] = List(identifier) - - override def inputType: SQLAny = SQLTypes.Any - - override def toSQL(base: String): String = sql - - override def painless: String = s" != null" - override def toPainless(base: String, idx: Int): String = { - if (nullable) - s"(def e$idx = $base; e$idx$painless)" - else - s"$base$painless" - } -} - -case class SQLCoalesce(values: List[PainlessScript]) - extends SQLTransformFunction[SQLAny, SQLType] - with SQLFunctionWithIdentifier { - def operator: SQLConditionalOperator = Coalesce - - override def fun: Option[SQLConditionalOperator] = Some(operator) - - override def args: List[PainlessScript] = values - - override def outputType: SQLType = SQLTypeUtils.leastCommonSuperType(args.map(_.out)) - - override def identifier: SQLIdentifier = SQLIdentifier("") - - override def inputType: SQLAny = SQLTypes.Any - - override def sql: String = s"$Coalesce(${values.map(_.sql).mkString(", ")})" - - // Reprend l’idée de SQLValues mais pour n’importe quel token - override def out: SQLType = SQLTypeUtils.leastCommonSuperType(values.map(_.out).distinct) - - override def applyType(in: SQLType): SQLType = out - - override def validate(): Either[String, Unit] = { - if (values.isEmpty) Left("COALESCE requires at least one argument") - else Right(()) - } - - override def toPainless(base: String, idx: Int): String = s"$base$painless" - - override def painless: String = { - require(values.nonEmpty, "COALESCE requires at least one argument") - - val checks = values - .take(values.length - 1) - .zipWithIndex - .map { case (v, index) => - var check = s"def v$index = ${SQLTypeUtils.coerce(v, out)};" - check += s"if (v$index != null) return v$index;" - check - } - .mkString(" ") - // final fallback - s"{ $checks return ${SQLTypeUtils.coerce(values.last, out)}; }" - } - - override def nullable: Boolean = values.forall(_.nullable) -} - -case class SQLNullIf(expr1: PainlessScript, expr2: PainlessScript) - extends SQLConditionalFunction[SQLAny] { - override def operator: SQLConditionalOperator = NullIf - - override def args: List[PainlessScript] = List(expr1, expr2) - - override def identifier: SQLIdentifier = SQLIdentifier("") - - override def inputType: SQLAny = SQLTypes.Any - - override def out: SQLType = expr1.out - - override def applyType(in: SQLType): SQLType = out - - override def toPainlessCall(callArgs: List[String]): String = { - callArgs match { - case List(arg0, arg1) => s"${arg0.trim} == ${arg1.trim} ? null : $arg0" - case _ => throw new IllegalArgumentException("NULLIF requires exactly two arguments") - } - } -} - -case class SQLCast(value: PainlessScript, targetType: SQLType, as: Boolean = true) - extends SQLTransformFunction[SQLType, SQLType] { - override def inputType: SQLType = value.out - override def outputType: SQLType = targetType - - override def args: List[PainlessScript] = List.empty - - override def sql: String = - s"$Cast(${value.sql} ${if (as) s"$Alias " else ""}${targetType.typeId})" - - override def toSQL(base: String): String = sql - - override def painless: String = - SQLTypeUtils.coerce(value, targetType) - - override def toPainless(base: String, idx: Int): String = - SQLTypeUtils.coerce(base, value.out, targetType, value.nullable) -} - -case class SQLCaseWhen( - expression: Option[PainlessScript], - conditions: List[(PainlessScript, PainlessScript)], - default: Option[PainlessScript] -) extends SQLTransformFunction[SQLAny, SQLAny] { - override def args: List[PainlessScript] = List.empty - - override def inputType: SQLAny = SQLTypes.Any - override def outputType: SQLAny = SQLTypes.Any - - override def sql: String = { - val exprPart = expression.map(e => s"$Case ${e.sql}").getOrElse(Case.sql) - val whenThen = conditions - .map { case (cond, res) => s"$When ${cond.sql} $Then ${res.sql}" } - .mkString(" ") - val elsePart = default.map(d => s" $Else ${d.sql}").getOrElse("") - s"$exprPart $whenThen$elsePart $End" - } - - override def out: SQLType = - SQLTypeUtils.leastCommonSuperType( - conditions.map(_._2.out) ++ default.map(_.out).toList - ) - - override def applyType(in: SQLType): SQLType = out - - override def validate(): Either[String, Unit] = { - if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") - else if ( - expression.isEmpty && conditions.exists { case (cond, _) => cond.out != SQLTypes.Boolean } - ) - Left("CASE WHEN conditions must be of type BOOLEAN") - else if ( - expression.isDefined && conditions.exists { case (cond, _) => - !SQLTypeUtils.matches(cond.out, expression.get.out) - } - ) - Left("CASE WHEN conditions must be of the same type as the expression") - else Right(()) - } - - override def painless: String = { - val base = - expression match { - case Some(expr) => - s"def expr = ${SQLTypeUtils.coerce(expr, expr.out)}; " - case _ => "" - } - val cases = conditions.zipWithIndex - .map { case ((cond, res), idx) => - val name = - cond match { - case e: Expression => - e.identifier.name - case i: Identifier => - i.name - case _ => "" - } - expression match { - case Some(expr) => - val c = SQLTypeUtils.coerce(cond, expr.out) - if (cond.sql == res.sql) { - s"def val$idx = $c; if (expr == val$idx) return val$idx;" - } else { - res match { - case i: Identifier if i.name == name && cond.isInstanceOf[Identifier] => - i.nullable = false - if (cond.asInstanceOf[Identifier].functions.isEmpty) - s"def val$idx = $c; if (expr == val$idx) return ${SQLTypeUtils.coerce(i.toPainless(s"val$idx"), i.out, out, nullable = false)};" - else { - cond.asInstanceOf[Identifier].nullable = false - s"def e$idx = ${i.checkNotNull}; def val$idx = e$idx != null ? ${SQLTypeUtils - .coerce(cond.asInstanceOf[Identifier].toPainless(s"e$idx"), cond.out, out, nullable = false)} : null; if (expr == val$idx) return ${SQLTypeUtils - .coerce(i.toPainless(s"e$idx"), i.out, out, nullable = false)};" - } - case _ => - s"if (expr == $c) return ${SQLTypeUtils.coerce(res, out)};" - } - } - case None => - val c = SQLTypeUtils.coerce(cond, SQLTypes.Boolean) - val r = - res match { - case i: Identifier if i.name == name && cond.isInstanceOf[Expression] => - i.nullable = false - SQLTypeUtils.coerce(i.toPainless("left"), i.out, out, nullable = false) - case _ => SQLTypeUtils.coerce(res, out) - } - s"if ($c) return $r;" - } - } - .mkString(" ") - val defaultCase = default - .map(d => s"def dval = ${SQLTypeUtils.coerce(d, out)}; return dval;") - .getOrElse("return null;") - s"{ $base$cases $defaultCase }" - } - - override def toPainless(base: String, idx: Int): String = s"$base$painless" - - override def nullable: Boolean = - conditions.exists { case (_, res) => res.nullable } || default.forall(_.nullable) -} - -case class SQLArithmeticExpression( - left: PainlessScript, - operator: ArithmeticOperator, - right: PainlessScript, - group: Boolean = false -) extends SQLTransformFunction[SQLNumeric, SQLNumeric] - with SQLBinaryFunction[SQLNumeric, SQLNumeric, SQLNumeric] { - - override def fun: Option[ArithmeticOperator] = Some(operator) - - override def inputType: SQLNumeric = SQLTypes.Numeric - override def outputType: SQLNumeric = SQLTypes.Numeric - - override def applyType(in: SQLType): SQLType = in - - override def sql: String = { - val expr = s"${left.sql}$operator${right.sql}" - if (group) - s"($expr)" - else - expr - } - - override def out: SQLType = - SQLTypeUtils.leastCommonSuperType(List(left.out, right.out)) - - override def validate(): Either[String, Unit] = { - for { - _ <- left.validate() - _ <- right.validate() - _ <- SQLValidator.validateTypesMatching(left.out, right.out) - } yield () - } - - override def nullable: Boolean = left.nullable || right.nullable - - override def toPainless(base: String, idx: Int): String = { - if (nullable) { - val l = left match { - case t: SQLTransformFunction[_, _] => - SQLTypeUtils.coerce(t.toPainless("", idx + 1), left.out, out, nullable = false) - case _ => SQLTypeUtils.coerce(left.painless, left.out, out, nullable = false) - } - val r = right match { - case t: SQLTransformFunction[_, _] => - SQLTypeUtils.coerce(t.toPainless("", idx + 1), right.out, out, nullable = false) - case _ => SQLTypeUtils.coerce(right.painless, right.out, out, nullable = false) - } - var expr = "" - if (left.nullable) - expr += s"def lv$idx = ($l); " - if (right.nullable) - expr += s"def rv$idx = ($r); " - if (left.nullable && right.nullable) - expr += s"(lv$idx == null || rv$idx == null) ? null : (lv$idx ${operator.painless} rv$idx)" - else if (left.nullable) - expr += s"(lv$idx == null) ? null : (lv$idx ${operator.painless} $r)" - else - expr += s"(rv$idx == null) ? null : ($l ${operator.painless} rv$idx)" - if (group) - expr = s"($expr)" - return s"$base$expr" - } - s"$base$painless" - } - - override def painless: String = { - val l = SQLTypeUtils.coerce(left, out) - val r = SQLTypeUtils.coerce(right, out) - val expr = s"$l ${operator.painless} $r" - if (group) - s"($expr)" - else - expr - } - -} - -sealed trait MathematicalFunction - extends SQLTransformFunction[SQLNumeric, SQLNumeric] - with SQLFunctionWithIdentifier { - override def inputType: SQLNumeric = SQLTypes.Numeric - - override def outputType: SQLNumeric = SQLTypes.Double - - def operator: UnaryArithmeticOperator - - override def fun: Option[PainlessScript] = Some(operator) - - override def identifier: SQLIdentifier = SQLIdentifier("", functions = this :: Nil) - -} - -case class SQLMathematicalFunction( - operator: UnaryArithmeticOperator, - arg: PainlessScript -) extends MathematicalFunction { - override def args: List[PainlessScript] = List(arg) -} - -case class SQLPow(arg: PainlessScript, exponent: Int) extends MathematicalFunction { - override def operator: UnaryArithmeticOperator = Pow - override def args: List[PainlessScript] = List(arg, SQLIntValue(exponent)) - override def nullable: Boolean = arg.nullable -} - -case class SQLRound(arg: PainlessScript, scale: Option[Int]) extends MathematicalFunction { - override def operator: UnaryArithmeticOperator = Round - - override def args: List[PainlessScript] = - List(arg) ++ scale.map(SQLIntValue(_)).toList - - override def toPainlessCall(callArgs: List[String]): String = - s"(def p = ${SQLPow(SQLIntValue(10), scale.getOrElse(0)).painless}; ${operator.painless}((${callArgs.head} * p) / p))" -} - -case class SQLSign(arg: PainlessScript) extends MathematicalFunction { - override def operator: UnaryArithmeticOperator = Sign - - override def args: List[PainlessScript] = List(arg) - - override def outputType: SQLNumeric = SQLTypes.Int - - override def painless: String = { - val ret = "arg0 > 0 ? 1 : (arg0 < 0 ? -1 : 0)" - if (arg.nullable) - s"(def arg0 = ${arg.painless}; arg0 != null ? ($ret) : null)" - else - s"(def arg0 = ${arg.painless}; $ret)" - } -} - -case class SQLAtan2(y: PainlessScript, x: PainlessScript) extends MathematicalFunction { - override def operator: UnaryArithmeticOperator = Atan2 - override def args: List[PainlessScript] = List(y, x) - override def nullable: Boolean = y.nullable || x.nullable -} - -sealed trait StringFunction[Out <: SQLType] - extends SQLTransformFunction[SQLVarchar, Out] - with SQLFunctionWithIdentifier { - override def inputType: SQLVarchar = SQLTypes.Varchar - - override def outputType: Out - - def operator: SQLStringOperator - - override def fun: Option[PainlessScript] = Some(operator) - - override def identifier: SQLIdentifier = SQLIdentifier("", functions = this :: Nil) - - override def toSQL(base: String): String = s"$sql($base)" - - override def sql: String = - if (args.isEmpty) - s"${fun.map(_.sql).getOrElse("")}" - else - super.sql -} - -case class SQLStringFunction(operator: SQLStringOperator) extends StringFunction[SQLVarchar] { - override def outputType: SQLVarchar = SQLTypes.Varchar - override def args: List[PainlessScript] = List.empty - -} - -case class SQLSubstring(str: PainlessScript, start: Int, length: Option[Int]) - extends StringFunction[SQLVarchar] { - override def outputType: SQLVarchar = SQLTypes.Varchar - override def operator: SQLStringOperator = Substring - - override def args: List[PainlessScript] = - List(str, SQLIntValue(start)) ++ length.map(l => SQLIntValue(l)).toList - - override def nullable: Boolean = str.nullable - - override def toPainlessCall(callArgs: List[String]): String = { - callArgs match { - // SUBSTRING(expr, start, length) - case List(arg0, arg1, arg2) => - s"(($arg1 - 1) < 0 || ($arg1 - 1 + $arg2) > $arg0.length()) ? null : $arg0.substring(($arg1 - 1), ($arg1 - 1 + $arg2))" - - // SUBSTRING(expr, start) - case List(arg0, arg1) => - s"(($arg1 - 1) < 0 || ($arg1 - 1) >= $arg0.length()) ? null : $arg0.substring(($arg1 - 1))" - - case _ => throw new IllegalArgumentException("SUBSTRING requires 2 or 3 arguments") - } - } - - override def validate(): Either[String, Unit] = - if (start < 1) - Left("SUBSTRING start position must be greater than or equal to 1 (SQL is 1-based)") - else if (length.exists(_ < 0)) - Left("SUBSTRING length must be non-negative") - else Right(()) - - override def toSQL(base: String): String = sql - -} - -case class SQLConcat(values: List[PainlessScript]) extends StringFunction[SQLVarchar] { - override def outputType: SQLVarchar = SQLTypes.Varchar - override def operator: SQLStringOperator = Concat - - override def args: List[PainlessScript] = values - - override def nullable: Boolean = values.exists(_.nullable) - - override def toPainlessCall(callArgs: List[String]): String = { - if (callArgs.isEmpty) - throw new IllegalArgumentException("CONCAT requires at least one argument") - else - callArgs.zipWithIndex - .map { case (arg, idx) => - SQLTypeUtils.coerce(arg, values(idx).out, SQLTypes.Varchar, nullable = false) - } - .mkString(operator.painless) - } - - override def validate(): Either[String, Unit] = - if (values.isEmpty) Left("CONCAT requires at least one argument") - else Right(()) - - override def toSQL(base: String): String = sql -} - -case object SQLLength extends StringFunction[SQLBigInt] { - override def outputType: SQLBigInt = SQLTypes.BigInt - override def operator: SQLStringOperator = Length - override def args: List[PainlessScript] = List.empty -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala index b25fcbb8..272eb37b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala @@ -1,5 +1,7 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.parser.SQLParser + import scala.util.matching.Regex /** Created by smanciot on 27/06/2018. @@ -7,7 +9,7 @@ import scala.util.matching.Regex object SQLImplicits { import scala.language.implicitConversions - implicit def queryToSQLCriteria(query: String): Option[SQLCriteria] = { + implicit def queryToSQLCriteria(query: String): Option[Criteria] = { val maybeQuery: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = query maybeQuery match { case Some(Left(l)) => l.where.flatMap(_.criteria) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLLimit.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLLimit.scala deleted file mode 100644 index 53939a85..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLLimit.scala +++ /dev/null @@ -1,5 +0,0 @@ -package app.softnetwork.elastic.sql - -case object Limit extends SQLExpr("limit") with SQLRegex - -case class SQLLimit(limit: Int) extends SQLExpr(s" limit $limit") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala index ed13841d..2d69b816 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql -case class SQLMultiSearchRequest(requests: Seq[SQLSearchRequest]) extends SQLToken { +case class SQLMultiSearchRequest(requests: Seq[SQLSearchRequest]) extends Token { override def sql: String = s"${requests.map(_.sql).mkString(" union ")}" def update(): SQLMultiSearchRequest = this.copy(requests = requests.map(_.update())) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOperator.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOperator.scala deleted file mode 100644 index 9dbe21c2..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOperator.scala +++ /dev/null @@ -1,148 +0,0 @@ -package app.softnetwork.elastic.sql - -trait SQLOperator extends SQLToken with PainlessScript with SQLRegex { - override def painless: String = this match { - case And => "&&" - case Or => "||" - case Not => "!" - case In => ".contains" - case Like | Match => ".matches" - case Eq => "==" - case Ne => "!=" - case Plus => ".plus" - case Minus => ".minus" - case IsNull => " == null" - case IsNotNull => " != null" - case _ => sql - } -} - -sealed trait BinaryOperator extends SQLOperator - -sealed trait ArithmeticOperator extends SQLOperator { - override def toString: String = s" $sql " -} - -sealed trait BinaryArithmeticOperator extends ArithmeticOperator with BinaryOperator - -sealed trait IntervalOperator extends BinaryArithmeticOperator with MathScript { - override def script: String = sql -} -case object Plus extends SQLExpr("+") with IntervalOperator { - override def painless: String = ".plus" -} -case object Minus extends SQLExpr("-") with IntervalOperator { - override def painless: String = ".minus" -} - -case object Add extends SQLExpr("+") with IntervalOperator -case object Subtract extends SQLExpr("-") with IntervalOperator - -case object Multiply extends SQLExpr("*") with BinaryArithmeticOperator -case object Divide extends SQLExpr("/") with BinaryArithmeticOperator -case object Modulo extends SQLExpr("%") with BinaryArithmeticOperator - -sealed trait UnaryArithmeticOperator extends ArithmeticOperator { - override def painless: String = s"Math.${sql.toLowerCase()}" -} - -case object Abs extends SQLExpr("abs") with UnaryArithmeticOperator -case object Ceil extends SQLExpr("ceil") with UnaryArithmeticOperator -case object Floor extends SQLExpr("floor") with UnaryArithmeticOperator -case object Round extends SQLExpr("round") with UnaryArithmeticOperator -case object Exp extends SQLExpr("exp") with UnaryArithmeticOperator -case object Log extends SQLExpr("log") with UnaryArithmeticOperator -case object Log10 extends SQLExpr("log10") with UnaryArithmeticOperator -case object Pow extends SQLExpr("pow") with UnaryArithmeticOperator -case object Sqrt extends SQLExpr("sqrt") with UnaryArithmeticOperator -case object Sign extends SQLExpr("sign") with UnaryArithmeticOperator -case object Pi extends SQLExpr("pi") with UnaryArithmeticOperator { - override def painless: String = "Math.PI" -} - -sealed trait TrigonometricOperator extends UnaryArithmeticOperator - -case object Sin extends SQLExpr("sin") with TrigonometricOperator -case object Asin extends SQLExpr("asin") with TrigonometricOperator -case object Cos extends SQLExpr("cos") with TrigonometricOperator -case object Acos extends SQLExpr("acos") with TrigonometricOperator -case object Tan extends SQLExpr("tan") with TrigonometricOperator -case object Atan extends SQLExpr("atan") with TrigonometricOperator -case object Atan2 extends SQLExpr("atan2") with TrigonometricOperator - -sealed trait SQLExpressionOperator extends SQLOperator - -sealed trait SQLComparisonOperator extends SQLExpressionOperator with PainlessScript { - def not: SQLComparisonOperator = this match { - case Eq => Ne - case Ne | Diff => Eq - case Ge => Lt - case Gt => Le - case Le => Gt - case Lt => Ge - } -} - -case object Eq extends SQLExpr("=") with SQLComparisonOperator -case object Ne extends SQLExpr("<>") with SQLComparisonOperator -case object Diff extends SQLExpr("!=") with SQLComparisonOperator -case object Ge extends SQLExpr(">=") with SQLComparisonOperator -case object Gt extends SQLExpr(">") with SQLComparisonOperator -case object Le extends SQLExpr("<=") with SQLComparisonOperator -case object Lt extends SQLExpr("<") with SQLComparisonOperator -case object In extends SQLExpr("in") with SQLComparisonOperator -case object Like extends SQLExpr("like") with SQLComparisonOperator -case object Between extends SQLExpr("between") with SQLComparisonOperator -case object IsNull extends SQLExpr("is null") with SQLComparisonOperator -case object IsNotNull extends SQLExpr("is not null") with SQLComparisonOperator - -case object Match extends SQLExpr("match") with SQLComparisonOperator -case object Against extends SQLExpr("against") with SQLRegex - -sealed trait SQLStringOperator extends SQLOperator { - override def painless: String = s".${sql.toLowerCase()}()" -} -case object Concat extends SQLExpr("concat") with SQLStringOperator { - override def painless: String = " + " -} -case object Lower extends SQLExpr("lower") with SQLStringOperator -case object Upper extends SQLExpr("upper") with SQLStringOperator -case object Trim extends SQLExpr("trim") with SQLStringOperator -//case object LTrim extends SQLExpr("ltrim") with SQLStringOperator -//case object RTrim extends SQLExpr("rtrim") with SQLStringOperator -case object Substring extends SQLExpr("substring") with SQLStringOperator { - override def painless: String = ".substring" -} -case object To extends SQLExpr("to") with SQLRegex -case object Length extends SQLExpr("length") with SQLStringOperator - -sealed trait SQLLogicalOperator extends SQLExpressionOperator - -case object Not extends SQLExpr("not") with SQLLogicalOperator - -sealed trait SQLPredicateOperator extends SQLLogicalOperator - -case object And extends SQLExpr("and") with SQLPredicateOperator -case object Or extends SQLExpr("or") with SQLPredicateOperator - -sealed trait SQLConditionalOperator extends SQLExpressionOperator -case object Coalesce extends SQLExpr("coalesce") with SQLConditionalOperator -case object IsNullFunction extends SQLExpr("isnull") with SQLConditionalOperator -case object IsNotNullFunction extends SQLExpr("isnotnull") with SQLConditionalOperator -case object NullIf extends SQLExpr("nullif") with SQLConditionalOperator -case object Exists extends SQLExpr("exists") with SQLConditionalOperator - -case object Cast extends SQLExpr("cast") with SQLConditionalOperator -case object Case extends SQLExpr("case") with SQLConditionalOperator - -case object When extends SQLExpr("when") with SQLRegex -case object Then extends SQLExpr("then") with SQLRegex -case object Else extends SQLExpr("else") with SQLRegex -case object End extends SQLExpr("end") with SQLRegex - -case object Union extends SQLExpr("union") with SQLOperator with SQLRegex - -sealed trait ElasticOperator extends SQLOperator with SQLRegex -case object Nested extends SQLExpr("nested") with ElasticOperator -case object Child extends SQLExpr("child") with ElasticOperator -case object Parent extends SQLExpr("parent") with ElasticOperator diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala deleted file mode 100644 index 4a04f005..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLOrderBy.scala +++ /dev/null @@ -1,23 +0,0 @@ -package app.softnetwork.elastic.sql - -case object OrderBy extends SQLExpr("order by") with SQLRegex - -sealed trait SortOrder extends SQLRegex - -case object Desc extends SQLExpr("desc") with SortOrder - -case object Asc extends SQLExpr("asc") with SortOrder - -case class SQLFieldSort( - field: String, - order: Option[SortOrder], - functions: List[SQLFunction] = List.empty -) extends SQLFunctionChain { - lazy val direction: SortOrder = order.getOrElse(Asc) - lazy val name: String = toSQL(field) - override def sql: String = s"$name $direction" -} - -case class SQLOrderBy(sorts: Seq[SQLFieldSort]) extends SQLToken { - override def sql: String = s" $OrderBy ${sorts.mkString(", ")}" -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala index 87ec07da..f3e3b75a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala @@ -1,22 +1,22 @@ package app.softnetwork.elastic.sql case class SQLSearchRequest( - select: SQLSelect = SQLSelect(), - from: SQLFrom, - where: Option[SQLWhere], - groupBy: Option[SQLGroupBy] = None, - having: Option[SQLHaving] = None, - orderBy: Option[SQLOrderBy] = None, - limit: Option[SQLLimit] = None, + select: Select = Select(), + from: From, + where: Option[Where], + groupBy: Option[GroupBy] = None, + having: Option[Having] = None, + orderBy: Option[OrderBy] = None, + limit: Option[Limit] = None, score: Option[Double] = None -) extends SQLToken { +) extends Token { override def sql: String = s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}" lazy val fieldAliases: Map[String, String] = select.fieldAliases lazy val tableAliases: Map[String, String] = from.tableAliases - lazy val unnests: Seq[(String, String, Option[SQLLimit])] = from.unnests - lazy val bucketNames: Map[String, SQLBucket] = groupBy.map(_.bucketNames).getOrElse(Map.empty) + lazy val unnests: Seq[(String, String, Option[Limit])] = from.unnests + lazy val bucketNames: Map[String, Bucket] = groupBy.map(_.bucketNames).getOrElse(Map.empty) lazy val sorts: Map[String, SortOrder] = orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap @@ -46,11 +46,11 @@ case class SQLSearchRequest( lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) - lazy val sources: Seq[String] = from.tables.collect { case SQLTable(source: SQLIdentifier, _) => + lazy val sources: Seq[String] = from.tables.collect { case Table(source: GenericIdentifier, _) => source.sql } - lazy val buckets: Seq[SQLBucket] = groupBy.map(_.buckets).getOrElse(Seq.empty) + lazy val buckets: Seq[Bucket] = groupBy.map(_.buckets).getOrElse(Seq.empty) override def validate(): Either[String, Unit] = { for { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/Select.scala similarity index 66% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/Select.scala index fcbb3021..b7564f80 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/Select.scala @@ -1,10 +1,15 @@ package app.softnetwork.elastic.sql -case object Select extends SQLExpr("select") with SQLRegex +import app.softnetwork.elastic.sql.function.{Function, FunctionChain} -sealed trait Field extends Updateable with SQLFunctionChain with PainlessScript { - def identifier: Identifier - def fieldAlias: Option[SQLAlias] +case object Select extends Expr("select") with TokenRegex + +case class Field( + identifier: GenericIdentifier, + fieldAlias: Option[Alias] = None +) extends Updateable + with FunctionChain + with PainlessScript { def isScriptField: Boolean = functions.nonEmpty && !aggregation && identifier.bucket.isEmpty override def sql: String = s"$identifier${asString(fieldAlias)}" lazy val sourceField: String = { @@ -27,9 +32,10 @@ sealed trait Field extends Updateable with SQLFunctionChain with PainlessScript } } - override def functions: List[SQLFunction] = identifier.functions + override def functions: List[Function] = identifier.functions - def update(request: SQLSearchRequest): Field + def update(request: SQLSearchRequest): Field = + this.copy(identifier = identifier.update(request)) def painless: String = identifier.painless @@ -38,32 +44,24 @@ sealed trait Field extends Updateable with SQLFunctionChain with PainlessScript override def validate(): Either[String, Unit] = identifier.validate() } -case class SQLField( - identifier: SQLIdentifier, - fieldAlias: Option[SQLAlias] = None -) extends Field { - def update(request: SQLSearchRequest): SQLField = - this.copy(identifier = identifier.update(request)) -} - -case object Except extends SQLExpr("except") with SQLRegex +case object Except extends Expr("except") with TokenRegex -case class SQLExcept(fields: Seq[Field]) extends Updateable { +case class Except(fields: Seq[Field]) extends Updateable { override def sql: String = s" $Except(${fields.mkString(",")})" - def update(request: SQLSearchRequest): SQLExcept = + def update(request: SQLSearchRequest): Except = this.copy(fields = fields.map(_.update(request))) } -case class SQLSelect( - fields: Seq[Field] = Seq(SQLField(identifier = SQLIdentifier("*"))), - except: Option[SQLExcept] = None +case class Select( + fields: Seq[Field] = Seq(Field(identifier = GenericIdentifier("*"))), + except: Option[Except] = None ) extends Updateable { override def sql: String = s"$Select ${fields.mkString(", ")}${except.getOrElse("")}" lazy val fieldAliases: Map[String, String] = fields.flatMap { field => field.fieldAlias.map(a => field.identifier.identifierName -> a.alias) }.toMap - def update(request: SQLSearchRequest): SQLSelect = + def update(request: SQLSearchRequest): Select = this.copy(fields = fields.map(_.update(request)), except = except.map(_.update(request))) override def validate(): Either[String, Unit] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLValidator.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/Validator.scala similarity index 66% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLValidator.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/Validator.scala index c041ce3d..5064c068 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLValidator.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/Validator.scala @@ -1,8 +1,11 @@ package app.softnetwork.elastic.sql -object SQLValidator { +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} +import app.softnetwork.elastic.sql.function.{Function, FunctionN} - def validateChain(functions: List[SQLFunction]): Either[String, Unit] = { +object Validator { + + def validateChain(functions: List[Function]): Either[String, Unit] = { // validate function chain type compatibility functions match { case Nil => return Right(()) @@ -12,7 +15,7 @@ object SQLValidator { case Some(left) => return left case None => } - val funcs = functions.collect { case f: SQLFunctionN[_, _] => f } + val funcs = functions.collect { case f: FunctionN[_, _] => f } funcs.sliding(2).foreach { case Seq(f1, f2) => validateTypesMatching(f2.outputType, f1.inputType) @@ -30,8 +33,8 @@ object SQLValidator { } } -trait SQLValidation { +trait Validation { def validate(): Either[String, Unit] = Right(()) } -case class SQLValidationError(message: String) extends Exception(message) +case class ValidationError(message: String) extends Exception(message) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/Where.scala similarity index 66% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/Where.scala index 44420425..627ebd6c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/Where.scala @@ -1,17 +1,27 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.function._ +import app.softnetwork.elastic.sql.function.cond.{ + ConditionalFunction, + IsNotNullFunction, + IsNullFunction +} +import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.operator._ + import scala.annotation.tailrec -case object Where extends SQLExpr("where") with SQLRegex +case object Where extends Expr("where") with TokenRegex -sealed trait SQLCriteria extends Updateable with PainlessScript { - def operator: SQLOperator +sealed trait Criteria extends Updateable with PainlessScript { + def operator: Operator def nested: Boolean = false - def limit: Option[SQLLimit] = None + def limit: Option[Limit] = None - def update(request: SQLSearchRequest): SQLCriteria + def update(request: SQLSearchRequest): Criteria def group: Boolean @@ -35,7 +45,7 @@ sealed trait SQLCriteria extends Updateable with PainlessScript { override def out: SQLType = SQLTypes.Boolean override def painless: String = this match { - case SQLPredicate(left, op, right, maybeNot, group) => + case Predicate(left, op, right, maybeNot, group) => val leftStr = left.painless val rightStr = right.painless val opStr = op match { @@ -48,24 +58,24 @@ sealed trait SQLCriteria extends Updateable with PainlessScript { else s"$leftStr $opStr $rightStr" case relation: ElasticRelation => asGroup(relation.criteria.painless) - case m: SQLMatch => asGroup(m.criteria.painless) + case m: MatchCriteria => asGroup(m.criteria.painless) case expr: Expression => asGroup(expr.painless) case _ => throw new IllegalArgumentException(s"Unsupported criteria: $this") } } -case class SQLPredicate( - leftCriteria: SQLCriteria, - operator: SQLPredicateOperator, - rightCriteria: SQLCriteria, +case class Predicate( + leftCriteria: Criteria, + operator: PredicateOperator, + rightCriteria: Criteria, not: Option[Not.type] = None, group: Boolean = false -) extends SQLCriteria { +) extends Criteria { override def sql = s"${if (group) s"($leftCriteria" else leftCriteria} $operator${not .map(_ => " not") .getOrElse("")} ${if (group) s"$rightCriteria)" else rightCriteria}" - override def update(request: SQLSearchRequest): SQLCriteria = { + override def update(request: SQLSearchRequest): Criteria = { val updatedPredicate = this.copy( leftCriteria = leftCriteria.update(request), rightCriteria = rightCriteria.update(request) @@ -77,10 +87,10 @@ case class SQLPredicate( updatedPredicate } - override lazy val limit: Option[SQLLimit] = leftCriteria.limit.orElse(rightCriteria.limit) + override lazy val limit: Option[Limit] = leftCriteria.limit.orElse(rightCriteria.limit) - private[this] def unnest(criteria: SQLCriteria): SQLCriteria = criteria match { - case p: SQLPredicate => + private[this] def unnest(criteria: Criteria): Criteria = criteria match { + case p: Predicate => p.copy( leftCriteria = unnest(p.leftCriteria), rightCriteria = unnest(p.rightCriteria) @@ -172,39 +182,39 @@ case class ElasticBoolQuery( } -sealed trait Expression extends SQLFunctionChain with ElasticFilter with SQLCriteria { // to fix output type as Boolean - def identifier: SQLIdentifier +sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // to fix output type as Boolean + def identifier: GenericIdentifier override def nested: Boolean = identifier.nested override def group: Boolean = false - override lazy val limit: Option[SQLLimit] = identifier.limit - override val functions: List[SQLFunction] = identifier.functions - def maybeValue: Option[SQLToken] + override lazy val limit: Option[Limit] = identifier.limit + override val functions: List[Function] = identifier.functions + def maybeValue: Option[Token] def maybeNot: Option[Not.type] def notAsString: String = maybeNot.map(v => s"$v ").getOrElse("") def valueAsString: String = maybeValue.map(v => s" $v").getOrElse("") override def sql = s"$identifier $notAsString$operator$valueAsString" override lazy val aggregation: Boolean = maybeValue match { - case Some(v: SQLFunctionChain) => identifier.aggregation || v.aggregation - case _ => identifier.aggregation + case Some(v: FunctionChain) => identifier.aggregation || v.aggregation + case _ => identifier.aggregation } def painlessNot: String = operator match { - case _: SQLComparisonOperator => "" - case _ => maybeNot.map(_.painless).getOrElse("") + case _: ComparisonOperator => "" + case _ => maybeNot.map(_.painless).getOrElse("") } def painlessOp: String = operator match { - case o: SQLComparisonOperator if maybeNot.isDefined => o.not.painless - case _ => operator.painless + case o: ComparisonOperator if maybeNot.isDefined => o.not.painless + case _ => operator.painless } def painlessValue: String = maybeValue .map { - case v: SQLValue[_] => v.painless - case v: SQLValues[_, _] => v.painless - case v: SQLIdentifier => v.painless - case v => v.sql + case v: Value[_] => v.painless + case v: Values[_, _] => v.painless + case v: GenericIdentifier => v.painless + case v => v.sql } .getOrElse("") /*{ operator match { @@ -217,9 +227,9 @@ sealed trait Expression extends SQLFunctionChain with ElasticFilter with SQLCrit val targetedType = maybeValue match { case Some(v) => v match { - case value: SQLValue[_] => value.out - case values: SQLValues[_, _] => values.out - case other => other.out + case value: Value[_] => value.out + case values: Values[_, _] => values.out + case other => other.out } case None => identifier.out } @@ -228,8 +238,8 @@ sealed trait Expression extends SQLFunctionChain with ElasticFilter with SQLCrit protected lazy val check: String = operator match { - case _: SQLComparisonOperator => s" $painlessOp $painlessValue" - case _ => s"$painlessOp($painlessValue)" + case _: ComparisonOperator => s" $painlessOp $painlessValue" + case _ => s"$painlessOp($painlessValue)" } override def painless: String = { @@ -247,7 +257,7 @@ sealed trait Expression extends SQLFunctionChain with ElasticFilter with SQLCrit v.validate() match { case Left(err) => Left(s"$err in expression: $this") case Right(_) => - SQLValidator.validateTypesMatching(identifier.out, v.out) match { + Validator.validateTypesMatching(identifier.out, v.out) match { case Left(_) => Left( s"Type mismatch: '${out.typeId}' is not compatible with '${v.out.typeId}' in expression: $this" @@ -261,18 +271,18 @@ sealed trait Expression extends SQLFunctionChain with ElasticFilter with SQLCrit } } -case class SQLExpression( - identifier: SQLIdentifier, - operator: SQLExpressionOperator, - value: SQLToken, +case class GenericExpression( + identifier: GenericIdentifier, + operator: ExpressionOperator, + value: Token, maybeNot: Option[Not.type] = None ) extends Expression { - override def maybeValue: Option[SQLToken] = Option(value) + override def maybeValue: Option[Token] = Option(value) - override def update(request: SQLSearchRequest): SQLCriteria = { + override def update(request: SQLSearchRequest): Criteria = { val updated = value match { - case id: SQLIdentifier => + case id: GenericIdentifier => this.copy(identifier = identifier.update(request), value = id.update(request)) case _ => this.copy(identifier = identifier.update(request)) } @@ -285,14 +295,14 @@ case class SQLExpression( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class SQLIsNull(identifier: SQLIdentifier) extends Expression { - override val operator: SQLOperator = IsNull +case class IsNullExpr(identifier: GenericIdentifier) extends Expression { + override val operator: Operator = IsNull - override def maybeValue: Option[SQLToken] = None + override def maybeValue: Option[Token] = None override def maybeNot: Option[Not.type] = None - override def update(request: SQLSearchRequest): SQLCriteria = { + override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -303,14 +313,14 @@ case class SQLIsNull(identifier: SQLIdentifier) extends Expression { override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class SQLIsNotNull(identifier: SQLIdentifier) extends Expression { - override val operator: SQLOperator = IsNotNull +case class IsNotNullExpr(identifier: GenericIdentifier) extends Expression { + override val operator: Operator = IsNotNull - override def maybeValue: Option[SQLToken] = None + override def maybeValue: Option[Token] = None override def maybeNot: Option[Not.type] = None - override def update(request: SQLSearchRequest): SQLCriteria = { + override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -321,28 +331,28 @@ case class SQLIsNotNull(identifier: SQLIdentifier) extends Expression { override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -sealed trait SQLCriteriaWithConditionalFunction[In <: SQLType] extends Expression { - def conditionalFunction: SQLConditionalFunction[In] - override def maybeValue: Option[SQLToken] = None +sealed trait CriteriaWithConditionalFunction[In <: SQLType] extends Expression { + def conditionalFunction: ConditionalFunction[In] + override def maybeValue: Option[Token] = None override def maybeNot: Option[Not.type] = None override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - override val functions: List[SQLFunction] = List(conditionalFunction) + override val functions: List[Function] = List(conditionalFunction) override def sql: String = conditionalFunction.sql } -object SQLConditionalFunctionAsCriteria { - def unapply(f: SQLConditionalFunction[_]): Option[SQLCriteria] = f match { - case SQLIsNullFunction(id) => Some(SQLIsNullCriteria(id)) - case SQLIsNotNullFunction(id) => Some(SQLIsNotNullCriteria(id)) - case _ => None +object ConditionalFunctionAsCriteria { + def unapply(f: ConditionalFunction[_]): Option[Criteria] = f match { + case IsNullFunction(id) => Some(IsNullCriteria(id)) + case IsNotNullFunction(id) => Some(IsNotNullCriteria(id)) + case _ => None } } -case class SQLIsNullCriteria(identifier: SQLIdentifier) - extends SQLCriteriaWithConditionalFunction[SQLAny] { - override val conditionalFunction: SQLConditionalFunction[SQLAny] = SQLIsNullFunction(identifier) - override val operator: SQLOperator = IsNull - override def update(request: SQLSearchRequest): SQLCriteria = { +case class IsNullCriteria(identifier: GenericIdentifier) + extends CriteriaWithConditionalFunction[SQLAny] { + override val conditionalFunction: ConditionalFunction[SQLAny] = IsNullFunction(identifier) + override val operator: Operator = IsNull + override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -358,13 +368,13 @@ case class SQLIsNullCriteria(identifier: SQLIdentifier) } -case class SQLIsNotNullCriteria(identifier: SQLIdentifier) - extends SQLCriteriaWithConditionalFunction[SQLAny] { - override val conditionalFunction: SQLConditionalFunction[SQLAny] = SQLIsNotNullFunction( +case class IsNotNullCriteria(identifier: GenericIdentifier) + extends CriteriaWithConditionalFunction[SQLAny] { + override val conditionalFunction: ConditionalFunction[SQLAny] = IsNotNullFunction( identifier ) - override val operator: SQLOperator = IsNotNull - override def update(request: SQLSearchRequest): SQLCriteria = { + override val operator: Operator = IsNotNull + override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -381,19 +391,19 @@ case class SQLIsNotNullCriteria(identifier: SQLIdentifier) } -case class SQLIn[R, +T <: SQLValue[R]]( - identifier: SQLIdentifier, - values: SQLValues[R, T], +case class InExpr[R, +T <: Value[R]]( + identifier: GenericIdentifier, + values: Values[R, T], maybeNot: Option[Not.type] = None -) extends Expression { this: SQLIn[R, T] => +) extends Expression { this: InExpr[R, T] => private[this] lazy val id = functions.headOption match { case Some(f) => s"$f($identifier)" case _ => s"$identifier" } override def sql = s"$id $notAsString$operator $values" - override def operator: SQLOperator = In - override def update(request: SQLSearchRequest): SQLCriteria = { + override def operator: Operator = In + override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -401,16 +411,16 @@ case class SQLIn[R, +T <: SQLValue[R]]( updated } - override def maybeValue: Option[SQLToken] = Some(values) + override def maybeValue: Option[Token] = Some(values) override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this override def painless: String = s"$painlessNot${identifier.painless}$painlessOp($painlessValue)" } -case class SQLBetween[+T]( - identifier: SQLIdentifier, - fromTo: SQLFromTo[T], +case class BetweenExpr[+T]( + identifier: GenericIdentifier, + fromTo: FromTo[T], maybeNot: Option[Not.type] ) extends Expression { private[this] lazy val id = functions.headOption match { @@ -419,8 +429,8 @@ case class SQLBetween[+T]( } override def sql = s"$id $notAsString$operator $fromTo" - override def operator: SQLOperator = Between - override def update(request: SQLSearchRequest): SQLCriteria = { + override def operator: Operator = Between + override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -428,63 +438,63 @@ case class SQLBetween[+T]( updated } - override def maybeValue: Option[SQLToken] = Some(fromTo) + override def maybeValue: Option[Token] = Some(fromTo) override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } case class ElasticGeoDistance( - identifier: SQLIdentifier, - distance: SQLStringValue, - lat: SQLDoubleValue, - lon: SQLDoubleValue + identifier: GenericIdentifier, + distance: StringValue, + lat: DoubleValue, + lon: DoubleValue ) extends Expression { override def sql = s"$Distance($identifier,($lat,$lon)) $operator $distance" - override val functions: List[SQLFunction] = List(Distance) - override def operator: SQLOperator = Le + override val functions: List[Function] = List(Distance) + override def operator: Operator = Le override def update(request: SQLSearchRequest): ElasticGeoDistance = this.copy(identifier = identifier.update(request)) - override def maybeValue: Option[SQLToken] = Some(distance) + override def maybeValue: Option[Token] = Some(distance) override def maybeNot: Option[Not.type] = None override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class SQLMatch( - identifiers: Seq[SQLIdentifier], - value: SQLStringValue -) extends SQLCriteria { +case class MatchCriteria( + identifiers: Seq[GenericIdentifier], + value: StringValue +) extends Criteria { override def sql: String = s"$operator (${identifiers.mkString(",")}) $Against ($value)" - override def operator: SQLOperator = Match - override def update(request: SQLSearchRequest): SQLCriteria = + override def operator: Operator = Match + override def update(request: SQLSearchRequest): Criteria = this.copy(identifiers = identifiers.map(_.update(request))) override lazy val nested: Boolean = identifiers.forall(_.nested) @tailrec - private[this] def toCriteria(matches: List[ElasticMatch], curr: SQLCriteria): SQLCriteria = + private[this] def toCriteria(matches: List[ElasticMatch], curr: Criteria): Criteria = matches match { case Nil => curr - case single :: Nil => SQLPredicate(curr, Or, single) - case first :: rest => toCriteria(rest, SQLPredicate(curr, Or, first)) + case single :: Nil => Predicate(curr, Or, single) + case first :: rest => toCriteria(rest, Predicate(curr, Or, first)) } - lazy val criteria: SQLCriteria = + lazy val criteria: Criteria = (identifiers.map(id => ElasticMatch(id, value, None)) match { case Nil => throw new IllegalArgumentException("No identifiers for MATCH") case single :: Nil => single case first :: rest => toCriteria(rest, first) }) match { - case p: SQLPredicate => p.copy(group = true) - case other => other + case p: Predicate => p.copy(group = true) + case other => other } override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = criteria match { - case predicate: SQLPredicate => predicate.copy(group = true).asFilter(currentQuery) - case _ => criteria.asFilter(currentQuery) + case predicate: Predicate => predicate.copy(group = true).asFilter(currentQuery) + case _ => criteria.asFilter(currentQuery) } override def matchCriteria: Boolean = true @@ -493,17 +503,17 @@ case class SQLMatch( } case class ElasticMatch( - identifier: SQLIdentifier, - value: SQLStringValue, + identifier: GenericIdentifier, + value: StringValue, options: Option[String] ) extends Expression { override def sql: String = s"$operator($identifier,$value${options.map(o => s""","$o"""").getOrElse("")})" - override def operator: SQLOperator = Match - override def update(request: SQLSearchRequest): SQLCriteria = + override def operator: Operator = Match + override def update(request: SQLSearchRequest): Criteria = this.copy(identifier = identifier.update(request)) - override def maybeValue: Option[SQLToken] = Some(value) + override def maybeValue: Option[Token] = Some(value) override def maybeNot: Option[Not.type] = None @@ -514,13 +524,13 @@ case class ElasticMatch( override def painless: String = s"$painlessNot${identifier.painless}$painlessOp($painlessValue)" } -sealed abstract class ElasticRelation(val criteria: SQLCriteria, val operator: ElasticOperator) - extends SQLCriteria +sealed abstract class ElasticRelation(val criteria: Criteria, val operator: ElasticOperator) + extends Criteria with ElasticFilter { override def sql = s"$operator($criteria)" - private[this] def rtype(criteria: SQLCriteria): Option[String] = criteria match { - case SQLPredicate(left, _, right, _, _) => rtype(left).orElse(rtype(right)) + private[this] def rtype(criteria: Criteria): Option[String] = criteria match { + case Predicate(left, _, right, _, _) => rtype(left).orElse(rtype(right)) case c: Expression => c.identifier.nestedType.orElse(c.identifier.name.split('.').headOption) case relation: ElasticRelation => relation.relationType @@ -535,15 +545,15 @@ sealed abstract class ElasticRelation(val criteria: SQLCriteria, val operator: E } -case class ElasticNested(override val criteria: SQLCriteria, override val limit: Option[SQLLimit]) +case class ElasticNested(override val criteria: Criteria, override val limit: Option[Limit]) extends ElasticRelation(criteria, Nested) { override def update(request: SQLSearchRequest): ElasticNested = this.copy(criteria = criteria.update(request)) override def nested: Boolean = true - private[this] def name(criteria: SQLCriteria): Option[String] = criteria match { - case SQLPredicate(left, _, right, _, _) => name(left).orElse(name(right)) + private[this] def name(criteria: Criteria): Option[String] = criteria match { + case Predicate(left, _, right, _, _) => name(left).orElse(name(right)) case c: Expression => c.identifier.innerHitsName.orElse(c.identifier.name.split('.').headOption) case n: ElasticNested => name(n.criteria) @@ -553,24 +563,23 @@ case class ElasticNested(override val criteria: SQLCriteria, override val limit: lazy val innerHitsName: Option[String] = name(criteria) } -case class ElasticChild(override val criteria: SQLCriteria) - extends ElasticRelation(criteria, Child) { +case class ElasticChild(override val criteria: Criteria) extends ElasticRelation(criteria, Child) { override def update(request: SQLSearchRequest): ElasticChild = this.copy(criteria = criteria.update(request)) } -case class ElasticParent(override val criteria: SQLCriteria) +case class ElasticParent(override val criteria: Criteria) extends ElasticRelation(criteria, Parent) { override def update(request: SQLSearchRequest): ElasticParent = this.copy(criteria = criteria.update(request)) } -case class SQLWhere(criteria: Option[SQLCriteria]) extends Updateable { +case class Where(criteria: Option[Criteria]) extends Updateable { override def sql: String = criteria match { case Some(c) => s" $Where $c" case _ => "" } - def update(request: SQLSearchRequest): SQLWhere = + def update(request: SQLSearchRequest): Where = this.copy(criteria = criteria.map(_.update(request))) override def validate(): Either[String, Unit] = criteria match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala new file mode 100644 index 00000000..58dda60a --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -0,0 +1,19 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.Expr + +package object aggregate { + + sealed trait AggregateFunction extends Function + + case object Count extends Expr("count") with AggregateFunction + + case object Min extends Expr("min") with AggregateFunction + + case object Max extends Expr("max") with AggregateFunction + + case object Avg extends Expr("avg") with AggregateFunction + + case object Sum extends Expr("sum") with AggregateFunction + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala new file mode 100644 index 00000000..efacba5d --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -0,0 +1,255 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.{ + Expr, + Expression, + GenericIdentifier, + Identifier, + PainlessScript, + TokenRegex +} +import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLBool, SQLType, SQLTypeUtils, SQLTypes} + +package object cond { + + sealed trait ConditionalOp extends PainlessScript with TokenRegex { + override def painless: String = sql + } + + case object Coalesce extends Expr("coalesce") with ConditionalOp + case object IsNullFunction extends Expr("isnull") with ConditionalOp + case object IsNotNullFunction extends Expr("isnotnull") with ConditionalOp + case object NullIf extends Expr("nullif") with ConditionalOp + case object Exists extends Expr("exists") with ConditionalOp + + case object Case extends Expr("case") with ConditionalOp + + case object When extends Expr("when") with TokenRegex + case object Then extends Expr("then") with TokenRegex + case object Else extends Expr("else") with TokenRegex + case object End extends Expr("end") with TokenRegex + + sealed trait ConditionalFunction[In <: SQLType] + extends TransformFunction[In, SQLBool] + with FunctionWithIdentifier { + def conditionalOp: ConditionalOp + + override def fun: Option[PainlessScript] = Some(conditionalOp) + + override def outputType: SQLBool = SQLTypes.Boolean + + override def toPainless(base: String, idx: Int): String = s"($base$painless)" + } + + case class IsNullFunction(identifier: GenericIdentifier) extends ConditionalFunction[SQLAny] { + override def conditionalOp: ConditionalOp = IsNullFunction + + override def args: List[PainlessScript] = List(identifier) + + override def inputType: SQLAny = SQLTypes.Any + + override def toSQL(base: String): String = sql + + override def painless: String = s" == null" + override def toPainless(base: String, idx: Int): String = { + if (nullable) + s"(def e$idx = $base; e$idx$painless)" + else + s"$base$painless" + } + } + + case class IsNotNullFunction(identifier: GenericIdentifier) extends ConditionalFunction[SQLAny] { + override def conditionalOp: ConditionalOp = IsNotNullFunction + + override def args: List[PainlessScript] = List(identifier) + + override def inputType: SQLAny = SQLTypes.Any + + override def toSQL(base: String): String = sql + + override def painless: String = s" != null" + override def toPainless(base: String, idx: Int): String = { + if (nullable) + s"(def e$idx = $base; e$idx$painless)" + else + s"$base$painless" + } + } + + case class Coalesce(values: List[PainlessScript]) + extends TransformFunction[SQLAny, SQLType] + with FunctionWithIdentifier { + def operator: ConditionalOp = Coalesce + + override def fun: Option[ConditionalOp] = Some(operator) + + override def args: List[PainlessScript] = values + + override def outputType: SQLType = SQLTypeUtils.leastCommonSuperType(args.map(_.out)) + + override def identifier: GenericIdentifier = GenericIdentifier("") + + override def inputType: SQLAny = SQLTypes.Any + + override def sql: String = s"$Coalesce(${values.map(_.sql).mkString(", ")})" + + // Reprend l’idée de SQLValues mais pour n’importe quel token + override def out: SQLType = SQLTypeUtils.leastCommonSuperType(values.map(_.out).distinct) + + override def applyType(in: SQLType): SQLType = out + + override def validate(): Either[String, Unit] = { + if (values.isEmpty) Left("COALESCE requires at least one argument") + else Right(()) + } + + override def toPainless(base: String, idx: Int): String = s"$base$painless" + + override def painless: String = { + require(values.nonEmpty, "COALESCE requires at least one argument") + + val checks = values + .take(values.length - 1) + .zipWithIndex + .map { case (v, index) => + var check = s"def v$index = ${SQLTypeUtils.coerce(v, out)};" + check += s"if (v$index != null) return v$index;" + check + } + .mkString(" ") + // final fallback + s"{ $checks return ${SQLTypeUtils.coerce(values.last, out)}; }" + } + + override def nullable: Boolean = values.forall(_.nullable) + } + + case class NullIf(expr1: PainlessScript, expr2: PainlessScript) + extends ConditionalFunction[SQLAny] { + override def conditionalOp: ConditionalOp = NullIf + + override def args: List[PainlessScript] = List(expr1, expr2) + + override def identifier: GenericIdentifier = GenericIdentifier("") + + override def inputType: SQLAny = SQLTypes.Any + + override def out: SQLType = expr1.out + + override def applyType(in: SQLType): SQLType = out + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0, arg1) => s"${arg0.trim} == ${arg1.trim} ? null : $arg0" + case _ => throw new IllegalArgumentException("NULLIF requires exactly two arguments") + } + } + } + + case class Case( + expression: Option[PainlessScript], + conditions: List[(PainlessScript, PainlessScript)], + default: Option[PainlessScript] + ) extends TransformFunction[SQLAny, SQLAny] { + override def args: List[PainlessScript] = List.empty + + override def inputType: SQLAny = SQLTypes.Any + override def outputType: SQLAny = SQLTypes.Any + + override def sql: String = { + val exprPart = expression.map(e => s"$Case ${e.sql}").getOrElse(Case.sql) + val whenThen = conditions + .map { case (cond, res) => s"$When ${cond.sql} $Then ${res.sql}" } + .mkString(" ") + val elsePart = default.map(d => s" $Else ${d.sql}").getOrElse("") + s"$exprPart $whenThen$elsePart $End" + } + + override def out: SQLType = + SQLTypeUtils.leastCommonSuperType( + conditions.map(_._2.out) ++ default.map(_.out).toList + ) + + override def applyType(in: SQLType): SQLType = out + + override def validate(): Either[String, Unit] = { + if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") + else if ( + expression.isEmpty && conditions.exists { case (cond, _) => cond.out != SQLTypes.Boolean } + ) + Left("CASE WHEN conditions must be of type BOOLEAN") + else if ( + expression.isDefined && conditions.exists { case (cond, _) => + !SQLTypeUtils.matches(cond.out, expression.get.out) + } + ) + Left("CASE WHEN conditions must be of the same type as the expression") + else Right(()) + } + + override def painless: String = { + val base = + expression match { + case Some(expr) => + s"def expr = ${SQLTypeUtils.coerce(expr, expr.out)}; " + case _ => "" + } + val cases = conditions.zipWithIndex + .map { case ((cond, res), idx) => + val name = + cond match { + case e: Expression => + e.identifier.name + case i: Identifier => + i.name + case _ => "" + } + expression match { + case Some(expr) => + val c = SQLTypeUtils.coerce(cond, expr.out) + if (cond.sql == res.sql) { + s"def val$idx = $c; if (expr == val$idx) return val$idx;" + } else { + res match { + case i: Identifier if i.name == name && cond.isInstanceOf[Identifier] => + i.nullable = false + if (cond.asInstanceOf[Identifier].functions.isEmpty) + s"def val$idx = $c; if (expr == val$idx) return ${SQLTypeUtils.coerce(i.toPainless(s"val$idx"), i.out, out, nullable = false)};" + else { + cond.asInstanceOf[Identifier].nullable = false + s"def e$idx = ${i.checkNotNull}; def val$idx = e$idx != null ? ${SQLTypeUtils + .coerce(cond.asInstanceOf[Identifier].toPainless(s"e$idx"), cond.out, out, nullable = false)} : null; if (expr == val$idx) return ${SQLTypeUtils + .coerce(i.toPainless(s"e$idx"), i.out, out, nullable = false)};" + } + case _ => + s"if (expr == $c) return ${SQLTypeUtils.coerce(res, out)};" + } + } + case None => + val c = SQLTypeUtils.coerce(cond, SQLTypes.Boolean) + val r = + res match { + case i: Identifier if i.name == name && cond.isInstanceOf[Expression] => + i.nullable = false + SQLTypeUtils.coerce(i.toPainless("left"), i.out, out, nullable = false) + case _ => SQLTypeUtils.coerce(res, out) + } + s"if ($c) return $r;" + } + } + .mkString(" ") + val defaultCase = default + .map(d => s"def dval = ${SQLTypeUtils.coerce(d, out)}; return dval;") + .getOrElse("return null;") + s"{ $base$cases $defaultCase }" + } + + override def toPainless(base: String, idx: Int): String = s"$base$painless" + + override def nullable: Boolean = + conditions.exists { case (_, res) => res.nullable } || default.forall(_.nullable) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala new file mode 100644 index 00000000..c9185020 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -0,0 +1,29 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.{Alias, Expr, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} + +package object convert { + + case object Cast extends Expr("cast") with TokenRegex + + case class Cast(value: PainlessScript, targetType: SQLType, as: Boolean = true) + extends TransformFunction[SQLType, SQLType] { + override def inputType: SQLType = value.out + override def outputType: SQLType = targetType + + override def args: List[PainlessScript] = List.empty + + override def sql: String = + s"$Cast(${value.sql} ${if (as) s"$Alias " else ""}${targetType.typeId})" + + override def toSQL(base: String): String = sql + + override def painless: String = + SQLTypeUtils.coerce(value, targetType) + + override def toPainless(base: String, idx: Int): String = + SQLTypeUtils.coerce(base, value.out, targetType, value.nullable) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala new file mode 100644 index 00000000..308ac591 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.Expr +import app.softnetwork.elastic.sql.operator.Operator + +package object geo { + + case object Distance extends Expr("distance") with Function with Operator + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala new file mode 100644 index 00000000..1d156766 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -0,0 +1,97 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, IntValue, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLTypes} + +package object math { + + sealed trait MathOp extends PainlessScript with TokenRegex { + override def painless: String = s"Math.${sql.toLowerCase()}" + override def toString: String = s" $sql " + } + + case object Abs extends Expr("abs") with MathOp + case object Ceil extends Expr("ceil") with MathOp + case object Floor extends Expr("floor") with MathOp + case object Round extends Expr("round") with MathOp + case object Exp extends Expr("exp") with MathOp + case object Log extends Expr("log") with MathOp + case object Log10 extends Expr("log10") with MathOp + case object Pow extends Expr("pow") with MathOp + case object Sqrt extends Expr("sqrt") with MathOp + case object Sign extends Expr("sign") with MathOp + case object Pi extends Expr("pi") with MathOp { + override def painless: String = "Math.PI" + } + + sealed trait Trigonometric extends MathOp + + case object Sin extends Expr("sin") with Trigonometric + case object Asin extends Expr("asin") with Trigonometric + case object Cos extends Expr("cos") with Trigonometric + case object Acos extends Expr("acos") with Trigonometric + case object Tan extends Expr("tan") with Trigonometric + case object Atan extends Expr("atan") with Trigonometric + case object Atan2 extends Expr("atan2") with Trigonometric + + sealed trait MathematicalFunction + extends TransformFunction[SQLNumeric, SQLNumeric] + with FunctionWithIdentifier { + override def inputType: SQLNumeric = SQLTypes.Numeric + + override def outputType: SQLNumeric = SQLTypes.Double + + def mathOp: MathOp + + override def fun: Option[PainlessScript] = Some(mathOp) + + override def identifier: GenericIdentifier = GenericIdentifier("", functions = this :: Nil) + + } + + case class MathematicalFunctionWithOp( + mathOp: MathOp, + arg: PainlessScript + ) extends MathematicalFunction { + override def args: List[PainlessScript] = List(arg) + } + + case class Pow(arg: PainlessScript, exponent: Int) extends MathematicalFunction { + override def mathOp: MathOp = Pow + override def args: List[PainlessScript] = List(arg, IntValue(exponent)) + override def nullable: Boolean = arg.nullable + } + + case class Round(arg: PainlessScript, scale: Option[Int]) extends MathematicalFunction { + override def mathOp: MathOp = Round + + override def args: List[PainlessScript] = + List(arg) ++ scale.map(IntValue(_)).toList + + override def toPainlessCall(callArgs: List[String]): String = + s"(def p = ${Pow(IntValue(10), scale.getOrElse(0)).painless}; ${mathOp.painless}((${callArgs.head} * p) / p))" + } + + case class Sign(arg: PainlessScript) extends MathematicalFunction { + override def mathOp: MathOp = Sign + + override def args: List[PainlessScript] = List(arg) + + override def outputType: SQLNumeric = SQLTypes.Int + + override def painless: String = { + val ret = "arg0 > 0 ? 1 : (arg0 < 0 ? -1 : 0)" + if (arg.nullable) + s"(def arg0 = ${arg.painless}; arg0 != null ? ($ret) : null)" + else + s"(def arg0 = ${arg.painless}; $ret)" + } + } + + case class Atan2(y: PainlessScript, x: PainlessScript) extends MathematicalFunction { + override def mathOp: MathOp = Atan2 + override def args: List[PainlessScript] = List(y, x) + override def nullable: Boolean = y.nullable || x.nullable + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala new file mode 100644 index 00000000..b9b767b1 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -0,0 +1,185 @@ +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression + +package object function { + + trait Function extends TokenRegex { + def toSQL(base: String): String = if (base.nonEmpty) s"$sql($base)" else sql + def applyType(in: SQLType): SQLType = out + private[this] var _expr: Token = Null + def expr_=(e: Token): Unit = { + _expr = e + } + def expr: Token = _expr + override def nullable: Boolean = expr.nullable + } + + trait FunctionWithIdentifier extends Function { + def identifier: GenericIdentifier //= SQLIdentifier("", functions = this :: Nil) + } + + trait FunctionWithValue[+T] extends Function { + def value: T + } + + object FunctionUtils { + def aggregateAndTransformFunctions( + chain: FunctionChain + ): (List[Function], List[Function]) = { + chain.functions.partition { + case _: AggregateFunction => true + case _ => false + } + } + + def transformFunctions(chain: FunctionChain): List[Function] = { + aggregateAndTransformFunctions(chain)._2 + } + + } + + trait FunctionChain extends Function { + def functions: List[Function] + + override def validate(): Either[String, Unit] = { + if (aggregations.size > 1) { + Left("Only one aggregation function is allowed in a function chain") + } else if (aggregations.size == 1 && !functions.head.isInstanceOf[AggregateFunction]) { + Left("Aggregation function must be the first function in the chain") + } else { + Validator.validateChain(functions) + } + } + + override def toSQL(base: String): String = + functions.reverse.foldLeft(base)((expr, fun) => { + fun.toSQL(expr) + }) + + def toScript: Option[String] = { + val orderedFunctions = FunctionUtils.transformFunctions(this).reverse + orderedFunctions.foldLeft(Option("")) { + case (expr, f: MathScript) if expr.isDefined => Option(s"${expr.get}${f.script}") + case (_, _) => None // ignore non math scripts + } match { + case Some(s) if s.nonEmpty => + out match { + case SQLTypes.Date => Some(s"$s/d") + case _ => Some(s) + } + case _ => None + } + } + + override def system: Boolean = functions.lastOption.exists(_.system) + + def applyTo(expr: Token): Unit = { + this.expr = expr + functions.reverse.foldLeft(expr) { (currentExpr, fun) => + fun.expr = currentExpr + fun + } + } + + private[this] lazy val aggregations = functions.collect { case af: AggregateFunction => + af + } + + lazy val aggregateFunction: Option[AggregateFunction] = aggregations.headOption + + lazy val aggregation: Boolean = aggregateFunction.isDefined + + override def in: SQLType = functions.lastOption.map(_.in).getOrElse(super.in) + + override def out: SQLType = { + val baseType = functions.lastOption.map(_.in).getOrElse(super.baseType) + functions.reverse.foldLeft(baseType) { (currentType, fun) => + fun.applyType(currentType) + } + } + + def arithmetic: Boolean = functions.nonEmpty && functions.forall { + case _: ArithmeticExpression => true + case _ => false + } + } + + trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { + def fun: Option[PainlessScript] = None + + def args: List[PainlessScript] + def argsSeparator: String = ", " + + def inputType: In + def outputType: Out + + override def in: SQLType = inputType + override def out: SQLType = outputType + + override def applyType(in: SQLType): SQLType = outputType + + override def sql: String = + s"${fun.map(_.sql).getOrElse("")}(${args.map(_.sql).mkString(argsSeparator)})" + + override def toSQL(base: String): String = s"$base$sql" + + override def painless: String = { + val nullCheck = + args + .filter(_.nullable) + .zipWithIndex + .map { case (_, i) => s"arg$i == null" } + .mkString(" || ") + + val assignments = + args + .filter(_.nullable) + .zipWithIndex + .map { case (a, i) => s"def arg$i = ${a.painless};" } + .mkString(" ") + + val callArgs = args.zipWithIndex + .map { case (a, i) => + if (a.nullable) + s"arg$i" + else + a.painless + } + + if (args.exists(_.nullable)) + s"($assignments ($nullCheck) ? null : ${toPainlessCall(callArgs)})" + else + s"${toPainlessCall(callArgs)}" + } + + def toPainlessCall(callArgs: List[String]): String = + if (callArgs.nonEmpty) + s"${fun.map(_.painless).getOrElse("")}(${callArgs.mkString(argsSeparator)})" + else + fun.map(_.painless).getOrElse("") + } + + trait BinaryFunction[In1 <: SQLType, In2 <: SQLType, Out <: SQLType] extends FunctionN[In2, Out] { + self: Function => + + def left: PainlessScript + def right: PainlessScript + + override def args: List[PainlessScript] = List(left, right) + + override def nullable: Boolean = left.nullable || right.nullable + } + + trait TransformFunction[In <: SQLType, Out <: SQLType] extends FunctionN[In, Out] { + def toPainless(base: String, idx: Int): String = { + if (nullable && base.nonEmpty) + s"(def e$idx = $base; e$idx != null ? e$idx$painless : null)" + else + s"$base$painless" + } + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala new file mode 100644 index 00000000..6ebfa488 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -0,0 +1,119 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, IntValue, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLType, SQLTypeUtils, SQLTypes, SQLVarchar} + +package object string { + + sealed trait StringOp extends PainlessScript with TokenRegex { + override def painless: String = s".${sql.toLowerCase()}()" + } + + case object Concat extends Expr("concat") with StringOp { + override def painless: String = " + " + } + case object Lower extends Expr("lower") with StringOp + case object Upper extends Expr("upper") with StringOp + case object Trim extends Expr("trim") with StringOp + //case object LTrim extends SQLExpr("ltrim") with SQLStringOperator + //case object RTrim extends SQLExpr("rtrim") with SQLStringOperator + case object Substring extends Expr("substring") with StringOp { + override def painless: String = ".substring" + } + case object To extends Expr("to") with TokenRegex + case object Length extends Expr("length") with StringOp + + sealed trait StringFunction[Out <: SQLType] + extends TransformFunction[SQLVarchar, Out] + with FunctionWithIdentifier { + override def inputType: SQLVarchar = SQLTypes.Varchar + + override def outputType: Out + + def stringOp: StringOp + + override def fun: Option[PainlessScript] = Some(stringOp) + + override def identifier: GenericIdentifier = GenericIdentifier("", functions = this :: Nil) + + override def toSQL(base: String): String = s"$sql($base)" + + override def sql: String = + if (args.isEmpty) + s"${fun.map(_.sql).getOrElse("")}" + else + super.sql + } + + case class StringFunctionWithOp(stringOp: StringOp) extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def args: List[PainlessScript] = List.empty + } + + case class Substring(str: PainlessScript, start: Int, length: Option[Int]) + extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def stringOp: StringOp = Substring + + override def args: List[PainlessScript] = + List(str, IntValue(start)) ++ length.map(l => IntValue(l)).toList + + override def nullable: Boolean = str.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + // SUBSTRING(expr, start, length) + case List(arg0, arg1, arg2) => + s"(($arg1 - 1) < 0 || ($arg1 - 1 + $arg2) > $arg0.length()) ? null : $arg0.substring(($arg1 - 1), ($arg1 - 1 + $arg2))" + + // SUBSTRING(expr, start) + case List(arg0, arg1) => + s"(($arg1 - 1) < 0 || ($arg1 - 1) >= $arg0.length()) ? null : $arg0.substring(($arg1 - 1))" + + case _ => throw new IllegalArgumentException("SUBSTRING requires 2 or 3 arguments") + } + } + + override def validate(): Either[String, Unit] = + if (start < 1) + Left("SUBSTRING start position must be greater than or equal to 1 (SQL is 1-based)") + else if (length.exists(_ < 0)) + Left("SUBSTRING length must be non-negative") + else Right(()) + + override def toSQL(base: String): String = sql + + } + + case class Concat(values: List[PainlessScript]) extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def stringOp: StringOp = Concat + + override def args: List[PainlessScript] = values + + override def nullable: Boolean = values.exists(_.nullable) + + override def toPainlessCall(callArgs: List[String]): String = { + if (callArgs.isEmpty) + throw new IllegalArgumentException("CONCAT requires at least one argument") + else + callArgs.zipWithIndex + .map { case (arg, idx) => + SQLTypeUtils.coerce(arg, values(idx).out, SQLTypes.Varchar, nullable = false) + } + .mkString(stringOp.painless) + } + + override def validate(): Either[String, Unit] = + if (values.isEmpty) Left("CONCAT requires at least one argument") + else Right(()) + + override def toSQL(base: String): String = sql + } + + case object SQLLength extends StringFunction[SQLBigInt] { + override def outputType: SQLBigInt = SQLTypes.BigInt + override def stringOp: StringOp = Length + override def args: List[PainlessScript] = List.empty + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala new file mode 100644 index 00000000..1f5493ac --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -0,0 +1,389 @@ +package app.softnetwork.elastic.sql.function + +import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, MathScript, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.operator.time._ +import app.softnetwork.elastic.sql.`type`.{ + SQLDate, + SQLDateTime, + SQLNumeric, + SQLTemporal, + SQLType, + SQLTypeUtils, + SQLTypes, + SQLVarchar +} +import app.softnetwork.elastic.sql.time.{TimeInterval, TimeUnit} + +package object time { + + sealed trait IntervalFunction[IO <: SQLTemporal] + extends TransformFunction[IO, IO] + with MathScript { + def operator: IntervalOperator + + override def fun: Option[IntervalOperator] = Some(operator) + + def interval: TimeInterval + + override def args: List[PainlessScript] = List(interval) + + override def argsSeparator: String = " " + override def sql: String = s"$operator${args.map(_.sql).mkString(argsSeparator)}" + + override def script: String = s"${operator.script}${interval.script}" + + private[this] var _out: SQLType = outputType + + override def out: SQLType = _out + + override def applyType(in: SQLType): SQLType = { + _out = interval.checkType(in).getOrElse(out) + _out + } + + override def validate(): Either[String, Unit] = interval.checkType(out) match { + case Left(err) => Left(err) + case Right(_) => Right(()) + } + + override def toPainless(base: String, idx: Int): String = + if (nullable) + s"(def e$idx = $base; e$idx != null ? ${SQLTypeUtils.coerce(s"e$idx", expr.out, out, nullable = false)}$painless : null)" + else + s"${SQLTypeUtils.coerce(base, expr.out, out, nullable = expr.nullable)}$painless" + } + + sealed trait AddInterval[IO <: SQLTemporal] extends IntervalFunction[IO] { + override def operator: IntervalOperator = Plus + } + + sealed trait SubtractInterval[IO <: SQLTemporal] extends IntervalFunction[IO] { + override def operator: IntervalOperator = Minus + } + + case class SQLAddInterval(interval: TimeInterval) extends AddInterval[SQLTemporal] { + override def inputType: SQLTemporal = SQLTypes.Temporal + override def outputType: SQLTemporal = SQLTypes.Temporal + } + + case class SQLSubtractInterval(interval: TimeInterval) extends SubtractInterval[SQLTemporal] { + override def inputType: SQLTemporal = SQLTypes.Temporal + override def outputType: SQLTemporal = SQLTypes.Temporal + } + + sealed trait DateTimeFunction extends Function { + def now: String = "ZonedDateTime.now(ZoneId.of('Z'))" + override def out: SQLType = SQLTypes.DateTime + } + + sealed trait DateFunction extends DateTimeFunction { + override def out: SQLType = SQLTypes.Date + } + + sealed trait TimeFunction extends DateTimeFunction { + override def out: SQLType = SQLTypes.Time + } + + sealed trait SystemFunction extends Function { + override def system: Boolean = true + } + + sealed trait CurrentFunction extends SystemFunction with PainlessScript + + sealed trait CurrentDateTimeFunction + extends DateTimeFunction + with CurrentFunction + with MathScript { + override def painless: String = now + override def script: String = "now" + } + + sealed trait CurrentDateFunction extends DateFunction with CurrentFunction with MathScript { + override def painless: String = s"$now.toLocalDate()" + override def script: String = "now" + } + + sealed trait CurrentTimeFunction extends TimeFunction with CurrentFunction { + override def painless: String = s"$now.toLocalTime()" + } + + case object CurrentDate extends Expr("current_date") with CurrentDateFunction + + case object CurentDateWithParens extends Expr("current_date()") with CurrentDateFunction + + case object CurrentTime extends Expr("current_time") with CurrentTimeFunction + + case object CurrentTimeWithParens extends Expr("current_time()") with CurrentTimeFunction + + case object CurrentTimestamp extends Expr("current_timestamp") with CurrentDateTimeFunction + + case object CurrentTimestampWithParens + extends Expr("current_timestamp()") + with CurrentDateTimeFunction + + case object Now extends Expr("now") with CurrentDateTimeFunction + + case object NowWithParens extends Expr("now()") with CurrentDateTimeFunction + + case object DateTrunc extends Expr("date_trunc") with TokenRegex with PainlessScript { + override def painless: String = ".truncatedTo" + } + + case class DateTrunc(identifier: GenericIdentifier, unit: TimeUnit) + extends DateTimeFunction + with TransformFunction[SQLTemporal, SQLTemporal] + with FunctionWithIdentifier { + override def fun: Option[PainlessScript] = Some(DateTrunc) + + override def args: List[PainlessScript] = List(unit) + + override def inputType: SQLTemporal = SQLTypes.Temporal // par défaut + override def outputType: SQLTemporal = SQLTypes.Temporal // idem + + override def sql: String = DateTrunc.sql + override def toSQL(base: String): String = { + s"$sql($base, ${unit.sql})" + } + } + + case object Extract extends Expr("extract") with TokenRegex with PainlessScript { + override def painless: String = ".get" + } + + case class Extract(unit: TimeUnit, override val sql: String = "extract") + extends DateTimeFunction + with TransformFunction[SQLTemporal, SQLNumeric] { + override def fun: Option[PainlessScript] = Some(Extract) + + override def args: List[PainlessScript] = List(unit) + + override def inputType: SQLTemporal = SQLTypes.Temporal + override def outputType: SQLNumeric = SQLTypes.Numeric + + override def toSQL(base: String): String = s"$sql(${unit.sql} from $base)" + + } + + import TimeUnit._ + + object YEAR extends Extract(Year, Year.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + object MONTH extends Extract(Month, Month.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + object DAY extends Extract(Day, Day.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + object HOUR extends Extract(Hour, Hour.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + object MINUTE extends Extract(Minute, Minute.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + object SECOND extends Extract(Second, Second.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + case object DateDiff extends Expr("date_diff") with TokenRegex with PainlessScript { + override def painless: String = ".between" + } + + case class DateDiff(end: PainlessScript, start: PainlessScript, unit: TimeUnit) + extends DateTimeFunction + with BinaryFunction[SQLDateTime, SQLDateTime, SQLNumeric] + with PainlessScript { + override def fun: Option[PainlessScript] = Some(DateDiff) + + override def inputType: SQLDateTime = SQLTypes.DateTime + override def outputType: SQLNumeric = SQLTypes.Numeric + + override def left: PainlessScript = start + override def right: PainlessScript = end + + override def sql: String = DateDiff.sql + + override def toSQL(base: String): String = s"$sql(${end.sql}, ${start.sql}, ${unit.sql})" + + override def toPainlessCall(callArgs: List[String]): String = + s"${unit.painless}${DateDiff.painless}(${callArgs.mkString(", ")})" + } + + case object DateAdd extends Expr("date_add") with TokenRegex + + case class DateAdd(identifier: GenericIdentifier, interval: TimeInterval) + extends DateFunction + with AddInterval[SQLDate] + with TransformFunction[SQLDate, SQLDate] + with FunctionWithIdentifier { + override def inputType: SQLDate = SQLTypes.Date + override def outputType: SQLDate = SQLTypes.Date + override def sql: String = DateAdd.sql + override def toSQL(base: String): String = { + s"$sql($base, ${interval.sql})" + } + } + + case object DateSub extends Expr("date_sub") with TokenRegex + + case class DateSub(identifier: GenericIdentifier, interval: TimeInterval) + extends DateFunction + with SubtractInterval[SQLDate] + with TransformFunction[SQLDate, SQLDate] + with FunctionWithIdentifier { + override def inputType: SQLDate = SQLTypes.Date + override def outputType: SQLDate = SQLTypes.Date + override def sql: String = DateSub.sql + override def toSQL(base: String): String = { + s"$sql($base, ${interval.sql})" + } + } + + case object ParseDate extends Expr("parse_date") with TokenRegex with PainlessScript { + override def painless: String = ".parse" + } + + case class ParseDate(identifier: GenericIdentifier, format: String) + extends DateFunction + with TransformFunction[SQLVarchar, SQLDate] + with FunctionWithIdentifier { + override def fun: Option[PainlessScript] = Some(ParseDate) + + override def args: List[PainlessScript] = List.empty + + override def inputType: SQLVarchar = SQLTypes.Varchar + override def outputType: SQLDate = SQLTypes.Date + + override def sql: String = ParseDate.sql + override def toSQL(base: String): String = { + s"$sql($base, '$format')" + } + + override def painless: String = throw new NotImplementedError("Use toPainless instead") + override def toPainless(base: String, idx: Int): String = + if (nullable) + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').parse(e$idx, LocalDate::from) : null)" + else + s"DateTimeFormatter.ofPattern('$format').parse($base, LocalDate::from)" + } + + case object FormatDate extends Expr("format_date") with TokenRegex with PainlessScript { + override def painless: String = ".format" + } + + case class FormatDate(identifier: GenericIdentifier, format: String) + extends DateFunction + with TransformFunction[SQLDate, SQLVarchar] + with FunctionWithIdentifier { + override def fun: Option[PainlessScript] = Some(FormatDate) + + override def args: List[PainlessScript] = List.empty + + override def inputType: SQLDate = SQLTypes.Date + override def outputType: SQLVarchar = SQLTypes.Varchar + + override def sql: String = FormatDate.sql + override def toSQL(base: String): String = { + s"$sql($base, '$format')" + } + + override def painless: String = throw new NotImplementedError("Use toPainless instead") + override def toPainless(base: String, idx: Int): String = + if (nullable) + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').format(e$idx) : null)" + else + s"DateTimeFormatter.ofPattern('$format').format($base)" + } + + case object DateTimeAdd extends Expr("datetime_add") with TokenRegex + + case class DateTimeAdd(identifier: GenericIdentifier, interval: TimeInterval) + extends DateTimeFunction + with AddInterval[SQLDateTime] + with TransformFunction[SQLDateTime, SQLDateTime] + with FunctionWithIdentifier { + override def inputType: SQLDateTime = SQLTypes.DateTime + override def outputType: SQLDateTime = SQLTypes.DateTime + override def sql: String = DateTimeAdd.sql + override def toSQL(base: String): String = { + s"$sql($base, ${interval.sql})" + } + } + + case object DateTimeSub extends Expr("datetime_sub") with TokenRegex + + case class DateTimeSub(identifier: GenericIdentifier, interval: TimeInterval) + extends DateTimeFunction + with SubtractInterval[SQLDateTime] + with TransformFunction[SQLDateTime, SQLDateTime] + with FunctionWithIdentifier { + override def inputType: SQLDateTime = SQLTypes.DateTime + override def outputType: SQLDateTime = SQLTypes.DateTime + override def sql: String = DateTimeSub.sql + override def toSQL(base: String): String = { + s"$sql($base, ${interval.sql})" + } + } + + case object ParseDateTime extends Expr("parse_datetime") with TokenRegex with PainlessScript { + override def painless: String = ".parse" + } + + case class ParseDateTime(identifier: GenericIdentifier, format: String) + extends DateTimeFunction + with TransformFunction[SQLVarchar, SQLDateTime] + with FunctionWithIdentifier { + override def fun: Option[PainlessScript] = Some(ParseDateTime) + + override def args: List[PainlessScript] = List.empty + + override def inputType: SQLVarchar = SQLTypes.Varchar + override def outputType: SQLDateTime = SQLTypes.DateTime + + override def sql: String = ParseDateTime.sql + override def toSQL(base: String): String = { + s"$sql($base, '$format')" + } + + override def painless: String = throw new NotImplementedError("Use toPainless instead") + override def toPainless(base: String, idx: Int): String = + if (nullable) + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').parse(e$idx, ZonedDateTime::from) : null)" + else + s"DateTimeFormatter.ofPattern('$format').parse($base, ZonedDateTime::from)" + } + + case object FormatDateTime extends Expr("format_datetime") with TokenRegex with PainlessScript { + override def painless: String = ".format" + } + + case class FormatDateTime(identifier: GenericIdentifier, format: String) + extends DateTimeFunction + with TransformFunction[SQLDateTime, SQLVarchar] + with FunctionWithIdentifier { + override def fun: Option[PainlessScript] = Some(FormatDateTime) + + override def args: List[PainlessScript] = List.empty + + override def inputType: SQLDateTime = SQLTypes.DateTime + override def outputType: SQLVarchar = SQLTypes.Varchar + + override def sql: String = FormatDateTime.sql + override def toSQL(base: String): String = { + s"$sql($base, '$format')" + } + + override def painless: String = throw new NotImplementedError("Use toPainless instead") + override def toPainless(base: String, idx: Int): String = + if (nullable) + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').format(e$idx) : null)" + else + s"DateTimeFormatter.ofPattern('$format').format($base)" + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala new file mode 100644 index 00000000..b7398c40 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -0,0 +1,83 @@ +package app.softnetwork.elastic.sql.operator.math + +import app.softnetwork.elastic.sql._ +import app.softnetwork.elastic.sql.`type`._ +import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} + +case class ArithmeticExpression( + left: PainlessScript, + operator: ArithmeticOperator, + right: PainlessScript, + group: Boolean = false +) extends TransformFunction[SQLNumeric, SQLNumeric] + with BinaryFunction[SQLNumeric, SQLNumeric, SQLNumeric] { + + override def fun: Option[ArithmeticOperator] = Some(operator) + + override def inputType: SQLNumeric = SQLTypes.Numeric + override def outputType: SQLNumeric = SQLTypes.Numeric + + override def applyType(in: SQLType): SQLType = in + + override def sql: String = { + val expr = s"${left.sql}$operator${right.sql}" + if (group) + s"($expr)" + else + expr + } + + override def out: SQLType = + SQLTypeUtils.leastCommonSuperType(List(left.out, right.out)) + + override def validate(): Either[String, Unit] = { + for { + _ <- left.validate() + _ <- right.validate() + _ <- Validator.validateTypesMatching(left.out, right.out) + } yield () + } + + override def nullable: Boolean = left.nullable || right.nullable + + override def toPainless(base: String, idx: Int): String = { + if (nullable) { + val l = left match { + case t: TransformFunction[_, _] => + SQLTypeUtils.coerce(t.toPainless("", idx + 1), left.out, out, nullable = false) + case _ => SQLTypeUtils.coerce(left.painless, left.out, out, nullable = false) + } + val r = right match { + case t: TransformFunction[_, _] => + SQLTypeUtils.coerce(t.toPainless("", idx + 1), right.out, out, nullable = false) + case _ => SQLTypeUtils.coerce(right.painless, right.out, out, nullable = false) + } + var expr = "" + if (left.nullable) + expr += s"def lv$idx = ($l); " + if (right.nullable) + expr += s"def rv$idx = ($r); " + if (left.nullable && right.nullable) + expr += s"(lv$idx == null || rv$idx == null) ? null : (lv$idx ${operator.painless} rv$idx)" + else if (left.nullable) + expr += s"(lv$idx == null) ? null : (lv$idx ${operator.painless} $r)" + else + expr += s"(rv$idx == null) ? null : ($l ${operator.painless} rv$idx)" + if (group) + expr = s"($expr)" + return s"$base$expr" + } + s"$base$painless" + } + + override def painless: String = { + val l = SQLTypeUtils.coerce(left, out) + val r = SQLTypeUtils.coerce(right, out) + val expr = s"$l ${operator.painless} $r" + if (group) + s"($expr)" + else + expr + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala new file mode 100644 index 00000000..42c882fc --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala @@ -0,0 +1,17 @@ +package app.softnetwork.elastic.sql.operator + +import app.softnetwork.elastic.sql.Expr + +package object math { + + sealed trait ArithmeticOperator extends Operator with BinaryOperator { + override def toString: String = s" $sql " + } + + case object Add extends Expr("+") with ArithmeticOperator + case object Subtract extends Expr("-") with ArithmeticOperator + case object Multiply extends Expr("*") with ArithmeticOperator + case object Divide extends Expr("/") with ArithmeticOperator + case object Modulo extends Expr("%") with ArithmeticOperator + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala new file mode 100644 index 00000000..3c09b572 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala @@ -0,0 +1,67 @@ +package app.softnetwork.elastic.sql + +package object operator { + + trait Operator extends Token with PainlessScript with TokenRegex { + override def painless: String = this match { + case And => "&&" + case Or => "||" + case Not => "!" + case In => ".contains" + case Like | Match => ".matches" + case Eq => "==" + case Ne => "!=" + case IsNull => " == null" + case IsNotNull => " != null" + case _ => sql + } + } + + trait BinaryOperator extends Operator + + trait ExpressionOperator extends Operator + + sealed trait ComparisonOperator extends ExpressionOperator with PainlessScript { + def not: ComparisonOperator = this match { + case Eq => Ne + case Ne | Diff => Eq + case Ge => Lt + case Gt => Le + case Le => Gt + case Lt => Ge + } + } + + case object Eq extends Expr("=") with ComparisonOperator + case object Ne extends Expr("<>") with ComparisonOperator + case object Diff extends Expr("!=") with ComparisonOperator + case object Ge extends Expr(">=") with ComparisonOperator + case object Gt extends Expr(">") with ComparisonOperator + case object Le extends Expr("<=") with ComparisonOperator + case object Lt extends Expr("<") with ComparisonOperator + case object In extends Expr("in") with ComparisonOperator + case object Like extends Expr("like") with ComparisonOperator + case object Between extends Expr("between") with ComparisonOperator + case object IsNull extends Expr("is null") with ComparisonOperator + case object IsNotNull extends Expr("is not null") with ComparisonOperator + + case object Match extends Expr("match") with ComparisonOperator + case object Against extends Expr("against") with TokenRegex + + sealed trait LogicalOperator extends ExpressionOperator + + case object Not extends Expr("not") with LogicalOperator + + sealed trait PredicateOperator extends LogicalOperator + + case object And extends Expr("and") with PredicateOperator + case object Or extends Expr("or") with PredicateOperator + + case object Union extends Expr("union") with Operator with TokenRegex + + sealed trait ElasticOperator extends Operator with TokenRegex + + case object Nested extends Expr("nested") with ElasticOperator + case object Child extends Expr("child") with ElasticOperator + case object Parent extends Expr("parent") with ElasticOperator +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala new file mode 100644 index 00000000..e824e90f --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala @@ -0,0 +1,24 @@ +package app.softnetwork.elastic.sql.operator + +import app.softnetwork.elastic.sql.{Expr, MathScript} + +package object time { + + sealed trait IntervalOperator extends Operator with BinaryOperator with MathScript { + override def script: String = sql + override def toString: String = s" $sql " + override def painless: String = this match { + case Plus => ".plus" + case Minus => ".minus" + case _ => sql + } + } + + case object Plus extends Expr("+") with IntervalOperator { + override def painless: String = ".plus" + } + case object Minus extends Expr("-") with IntervalOperator { + override def painless: String = ".minus" + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 522e8358..9e4ed209 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1,5 +1,8 @@ package app.softnetwork.elastic +import app.softnetwork.elastic.sql.function.aggregate.{Max, Min} +import app.softnetwork.elastic.sql.operator._ + import java.security.MessageDigest import java.util.regex.Pattern import scala.reflect.runtime.universe._ @@ -10,14 +13,17 @@ import scala.util.matching.Regex */ package object sql { + import app.softnetwork.elastic.sql.function._ + import app.softnetwork.elastic.sql.`type`._ + import scala.language.implicitConversions - implicit def asString(token: Option[_ <: SQLToken]): String = token match { + implicit def asString(token: Option[_ <: Token]): String = token match { case Some(t) => t.toString case _ => "" } - trait SQLToken extends Serializable with SQLValidation { + trait Token extends Serializable with Validation { def sql: String override def toString: String = sql def baseType: SQLType = SQLTypes.Any @@ -27,30 +33,30 @@ package object sql { def nullable: Boolean = !system } - trait PainlessScript extends SQLToken { + trait PainlessScript extends Token { def painless: String def nullValue: String = "null" } - trait MathScript extends SQLToken { + trait MathScript extends Token { def script: String } - trait Updateable extends SQLToken { + trait Updateable extends Token { def update(request: SQLSearchRequest): Updateable } - abstract class SQLExpr(override val sql: String) extends SQLToken + abstract class Expr(override val sql: String) extends Token - case object Distinct extends SQLExpr("distinct") with SQLRegex + case object Distinct extends Expr("distinct") with TokenRegex - abstract class SQLValue[+T](val value: T)(implicit ev$1: T => Ordered[T]) - extends SQLToken + abstract class Value[+T](val value: T)(implicit ev$1: T => Ordered[T]) + extends Token with PainlessScript - with SQLFunctionWithValue[T] { + with FunctionWithValue[T] { def choose[R >: T]( values: Seq[R], - operator: Option[SQLExpressionOperator], + operator: Option[ExpressionOperator], separator: String = "|" )(implicit ev: R => Ordered[R]): Option[R] = { if (values.isEmpty) @@ -76,24 +82,24 @@ package object sql { override def nullable: Boolean = false } - case object SQLNull extends SQLValue[Null](null) { + case object Null extends Value[Null](null) { override def sql: String = "null" override def painless: String = "null" override def nullable: Boolean = true override def out: SQLType = SQLTypes.Null } - case class SQLBoolean(override val value: Boolean) extends SQLValue[Boolean](value) { + case class BooleanValue(override val value: Boolean) extends Value[Boolean](value) { override def sql: String = value.toString override def out: SQLType = SQLTypes.Boolean } - case class SQLCharValue(override val value: Char) extends SQLValue[Char](value) { + case class CharValue(override val value: Char) extends Value[Char](value) { override def sql: String = s"""'$value'""" override def out: SQLType = SQLTypes.Char } - case class SQLStringValue(override val value: String) extends SQLValue[String](value) { + case class StringValue(override val value: String) extends Value[String](value) { override def sql: String = s"""'$value'""" import SQLImplicits._ private lazy val pattern: Pattern = value.pattern @@ -108,7 +114,7 @@ package object sql { } override def choose[R >: String]( values: Seq[R], - operator: Option[SQLExpressionOperator], + operator: Option[ExpressionOperator], separator: String = "|" )(implicit ev: R => Ordered[R]): Option[R] = { operator match { @@ -122,13 +128,13 @@ package object sql { override def out: SQLType = SQLTypes.Varchar } - sealed abstract class SQLNumericValue[T: Numeric](override val value: T)(implicit + sealed abstract class NumericValue[T: Numeric](override val value: T)(implicit ev$1: T => Ordered[T] - ) extends SQLValue[T](value) { + ) extends Value[T](value) { override def sql: String = value.toString override def choose[R >: T]( values: Seq[R], - operator: Option[SQLExpressionOperator], + operator: Option[ExpressionOperator], separator: String = "|" )(implicit ev: R => Ordered[R]): Option[R] = { operator match { @@ -156,48 +162,48 @@ package object sql { override def out: SQLNumeric = SQLTypes.Numeric } - case class SQLByteValue(override val value: Byte) extends SQLNumericValue[Byte](value) { + case class ByteValue(override val value: Byte) extends NumericValue[Byte](value) { override def out: SQLNumeric = SQLTypes.TinyInt } - case class SQLShortValue(override val value: Short) extends SQLNumericValue[Short](value) { + case class ShortValue(override val value: Short) extends NumericValue[Short](value) { override def out: SQLNumeric = SQLTypes.SmallInt } - case class SQLIntValue(override val value: Int) extends SQLNumericValue[Int](value) { + case class IntValue(override val value: Int) extends NumericValue[Int](value) { override def out: SQLNumeric = SQLTypes.Int } - case class SQLLongValue(override val value: Long) extends SQLNumericValue[Long](value) { + case class LongValue(override val value: Long) extends NumericValue[Long](value) { override def out: SQLNumeric = SQLTypes.BigInt } - case class SQLFloatValue(override val value: Float) extends SQLNumericValue[Float](value) { + case class FloatValue(override val value: Float) extends NumericValue[Float](value) { override def out: SQLNumeric = SQLTypes.Real } - case class SQLDoubleValue(override val value: Double) extends SQLNumericValue[Double](value) { + case class DoubleValue(override val value: Double) extends NumericValue[Double](value) { override def out: SQLNumeric = SQLTypes.Double } - case object SQLPiValue extends SQLValue[Double](Math.PI) { + case object PiValue extends Value[Double](Math.PI) { override def sql: String = "pi" override def painless: String = "Math.PI" override def out: SQLNumeric = SQLTypes.Double } - case object SQLEValue extends SQLValue[Double](Math.E) { + case object EValue extends Value[Double](Math.E) { override def sql: String = "e" override def painless: String = "Math.E" override def out: SQLNumeric = SQLTypes.Double } - sealed abstract class SQLFromTo[+T](val from: SQLValue[T], val to: SQLValue[T]) extends SQLToken { + sealed abstract class FromTo[+T](val from: Value[T], val to: Value[T]) extends Token { override def sql = s"${from.sql} and ${to.sql}" } - case class SQLLiteralFromTo(override val from: SQLStringValue, override val to: SQLStringValue) - extends SQLFromTo[String](from, to) { + case class LiteralFromTo(override val from: StringValue, override val to: StringValue) + extends FromTo[String](from, to) { def between: Seq[String] => Boolean = { _.exists { s => s >= from.value && s <= to.value } } @@ -206,8 +212,8 @@ package object sql { } } - case class SQLLongFromTo(override val from: SQLLongValue, override val to: SQLLongValue) - extends SQLFromTo[Long](from, to) { + case class LongFromTo(override val from: LongValue, override val to: LongValue) + extends FromTo[Long](from, to) { def between: Seq[Long] => Boolean = { _.exists { n => n >= from.value && n <= to.value } } @@ -216,8 +222,8 @@ package object sql { } } - case class SQLDoubleFromTo(override val from: SQLDoubleValue, override val to: SQLDoubleValue) - extends SQLFromTo[Double](from, to) { + case class DoubleFromTo(override val from: DoubleValue, override val to: DoubleValue) + extends FromTo[Double](from, to) { def between: Seq[Double] => Boolean = { _.exists { n => n >= from.value && n <= to.value } } @@ -226,8 +232,8 @@ package object sql { } } - sealed abstract class SQLValues[+R: TypeTag, +T <: SQLValue[R]](val values: Seq[T]) - extends SQLToken + sealed abstract class Values[+R: TypeTag, +T <: Value[R]](val values: Seq[T]) + extends Token with PainlessScript { override def sql = s"(${values.map(_.sql).mkString(",")})" override def painless: String = s"[${values.map(_.painless).mkString(",")}]" @@ -236,8 +242,8 @@ package object sql { override def out: SQLArray = SQLTypes.Array(SQLTypes.Any) } - case class SQLStringValues(override val values: Seq[SQLStringValue]) - extends SQLValues[String, SQLValue[String]](values) { + case class StringValues(override val values: Seq[StringValue]) + extends Values[String, Value[String]](values) { def eq: Seq[String] => Boolean = { _.exists { s => innerValues.exists(_.contentEquals(s)) } } @@ -247,8 +253,8 @@ package object sql { override def out: SQLArray = SQLTypes.Array(SQLTypes.Varchar) } - class SQLNumericValues[R: TypeTag](override val values: Seq[SQLNumericValue[R]]) - extends SQLValues[R, SQLNumericValue[R]](values) { + class NumericValues[R: TypeTag](override val values: Seq[NumericValue[R]]) + extends Values[R, NumericValue[R]](values) { def eq: Seq[R] => Boolean = { _.exists { n => innerValues.contains(n) } } @@ -258,43 +264,40 @@ package object sql { override def out: SQLArray = SQLTypes.Array(SQLTypes.Numeric) } - case class SQLByteValues(override val values: Seq[SQLByteValue]) - extends SQLNumericValues[Byte](values) { + case class ByteValues(override val values: Seq[ByteValue]) extends NumericValues[Byte](values) { override def out: SQLArray = SQLTypes.Array(SQLTypes.TinyInt) } - case class SQLShortValues(override val values: Seq[SQLShortValue]) - extends SQLNumericValues[Short](values) { + case class ShortValues(override val values: Seq[ShortValue]) + extends NumericValues[Short](values) { override def out: SQLArray = SQLTypes.Array(SQLTypes.SmallInt) } - case class SQLIntValues(override val values: Seq[SQLIntValue]) - extends SQLNumericValues[Int](values) { + case class IntValues(override val values: Seq[IntValue]) extends NumericValues[Int](values) { override def out: SQLArray = SQLTypes.Array(SQLTypes.Int) } - case class SQLLongValues(override val values: Seq[SQLLongValue]) - extends SQLNumericValues[Long](values) { + case class LongValues(override val values: Seq[LongValue]) extends NumericValues[Long](values) { override def out: SQLArray = SQLTypes.Array(SQLTypes.BigInt) } - case class SQLFloatValues(override val values: Seq[SQLFloatValue]) - extends SQLNumericValues[Float](values) { + case class FloatValues(override val values: Seq[FloatValue]) + extends NumericValues[Float](values) { override def out: SQLArray = SQLTypes.Array(SQLTypes.Real) } - case class SQLDoubleValues(override val values: Seq[SQLDoubleValue]) - extends SQLNumericValues[Double](values) { + case class DoubleValues(override val values: Seq[DoubleValue]) + extends NumericValues[Double](values) { override def out: SQLArray = SQLTypes.Array(SQLTypes.Double) } def choose[T]( values: Seq[T], - criteria: Option[SQLCriteria], - function: Option[SQLFunction] = None + criteria: Option[Criteria], + function: Option[Function] = None )(implicit ev$1: T => Ordered[T]): Option[T] = { criteria match { - case Some(SQLExpression(_, operator, value: SQLValue[T] @unchecked, _)) => + case Some(GenericExpression(_, operator, value: Value[T] @unchecked, _)) => value.choose[T](values, Some(operator)) case _ => function match { @@ -322,9 +325,9 @@ package object sql { s"""${if (startWith) ".*"}$v${if (endWith) ".*"}""" } - case object Alias extends SQLExpr("as") with SQLRegex + case object Alias extends Expr("as") with TokenRegex - case class SQLAlias(alias: String) extends SQLExpr(s" ${Alias.sql} $alias") + case class Alias(alias: String) extends Expr(s" ${Alias.sql} $alias") object AliasUtils { private val MaxAliasLength = 50 @@ -361,23 +364,23 @@ package object sql { } } - trait SQLRegex extends SQLToken { + trait TokenRegex extends Token { lazy val regex: Regex = s"\\b(?i)$sql\\b".r } - trait SQLSource extends Updateable { + trait Source extends Updateable { def name: String - def update(request: SQLSearchRequest): SQLSource + def update(request: SQLSearchRequest): Source } - trait Identifier extends SQLToken with SQLSource with SQLFunctionChain with PainlessScript { + trait Identifier extends Token with Source with FunctionChain with PainlessScript { def name: String def tableAlias: Option[String] def distinct: Boolean def nested: Boolean def fieldAlias: Option[String] - def bucket: Option[SQLBucket] + def bucket: Option[Bucket] override def sql: String = { var parts: Seq[String] = name.split("\\.").toSeq tableAlias match { @@ -416,13 +419,13 @@ package object sql { else "" def toPainless(base: String): String = { - val orderedFunctions = SQLFunctionUtils.transformFunctions(this).reverse + val orderedFunctions = FunctionUtils.transformFunctions(this).reverse var expr = base orderedFunctions.zipWithIndex.foreach { case (f, idx) => f match { - case f: SQLTransformFunction[_, _] => expr = f.toPainless(expr, idx) - case f: PainlessScript => expr = s"$expr${f.painless}" - case f => expr = f.toSQL(expr) // fallback + case f: TransformFunction[_, _] => expr = f.toPainless(expr, idx) + case f: PainlessScript => expr = s"$expr${f.painless}" + case f => expr = f.toSQL(expr) // fallback } } expr @@ -451,18 +454,18 @@ package object sql { } - case class SQLIdentifier( + case class GenericIdentifier( name: String, tableAlias: Option[String] = None, distinct: Boolean = false, nested: Boolean = false, - limit: Option[SQLLimit] = None, - functions: List[SQLFunction] = List.empty, + limit: Option[Limit] = None, + functions: List[Function] = List.empty, fieldAlias: Option[String] = None, - bucket: Option[SQLBucket] = None + bucket: Option[Bucket] = None ) extends Identifier { - def update(request: SQLSearchRequest): SQLIdentifier = { + def update(request: SQLSearchRequest): GenericIdentifier = { val parts: Seq[String] = name.split("\\.").toSeq if (request.tableAliases.values.toSeq.contains(parts.head)) { request.unnests.find(_._1 == parts.head) match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala similarity index 67% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala index 21050a86..68860ba1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala @@ -1,10 +1,23 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.`type`._ +import app.softnetwork.elastic.sql.function._ +import app.softnetwork.elastic.sql.function.aggregate._ +import app.softnetwork.elastic.sql.function.cond._ +import app.softnetwork.elastic.sql.function.convert._ +import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.function.math._ +import app.softnetwork.elastic.sql.function.string._ +import app.softnetwork.elastic.sql.function.time._ +import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.operator.math._ +import app.softnetwork.elastic.sql.time.TimeUnit._ +import app.softnetwork.elastic.sql.time._ +import app.softnetwork.elastic.sql._ +import scala.language.implicitConversions import scala.util.parsing.combinator.{PackratParsers, RegexParsers} import scala.util.parsing.input.CharSequenceReader -import TimeUnit._ - -import scala.language.implicitConversions /** Created by smanciot on 27/06/2018. * @@ -26,7 +39,7 @@ object SQLParser case s ~ f ~ w ~ g ~ h ~ o ~ l => val request = SQLSearchRequest(s, f, w, g, h, o, l).update() request.validate() match { - case Left(error) => throw SQLValidationError(error) + case Left(error) => throw ValidationError(error) case _ => } request @@ -61,31 +74,31 @@ case class SQLParserError(msg: String) extends SQLCompilationError trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => - def literal: PackratParser[SQLStringValue] = - """"[^"]*"|'[^']*'""".r ^^ (str => SQLStringValue(str.substring(1, str.length - 1))) + def literal: PackratParser[StringValue] = + """"[^"]*"|'[^']*'""".r ^^ (str => StringValue(str.substring(1, str.length - 1))) - def long: PackratParser[SQLLongValue] = - """(-)?(0|[1-9]\d*)""".r ^^ (str => SQLLongValue(str.toLong)) + def long: PackratParser[LongValue] = + """(-)?(0|[1-9]\d*)""".r ^^ (str => LongValue(str.toLong)) - def double: PackratParser[SQLDoubleValue] = - """(-)?(\d+\.\d+)""".r ^^ (str => SQLDoubleValue(str.toDouble)) + def double: PackratParser[DoubleValue] = + """(-)?(\d+\.\d+)""".r ^^ (str => DoubleValue(str.toDouble)) - def pi: PackratParser[SQLValue[Double]] = - Pi.regex ^^ (_ => SQLPiValue) + def pi: PackratParser[Value[Double]] = + Pi.regex ^^ (_ => PiValue) - def boolean: PackratParser[SQLBoolean] = - """(true|false)""".r ^^ (bool => SQLBoolean(bool.toBoolean)) + def boolean: PackratParser[BooleanValue] = + """(true|false)""".r ^^ (bool => BooleanValue(bool.toBoolean)) - def value_identifier: PackratParser[SQLIdentifier] = (literal | long | double | pi | boolean) ^^ { - v => - SQLIdentifier("", functions = v :: Nil) - } + def value_identifier: PackratParser[GenericIdentifier] = + (literal | long | double | pi | boolean) ^^ { v => + GenericIdentifier("", functions = v :: Nil) + } - def start: PackratParser[SQLDelimiter] = "(" ^^ (_ => StartPredicate) + def start: PackratParser[Delimiter] = "(" ^^ (_ => StartPredicate) - def end: PackratParser[SQLDelimiter] = ")" ^^ (_ => EndPredicate) + def end: PackratParser[Delimiter] = ")" ^^ (_ => EndPredicate) - def separator: PackratParser[SQLDelimiter] = "," ^^ (_ => Separator) + def separator: PackratParser[Delimiter] = "," ^^ (_ => Separator) def count: PackratParser[AggregateFunction] = Count.regex ^^ (_ => Count) @@ -116,7 +129,7 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def time_unit: PackratParser[TimeUnit] = year | month | quarter | week | day | hour | minute | second - def parens: PackratParser[List[SQLDelimiter]] = + def parens: PackratParser[List[Delimiter]] = start ~ end ^^ { case s ~ e => s :: e :: Nil } def current_date: PackratParser[CurrentFunction] = @@ -138,9 +151,9 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => if (p.isDefined) NowWithParens else Now } - def add: PackratParser[IntervalOperator] = Add.sql ^^ (_ => Add) + def add: PackratParser[ArithmeticOperator] = Add.sql ^^ (_ => Add) - def subtract: PackratParser[IntervalOperator] = Subtract.sql ^^ (_ => Subtract) + def subtract: PackratParser[ArithmeticOperator] = Subtract.sql ^^ (_ => Subtract) def multiply: PackratParser[ArithmeticOperator] = Multiply.sql ^^ (_ => Multiply) @@ -150,7 +163,7 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def factor: PackratParser[PainlessScript] = "(" ~> arithmeticExpressionLevel2 <~ ")" ^^ { - case expr: SQLArithmeticExpression => + case expr: ArithmeticExpression => expr.copy(group = true) case other => other } | valueExpr @@ -158,7 +171,7 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def arithmeticExpressionLevel1: Parser[PainlessScript] = factor ~ rep((multiply | divide | modulo) ~ factor) ^^ { case left ~ list => list.foldLeft(left) { case (acc, op ~ right) => - SQLArithmeticExpression(acc, op, right) + ArithmeticExpression(acc, op, right) } } @@ -166,17 +179,18 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => arithmeticExpressionLevel1 ~ rep((add | subtract) ~ arithmeticExpressionLevel1) ^^ { case left ~ list => list.foldLeft(left) { case (acc, op ~ right) => - SQLArithmeticExpression(acc, op, right) + ArithmeticExpression(acc, op, right) } } - def identifierWithArithmeticExpression: Parser[SQLIdentifier] = arithmeticExpressionLevel2 ^^ { - case af: SQLArithmeticExpression => SQLIdentifier("", functions = af :: Nil) - case id: SQLIdentifier => id - case f: SQLFunctionWithIdentifier => f.identifier - case f: SQLFunction => SQLIdentifier("", functions = f :: Nil) - case other => throw new Exception(s"Unexpected expression $other") - } + def identifierWithArithmeticExpression: Parser[GenericIdentifier] = + arithmeticExpressionLevel2 ^^ { + case af: ArithmeticExpression => GenericIdentifier("", functions = af :: Nil) + case id: GenericIdentifier => id + case f: FunctionWithIdentifier => f.identifier + case f: Function => GenericIdentifier("", functions = f :: Nil) + case other => throw new Exception(s"Unexpected expression $other") + } def interval: PackratParser[TimeInterval] = Interval.regex ~ long ~ time_unit ^^ { case _ ~ l ~ u => @@ -193,80 +207,80 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => SQLSubtractInterval(it) } - def intervalFunction: PackratParser[SQLTransformFunction[SQLTemporal, SQLTemporal]] = + def intervalFunction: PackratParser[TransformFunction[SQLTemporal, SQLTemporal]] = add_interval | substract_interval - def identifierWithIntervalFunction: PackratParser[SQLIdentifier] = + def identifierWithIntervalFunction: PackratParser[GenericIdentifier] = (identifierWithFunction | identifier) ~ intervalFunction ^^ { case i ~ f => i.copy(functions = f +: i.functions) } - def identifierWithSystemFunction: PackratParser[SQLIdentifier] = + def identifierWithSystemFunction: PackratParser[GenericIdentifier] = (current_date | current_time | current_timestamp | now) ~ intervalFunction.? ^^ { case f1 ~ f2 => f2 match { - case Some(f) => SQLIdentifier("", functions = List(f, f1)) - case None => SQLIdentifier("", functions = List(f1)) + case Some(f) => GenericIdentifier("", functions = List(f, f1)) + case None => GenericIdentifier("", functions = List(f1)) } } - def date_trunc: PackratParser[SQLFunctionWithIdentifier] = + def date_trunc: PackratParser[FunctionWithIdentifier] = "(?i)date_trunc".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { case _ ~ _ ~ i ~ _ ~ u ~ _ => DateTrunc(i, u) } - def extract_identifier: PackratParser[SQLIdentifier] = + def extract_identifier: PackratParser[GenericIdentifier] = "(?i)extract".r ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => i.copy(functions = Extract(u) +: i.functions) } - def extract_year: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extract_year: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = Year.regex ^^ (_ => YEAR) - def extract_month: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extract_month: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = Month.regex ^^ (_ => MONTH) - def extract_day: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extract_day: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = Day.regex ^^ (_ => DAY) - def extract_hour: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extract_hour: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = Hour.regex ^^ (_ => HOUR) - def extract_minute: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extract_minute: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = Minute.regex ^^ (_ => MINUTE) - def extract_second: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extract_second: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = Second.regex ^^ (_ => SECOND) - def extractors: PackratParser[SQLTransformFunction[SQLTemporal, SQLNumeric]] = + def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = extract_year | extract_month | extract_day | extract_hour | extract_minute | extract_second - def date_add: PackratParser[DateFunction with SQLFunctionWithIdentifier] = + def date_add: PackratParser[DateFunction with FunctionWithIdentifier] = "(?i)date_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateAdd(i, t) } - def date_sub: PackratParser[DateFunction with SQLFunctionWithIdentifier] = + def date_sub: PackratParser[DateFunction with FunctionWithIdentifier] = "(?i)date_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateSub(i, t) } - def parse_date: PackratParser[DateFunction with SQLFunctionWithIdentifier] = + def parse_date: PackratParser[DateFunction with FunctionWithIdentifier] = "(?i)parse_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { - case l: SQLStringValue => - ParseDate(SQLIdentifier("", functions = l :: Nil), f.value) - case i: SQLIdentifier => + case l: StringValue => + ParseDate(GenericIdentifier("", functions = l :: Nil), f.value) + case i: GenericIdentifier => ParseDate(i, f.value) } } - def format_date: PackratParser[DateFunction with SQLFunctionWithIdentifier] = + def format_date: PackratParser[DateFunction with FunctionWithIdentifier] = "(?i)format_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ f ~ _ => FormatDate(i, f.value) @@ -274,30 +288,30 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def date_functions: PackratParser[DateFunction] = date_add | date_sub | parse_date | format_date - def datetime_add: PackratParser[DateTimeFunction with SQLFunctionWithIdentifier] = + def datetime_add: PackratParser[DateTimeFunction with FunctionWithIdentifier] = "(?i)datetime_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateTimeAdd(i, t) } - def datetime_sub: PackratParser[DateTimeFunction with SQLFunctionWithIdentifier] = + def datetime_sub: PackratParser[DateTimeFunction with FunctionWithIdentifier] = "(?i)datetime_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateTimeSub(i, t) } - def parse_datetime: PackratParser[DateTimeFunction with SQLFunctionWithIdentifier] = + def parse_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = "(?i)parse_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: SQLLiteral => - ParseDateTime(SQLIdentifier("", functions = l :: Nil), f.value) - case i: SQLIdentifier => + ParseDateTime(GenericIdentifier("", functions = l :: Nil), f.value) + case i: GenericIdentifier => ParseDateTime(i, f.value) } } - def format_datetime: PackratParser[DateTimeFunction with SQLFunctionWithIdentifier] = + def format_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = "(?i)format_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ f ~ _ => FormatDateTime(i, f.value) @@ -308,9 +322,9 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def aggregates: PackratParser[AggregateFunction] = count | min | max | avg | sum - def distance: PackratParser[SQLFunction] = Distance.regex ^^ (_ => Distance) + def distance: PackratParser[Function] = Distance.regex ^^ (_ => Distance) - def identifierWithTemporalFunction: PackratParser[SQLIdentifier] = + def identifierWithTemporalFunction: PackratParser[GenericIdentifier] = rep1sep( date_trunc | extractors | date_functions | datetime_functions, start @@ -321,30 +335,30 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => case Some(id) => id.copy(functions = id.functions ++ f) case None => f.lastOption match { - case Some(fi: SQLFunctionWithIdentifier) => + case Some(fi: FunctionWithIdentifier) => fi.identifier.copy(functions = f ++ fi.identifier.functions) - case _ => SQLIdentifier("", functions = f) + case _ => GenericIdentifier("", functions = f) } } } - def date_diff: PackratParser[SQLBinaryFunction[_, _, _]] = + def date_diff: PackratParser[BinaryFunction[_, _, _]] = "(?i)date_diff".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) } - def date_diff_identifier: PackratParser[SQLIdentifier] = date_diff ^^ { dd => - SQLIdentifier("", functions = dd :: Nil) + def date_diff_identifier: PackratParser[GenericIdentifier] = date_diff ^^ { dd => + GenericIdentifier("", functions = dd :: Nil) } - def is_null: PackratParser[SQLConditionalFunction[_]] = + def is_null: PackratParser[ConditionalFunction[_]] = "(?i)isnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => SQLIsNullFunction(i) + case _ ~ _ ~ i ~ _ => IsNullFunction(i) } - def is_notnull: PackratParser[SQLConditionalFunction[_]] = + def is_notnull: PackratParser[ConditionalFunction[_]] = "(?i)isnotnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => SQLIsNotNullFunction(i) + case _ ~ _ ~ i ~ _ => IsNotNullFunction(i) } def valueExpr: PackratParser[PainlessScript] = @@ -363,17 +377,17 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => boolean | identifier - def coalesce: PackratParser[SQLCoalesce] = + def coalesce: PackratParser[Coalesce] = Coalesce.regex ~ start ~ rep1sep( valueExpr, separator ) ~ end ^^ { case _ ~ _ ~ ids ~ _ => - SQLCoalesce(ids) + Coalesce(ids) } - def nullif: PackratParser[SQLNullIf] = + def nullif: PackratParser[NullIf] = NullIf.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ end ^^ { - case _ ~ _ ~ id1 ~ _ ~ id2 ~ _ => SQLNullIf(id1, id2) + case _ ~ _ ~ id1 ~ _ ~ id2 ~ _ => NullIf(id1, id2) } def start_case: PackratParser[StartCase.type] = Case.regex ^^ (_ => StartCase) @@ -390,115 +404,116 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => when_case ~ (whereCriteria | valueExpr) ~ then_case.? ~ valueExpr ^^ { case _ ~ c ~ _ ~ r => c match { case p: PainlessScript => p -> r - case rawTokens: List[SQLToken] => + case rawTokens: List[Token] => processTokens(rawTokens) match { case Some(criteria) => criteria -> r - case _ => SQLNull -> r + case _ => Null -> r } } } def case_else: Parser[PainlessScript] = else_case ~ valueExpr ^^ { case _ ~ r => r } - def case_when: PackratParser[SQLCaseWhen] = + def case_when: PackratParser[Case] = start_case ~ valueExpr.? ~ rep1(case_condition) ~ case_else.? ~ end_case ^^ { - case _ ~ e ~ c ~ r ~ _ => SQLCaseWhen(e, c, r) + case _ ~ e ~ c ~ r ~ _ => Case(e, c, r) } - def case_when_identifier: Parser[SQLIdentifier] = case_when ^^ { cw => - SQLIdentifier("", functions = cw :: Nil) + def case_when_identifier: Parser[GenericIdentifier] = case_when ^^ { cw => + GenericIdentifier("", functions = cw :: Nil) } - def logical_functions: PackratParser[SQLTransformFunction[_, _]] = + def logical_functions: PackratParser[TransformFunction[_, _]] = is_null | is_notnull | coalesce | nullif | case_when - private[this] def abs: PackratParser[UnaryArithmeticOperator] = Abs.regex ^^ (_ => Abs) + private[this] def abs: PackratParser[MathOp] = Abs.regex ^^ (_ => Abs) - private[this] def ceil: PackratParser[UnaryArithmeticOperator] = Ceil.regex ^^ (_ => Ceil) + private[this] def ceil: PackratParser[MathOp] = Ceil.regex ^^ (_ => Ceil) - private[this] def floor: PackratParser[UnaryArithmeticOperator] = Floor.regex ^^ (_ => Floor) + private[this] def floor: PackratParser[MathOp] = Floor.regex ^^ (_ => Floor) - private[this] def exp: PackratParser[UnaryArithmeticOperator] = Exp.regex ^^ (_ => Exp) + private[this] def exp: PackratParser[MathOp] = Exp.regex ^^ (_ => Exp) - private[this] def sqrt: PackratParser[UnaryArithmeticOperator] = Sqrt.regex ^^ (_ => Sqrt) + private[this] def sqrt: PackratParser[MathOp] = Sqrt.regex ^^ (_ => Sqrt) - private[this] def log: PackratParser[UnaryArithmeticOperator] = Log.regex ^^ (_ => Log) + private[this] def log: PackratParser[MathOp] = Log.regex ^^ (_ => Log) - private[this] def log10: PackratParser[UnaryArithmeticOperator] = Log10.regex ^^ (_ => Log10) + private[this] def log10: PackratParser[MathOp] = Log10.regex ^^ (_ => Log10) - implicit def functionAsIdentifier(mf: SQLFunction): SQLIdentifier = mf match { - case id: SQLIdentifier => id - case fid: SQLFunctionWithIdentifier => fid.identifier - case _ => SQLIdentifier("", functions = mf :: Nil) + implicit def functionAsIdentifier(mf: Function): GenericIdentifier = mf match { + case id: GenericIdentifier => id + case fid: FunctionWithIdentifier => fid.identifier + case _ => GenericIdentifier("", functions = mf :: Nil) } def arithmeticFunction: PackratParser[MathematicalFunction] = (abs | ceil | exp | floor | log | log10 | sqrt) ~ start ~ valueExpr ~ end ^^ { - case op ~ _ ~ v ~ _ => SQLMathematicalFunction(op, v) + case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) } - private[this] def sin: PackratParser[TrigonometricOperator] = Sin.regex ^^ (_ => Sin) + private[this] def sin: PackratParser[Trigonometric] = Sin.regex ^^ (_ => Sin) - private[this] def asin: PackratParser[TrigonometricOperator] = Asin.regex ^^ (_ => Asin) + private[this] def asin: PackratParser[Trigonometric] = Asin.regex ^^ (_ => Asin) - private[this] def cos: PackratParser[TrigonometricOperator] = Cos.regex ^^ (_ => Cos) + private[this] def cos: PackratParser[Trigonometric] = Cos.regex ^^ (_ => Cos) - private[this] def acos: PackratParser[TrigonometricOperator] = Acos.regex ^^ (_ => Acos) + private[this] def acos: PackratParser[Trigonometric] = Acos.regex ^^ (_ => Acos) - private[this] def tan: PackratParser[TrigonometricOperator] = Tan.regex ^^ (_ => Tan) + private[this] def tan: PackratParser[Trigonometric] = Tan.regex ^^ (_ => Tan) - private[this] def atan: PackratParser[TrigonometricOperator] = Atan.regex ^^ (_ => Atan) + private[this] def atan: PackratParser[Trigonometric] = Atan.regex ^^ (_ => Atan) - private[this] def atan2: PackratParser[TrigonometricOperator] = Atan2.regex ^^ (_ => Atan2) + private[this] def atan2: PackratParser[Trigonometric] = Atan2.regex ^^ (_ => Atan2) def atan2Function: PackratParser[MathematicalFunction] = atan2 ~ start ~ (double | valueExpr) ~ separator ~ (double | valueExpr) ~ end ^^ { - case _ ~ _ ~ y ~ _ ~ x ~ _ => SQLAtan2(y, x) + case _ ~ _ ~ y ~ _ ~ x ~ _ => Atan2(y, x) } def trigonometricFunction: PackratParser[MathematicalFunction] = atan2Function | ((sin | asin | cos | acos | tan | atan) ~ start ~ valueExpr ~ end ^^ { - case op ~ _ ~ v ~ _ => SQLMathematicalFunction(op, v) + case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) }) - private[this] def round: PackratParser[UnaryArithmeticOperator] = Round.regex ^^ (_ => Round) + private[this] def round: PackratParser[MathOp] = Round.regex ^^ (_ => Round) def roundFunction: PackratParser[MathematicalFunction] = round ~ start ~ valueExpr ~ separator.? ~ long.? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ _ => - SQLRound(v, s.map(_.value.toInt)) + Round(v, s.map(_.value.toInt)) } - private[this] def pow: PackratParser[UnaryArithmeticOperator] = Pow.regex ^^ (_ => Pow) + private[this] def pow: PackratParser[MathOp] = Pow.regex ^^ (_ => Pow) def powFunction: PackratParser[MathematicalFunction] = pow ~ start ~ valueExpr ~ separator ~ long ~ end ^^ { case _ ~ _ ~ v1 ~ _ ~ e ~ _ => - SQLPow(v1, e.value.toInt) + Pow(v1, e.value.toInt) } - private[this] def sign: PackratParser[UnaryArithmeticOperator] = Sign.regex ^^ (_ => Sign) + private[this] def sign: PackratParser[MathOp] = Sign.regex ^^ (_ => Sign) def signFunction: PackratParser[MathematicalFunction] = - sign ~ start ~ valueExpr ~ end ^^ { case _ ~ _ ~ v ~ _ => SQLSign(v) } + sign ~ start ~ valueExpr ~ end ^^ { case _ ~ _ ~ v ~ _ => Sign(v) } def mathematicalFunction: PackratParser[MathematicalFunction] = arithmeticFunction | trigonometricFunction | roundFunction | powFunction | signFunction - def mathematicalFunctionWithIdentifier: PackratParser[SQLIdentifier] = mathematicalFunction ^^ { - mf => mf.identifier - } + def mathematicalFunctionWithIdentifier: PackratParser[GenericIdentifier] = + mathematicalFunction ^^ { mf => + mf.identifier + } def concatFunction: PackratParser[StringFunction[SQLVarchar]] = Concat.regex ~ start ~ rep1sep(valueExpr, separator) ~ end ^^ { case _ ~ _ ~ vs ~ _ => - SQLConcat(vs) + Concat(vs) } def substringFunction: PackratParser[StringFunction[SQLVarchar]] = Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((To.regex | separator) ~ long).? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ eOpt ~ _ => - SQLSubstring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) + Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) } - def stringFunctionWithIdentifier: PackratParser[SQLIdentifier] = + def stringFunctionWithIdentifier: PackratParser[GenericIdentifier] = (concatFunction | substringFunction) ^^ { sf => sf.identifier } @@ -510,24 +525,24 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def lower: PackratParser[StringFunction[SQLVarchar]] = Lower.regex ^^ { _ => - SQLStringFunction(Lower) + StringFunctionWithOp(Lower) } def upper: PackratParser[StringFunction[SQLVarchar]] = Upper.regex ^^ { _ => - SQLStringFunction(Upper) + StringFunctionWithOp(Upper) } def trim: PackratParser[StringFunction[SQLVarchar]] = Trim.regex ^^ { _ => - SQLStringFunction(Trim) + StringFunctionWithOp(Trim) } def string_functions: Parser[ StringFunction[_] ] = /*concatFunction | substringFunction |*/ length | lower | upper | trim - def sql_functions: PackratParser[SQLFunction] = + def sql_functions: PackratParser[Function] = aggregates | distance | date_diff | date_trunc | extractors | date_functions | datetime_functions | logical_functions | string_functions //private val regexIdentifier = """[\*a-zA-Z_\-][a-zA-Z0-9_\-\.\[\]\*]*""" @@ -657,9 +672,9 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => private val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex - def identifier: PackratParser[SQLIdentifier] = + def identifier: PackratParser[GenericIdentifier] = Distinct.regex.? ~ identifierRegex ^^ { case d ~ i => - SQLIdentifier( + GenericIdentifier( i, None, d.isDefined @@ -702,15 +717,13 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def sql_type: PackratParser[SQLType] = char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type - private[this] def castFunctionWithIdentifier: PackratParser[SQLIdentifier] = + private[this] def castFunctionWithIdentifier: PackratParser[GenericIdentifier] = "(?i)cast".r ~ start ~ (identifierWithTransformation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => - i.copy(functions = - a.toList ++ (SQLCast(i, targetType = t, as = as.isDefined) +: i.functions) - ) + i.copy(functions = a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) } - private[this] def dateFunctionWithIdentifier: PackratParser[SQLIdentifier] = + private[this] def dateFunctionWithIdentifier: PackratParser[GenericIdentifier] = (parse_date | format_date | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => af match { case Some(f) => t.identifier.copy(functions = f +: t +: t.identifier.functions) @@ -718,7 +731,7 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => } } - private[this] def dateTimeFunctionWithIdentifier: PackratParser[SQLIdentifier] = + private[this] def dateTimeFunctionWithIdentifier: PackratParser[GenericIdentifier] = (date_trunc | parse_datetime | format_datetime | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { case t ~ af => af match { @@ -727,21 +740,21 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => } } - private[this] def conditionalFunctionWithIdentifier: PackratParser[SQLIdentifier] = + private[this] def conditionalFunctionWithIdentifier: PackratParser[GenericIdentifier] = (is_null | is_notnull | coalesce | nullif) ^^ { t => t.identifier.copy(functions = t +: t.identifier.functions) } - def identifierWithTransformation: PackratParser[SQLIdentifier] = + def identifierWithTransformation: PackratParser[GenericIdentifier] = mathematicalFunctionWithIdentifier | castFunctionWithIdentifier | conditionalFunctionWithIdentifier | dateFunctionWithIdentifier | dateTimeFunctionWithIdentifier | stringFunctionWithIdentifier - def identifierWithAggregation: PackratParser[SQLIdentifier] = + def identifierWithAggregation: PackratParser[GenericIdentifier] = aggregates ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case a ~ _ ~ i ~ _ => i.copy(functions = a +: i.functions) } - def identifierWithFunction: PackratParser[SQLIdentifier] = + def identifierWithFunction: PackratParser[GenericIdentifier] = rep1sep( sql_functions, start @@ -751,9 +764,9 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => i match { case None => f.lastOption match { - case Some(fi: SQLFunctionWithIdentifier) => + case Some(fi: FunctionWithIdentifier) => fi.identifier.copy(functions = f ++ fi.identifier.functions) - case _ => SQLIdentifier("", functions = f) + case _ => GenericIdentifier("", functions = f) } case Some(id) => id.copy(functions = id.functions ++ f) } @@ -762,12 +775,12 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => private val regexAlias = """\b(?!(?i)as\b)\b(?!(?i)except\b)\b(?!(?i)where\b)\b(?!(?i)filter\b)\b(?!(?i)from\b)\b(?!(?i)group\b)\b(?!(?i)having\b)\b(?!(?i)order\b)\b(?!(?i)limit\b)[a-zA-Z0-9_]*""" - def alias: PackratParser[SQLAlias] = Alias.regex.? ~ regexAlias.r ^^ { case _ ~ b => SQLAlias(b) } + def alias: PackratParser[Alias] = Alias.regex.? ~ regexAlias.r ^^ { case _ ~ b => Alias(b) } def field: PackratParser[Field] = (identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | case_when_identifier | identifier) ~ alias.? ^^ { case i ~ a => - SQLField(i, a) + Field(i, a) } } @@ -775,17 +788,17 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => trait SQLSelectParser { self: SQLParser with SQLWhereParser => - def except: PackratParser[SQLExcept] = Except.regex ~ start ~ rep1sep(field, separator) ~ end ^^ { + def except: PackratParser[Except] = Except.regex ~ start ~ rep1sep(field, separator) ~ end ^^ { case _ ~ _ ~ e ~ _ => - SQLExcept(e) + Except(e) } - def select: PackratParser[SQLSelect] = + def select: PackratParser[Select] = Select.regex ~ rep1sep( field, separator ) ~ except.? ^^ { case _ ~ fields ~ e => - SQLSelect(fields, e) + Select(fields, e) } } @@ -793,16 +806,16 @@ trait SQLSelectParser { trait SQLFromParser { self: SQLParser with SQLLimitParser => - def unnest: PackratParser[SQLTable] = + def unnest: PackratParser[Table] = Unnest.regex ~ start ~ identifier ~ limit.? ~ end ~ alias ^^ { case _ ~ _ ~ i ~ l ~ _ ~ a => - SQLTable(SQLUnnest(i, l), Some(a)) + Table(Unnest(i, l), Some(a)) } - def table: PackratParser[SQLTable] = identifier ~ alias.? ^^ { case i ~ a => SQLTable(i, a) } + def table: PackratParser[Table] = identifier ~ alias.? ^^ { case i ~ a => Table(i, a) } - def from: PackratParser[SQLFrom] = From.regex ~ rep1sep(unnest | table, separator) ^^ { + def from: PackratParser[From] = From.regex ~ rep1sep(unnest | table, separator) ^^ { case _ ~ tables => - SQLFrom(tables) + From(tables) } } @@ -810,128 +823,128 @@ trait SQLFromParser { trait SQLWhereParser { self: SQLParser with SQLGroupByParser with SQLOrderByParser => - def isNull: PackratParser[SQLCriteria] = identifier ~ IsNull.regex ^^ { case i ~ _ => - SQLIsNull(i) + def isNull: PackratParser[Criteria] = identifier ~ IsNull.regex ^^ { case i ~ _ => + IsNullExpr(i) } - def isNotNull: PackratParser[SQLCriteria] = identifier ~ IsNotNull.regex ^^ { case i ~ _ => - SQLIsNotNull(i) + def isNotNull: PackratParser[Criteria] = identifier ~ IsNotNull.regex ^^ { case i ~ _ => + IsNotNullExpr(i) } - private def eq: PackratParser[SQLComparisonOperator] = Eq.sql ^^ (_ => Eq) + private def eq: PackratParser[ComparisonOperator] = Eq.sql ^^ (_ => Eq) - private def ne: PackratParser[SQLComparisonOperator] = Ne.sql ^^ (_ => Ne) + private def ne: PackratParser[ComparisonOperator] = Ne.sql ^^ (_ => Ne) - private def diff: PackratParser[SQLComparisonOperator] = Diff.sql ^^ (_ => Diff) + private def diff: PackratParser[ComparisonOperator] = Diff.sql ^^ (_ => Diff) - private def any_identifier: PackratParser[SQLIdentifier] = + private def any_identifier: PackratParser[GenericIdentifier] = identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | date_diff_identifier | extract_identifier | identifier - private def equality: PackratParser[SQLExpression] = + private def equality: PackratParser[GenericExpression] = not.? ~ any_identifier ~ (eq | ne | diff) ~ (boolean | literal | double | pi | long | any_identifier) ^^ { - case n ~ i ~ o ~ v => SQLExpression(i, o, v, n) + case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) } - def like: PackratParser[SQLExpression] = + def like: PackratParser[GenericExpression] = any_identifier ~ not.? ~ Like.regex ~ literal ^^ { case i ~ n ~ _ ~ v => - SQLExpression(i, Like, v, n) + GenericExpression(i, Like, v, n) } - private def ge: PackratParser[SQLComparisonOperator] = Ge.sql ^^ (_ => Ge) + private def ge: PackratParser[ComparisonOperator] = Ge.sql ^^ (_ => Ge) - def gt: PackratParser[SQLComparisonOperator] = Gt.sql ^^ (_ => Gt) + def gt: PackratParser[ComparisonOperator] = Gt.sql ^^ (_ => Gt) - private def le: PackratParser[SQLComparisonOperator] = Le.sql ^^ (_ => Le) + private def le: PackratParser[ComparisonOperator] = Le.sql ^^ (_ => Le) - def lt: PackratParser[SQLComparisonOperator] = Lt.sql ^^ (_ => Lt) + def lt: PackratParser[ComparisonOperator] = Lt.sql ^^ (_ => Lt) - private def comparison: PackratParser[SQLExpression] = + private def comparison: PackratParser[GenericExpression] = not.? ~ any_identifier ~ (ge | gt | le | lt) ~ (double | pi | long | literal | any_identifier) ^^ { - case n ~ i ~ o ~ v => SQLExpression(i, o, v, n) + case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) } - def in: PackratParser[SQLExpressionOperator] = In.regex ^^ (_ => In) + def in: PackratParser[ExpressionOperator] = In.regex ^^ (_ => In) - private def inLiteral: PackratParser[SQLCriteria] = + private def inLiteral: PackratParser[Criteria] = any_identifier ~ not.? ~ in ~ start ~ rep1sep(literal, separator) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => - SQLIn( + InExpr( i, - SQLStringValues(v), + StringValues(v), n ) } - private def inDoubles: PackratParser[SQLCriteria] = + private def inDoubles: PackratParser[Criteria] = any_identifier ~ not.? ~ in ~ start ~ rep1sep( double, separator ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => - SQLIn( + InExpr( i, - SQLDoubleValues(v), + DoubleValues(v), n ) } - private def inLongs: PackratParser[SQLCriteria] = + private def inLongs: PackratParser[Criteria] = any_identifier ~ not.? ~ in ~ start ~ rep1sep( long, separator ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => - SQLIn( + InExpr( i, - SQLLongValues(v), + LongValues(v), n ) } - def between: PackratParser[SQLCriteria] = + def between: PackratParser[Criteria] = any_identifier ~ not.? ~ Between.regex ~ literal ~ and ~ literal ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, SQLLiteralFromTo(from, to), n) + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LiteralFromTo(from, to), n) } - def betweenLongs: PackratParser[SQLCriteria] = + def betweenLongs: PackratParser[Criteria] = any_identifier ~ not.? ~ Between.regex ~ long ~ and ~ long ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, SQLLongFromTo(from, to), n) + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LongFromTo(from, to), n) } - def betweenDoubles: PackratParser[SQLCriteria] = + def betweenDoubles: PackratParser[Criteria] = any_identifier ~ not.? ~ Between.regex ~ double ~ and ~ double ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => SQLBetween(i, SQLDoubleFromTo(from, to), n) + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) } - def sql_distance: PackratParser[SQLCriteria] = + def sql_distance: PackratParser[Criteria] = distance ~ start ~ identifier ~ separator ~ start ~ double ~ separator ~ double ~ end ~ end ~ le ~ literal ^^ { case _ ~ _ ~ i ~ _ ~ _ ~ lat ~ _ ~ lon ~ _ ~ _ ~ _ ~ d => ElasticGeoDistance(i, d, lat, lon) } - def matchCriteria: PackratParser[SQLMatch] = + def matchCriteria: PackratParser[MatchCriteria] = Match.regex ~ start ~ rep1sep( any_identifier, separator ) ~ end ~ Against.regex ~ start ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ _ ~ _ ~ l ~ _ => - SQLMatch(i, l) + MatchCriteria(i, l) } - def and: PackratParser[SQLPredicateOperator] = And.regex ^^ (_ => And) + def and: PackratParser[PredicateOperator] = And.regex ^^ (_ => And) - def or: PackratParser[SQLPredicateOperator] = Or.regex ^^ (_ => Or) + def or: PackratParser[PredicateOperator] = Or.regex ^^ (_ => Or) def not: PackratParser[Not.type] = Not.regex ^^ (_ => Not) - def logical_criteria: PackratParser[SQLCriteria] = - (is_null | is_notnull) ^^ { case SQLConditionalFunctionAsCriteria(c) => + def logical_criteria: PackratParser[Criteria] = + (is_null | is_notnull) ^^ { case ConditionalFunctionAsCriteria(c) => c } - def criteria: PackratParser[SQLCriteria] = + def criteria: PackratParser[Criteria] = (equality | like | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ sql_distance | matchCriteria | logical_criteria) ^^ ( c => c ) - def predicate: PackratParser[SQLPredicate] = criteria ~ (and | or) ~ not.? ~ criteria ^^ { - case l ~ o ~ n ~ r => SQLPredicate(l, o, r, n) + def predicate: PackratParser[Predicate] = criteria ~ (and | or) ~ not.? ~ criteria ^^ { + case l ~ o ~ n ~ r => Predicate(l, o, r, n) } def nestedCriteria: PackratParser[ElasticRelation] = @@ -960,19 +973,19 @@ trait SQLWhereParser { case _ ~ _ ~ p ~ _ => ElasticParent(p) } - private def allPredicate: PackratParser[SQLCriteria] = + private def allPredicate: PackratParser[Criteria] = nestedPredicate | childPredicate | parentPredicate | predicate - private def allCriteria: PackratParser[SQLToken] = + private def allCriteria: PackratParser[Token] = nestedCriteria | childCriteria | parentCriteria | criteria - def whereCriteria: PackratParser[List[SQLToken]] = rep1( + def whereCriteria: PackratParser[List[Token]] = rep1( allPredicate | allCriteria | start | or | and | end | then_case ) - def where: PackratParser[SQLWhere] = + def where: PackratParser[Where] = Where.regex ~ whereCriteria ^^ { case _ ~ rawTokens => - SQLWhere(processTokens(rawTokens)) + Where(processTokens(rawTokens)) } import scala.annotation.tailrec @@ -1011,43 +1024,43 @@ trait SQLWhereParser { */ @tailrec private def processTokensHelper( - tokens: List[SQLToken], - stack: List[SQLToken] - ): Option[SQLCriteria] = { + tokens: List[Token], + stack: List[Token] + ): Option[Criteria] = { tokens match { case Nil => stack match { - case (right: SQLCriteria) :: (op: SQLPredicateOperator) :: (left: SQLCriteria) :: Nil => + case (right: Criteria) :: (op: PredicateOperator) :: (left: Criteria) :: Nil => Option( - SQLPredicate(left, op, right) + Predicate(left, op, right) ) case _ => - stack.headOption.collect { case c: SQLCriteria => c } + stack.headOption.collect { case c: Criteria => c } } case (_: StartDelimiter) :: rest => val (subTokens, remainingTokens) = extractSubTokens(rest, 1) val subCriteria = processSubTokens(subTokens) match { - case p: SQLPredicate => p.copy(group = true) - case c => c + case p: Predicate => p.copy(group = true) + case c => c } processTokensHelper(remainingTokens, subCriteria :: stack) - case (c: SQLCriteria) :: rest => + case (c: Criteria) :: rest => stack match { - case (op: SQLPredicateOperator) :: (left: SQLCriteria) :: tail => - val predicate = SQLPredicate(left, op, c) + case (op: PredicateOperator) :: (left: Criteria) :: tail => + val predicate = Predicate(left, op, c) processTokensHelper(rest, predicate :: tail) case _ => processTokensHelper(rest, c :: stack) } - case (op: SQLPredicateOperator) :: rest => + case (op: PredicateOperator) :: rest => stack match { - case (right: SQLCriteria) :: (left: SQLCriteria) :: tail => - val predicate = SQLPredicate(left, op, right) + case (right: Criteria) :: (left: Criteria) :: tail => + val predicate = Predicate(left, op, right) processTokensHelper(rest, predicate :: tail) - case (right: SQLCriteria) :: (o: SQLPredicateOperator) :: tail => + case (right: Criteria) :: (o: PredicateOperator) :: tail => tail match { - case (left: SQLCriteria) :: tt => - val predicate = SQLPredicate(left, op, right) + case (left: Criteria) :: tt => + val predicate = Predicate(left, op, right) processTokensHelper(rest, o :: predicate :: tt) case _ => processTokensHelper(rest, op :: stack) @@ -1055,7 +1068,7 @@ trait SQLWhereParser { case _ :: Nil => processTokensHelper(rest, op :: stack) case _ => - throw SQLValidationError("Invalid stack state for predicate creation") + throw ValidationError("Invalid stack state for predicate creation") } case ThenCase :: _ => processTokensHelper(Nil, stack) // exit processing on THEN @@ -1073,8 +1086,8 @@ trait SQLWhereParser { * @return */ protected def processTokens( - tokens: List[SQLToken] - ): Option[SQLCriteria] = { + tokens: List[Token] + ): Option[Criteria] = { processTokensHelper(tokens, Nil) } @@ -1086,9 +1099,9 @@ trait SQLWhereParser { * - list of SQL tokens * @return */ - private def processSubTokens(tokens: List[SQLToken]): SQLCriteria = { + private def processSubTokens(tokens: List[Token]): Criteria = { processTokensHelper(tokens, Nil).getOrElse( - throw SQLValidationError("Empty sub-expression") + throw ValidationError("Empty sub-expression") ) } @@ -1106,12 +1119,12 @@ trait SQLWhereParser { */ @tailrec private def extractSubTokens( - tokens: List[SQLToken], + tokens: List[Token], openCount: Int, - subTokens: List[SQLToken] = Nil - ): (List[SQLToken], List[SQLToken]) = { + subTokens: List[Token] = Nil + ): (List[Token], List[Token]) = { tokens match { - case Nil => throw SQLValidationError("Unbalanced parentheses") + case Nil => throw ValidationError("Unbalanced parentheses") case (start: StartDelimiter) :: rest => extractSubTokens(rest, openCount + 1, start :: subTokens) case (end: EndDelimiter) :: rest => @@ -1126,13 +1139,13 @@ trait SQLWhereParser { trait SQLGroupByParser { self: SQLParser with SQLWhereParser => - def bucket: PackratParser[SQLBucket] = identifier ^^ { i => - SQLBucket(i) + def bucket: PackratParser[Bucket] = identifier ^^ { i => + Bucket(i) } - def groupBy: PackratParser[SQLGroupBy] = + def groupBy: PackratParser[GroupBy] = GroupBy.regex ~ rep1sep(bucket, separator) ^^ { case _ ~ buckets => - SQLGroupBy(buckets) + GroupBy(buckets) } } @@ -1140,8 +1153,8 @@ trait SQLGroupByParser { trait SQLHavingParser { self: SQLParser with SQLWhereParser => - def having: PackratParser[SQLHaving] = Having.regex ~> whereCriteria ^^ { rawTokens => - SQLHaving( + def having: PackratParser[Having] = Having.regex ~> whereCriteria ^^ { rawTokens => + Having( processTokens(rawTokens) ) } @@ -1158,22 +1171,21 @@ trait SQLOrderByParser { private def fieldName: PackratParser[String] = """\b(?!(?i)limit\b)[a-zA-Z_][a-zA-Z0-9_]*""".r ^^ (f => f) - def fieldWithFunction: PackratParser[(String, List[SQLFunction])] = + def fieldWithFunction: PackratParser[(String, List[Function])] = rep1sep(sql_functions, start) ~ start.? ~ fieldName ~ rep1(end) ^^ { case f ~ _ ~ n ~ _ => (n, f) } - def sort: PackratParser[SQLFieldSort] = + def sort: PackratParser[FieldSort] = (fieldWithFunction | fieldName) ~ (asc | desc).? ^^ { case f ~ o => f match { - case i: (String, List[SQLFunction]) => SQLFieldSort(i._1, o, i._2) - case s: String => SQLFieldSort(s, o, List.empty) + case i: (String, List[Function]) => FieldSort(i._1, o, i._2) + case s: String => FieldSort(s, o, List.empty) } } - def orderBy: PackratParser[SQLOrderBy] = OrderBy.regex ~ rep1sep(sort, separator) ^^ { - case _ ~ s => - SQLOrderBy(s) + def orderBy: PackratParser[OrderBy] = OrderBy.regex ~ rep1sep(sort, separator) ^^ { case _ ~ s => + OrderBy(s) } } @@ -1181,8 +1193,8 @@ trait SQLOrderByParser { trait SQLLimitParser { self: SQLParser => - def limit: PackratParser[SQLLimit] = Limit.regex ~ long ^^ { case _ ~ i => - SQLLimit(i.value.toInt) + def limit: PackratParser[Limit] = Limit.regex ~ long ^^ { case _ ~ i => + Limit(i.value.toInt) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala new file mode 100644 index 00000000..a92d9ca8 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -0,0 +1,108 @@ +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.`type`._ + +import scala.util.matching.Regex + +package object time { + + sealed trait TimeUnit extends PainlessScript with MathScript { + lazy val regex: Regex = s"\\b(?i)$sql(s)?\\b".r + + override def painless: String = s"ChronoUnit.${sql.toUpperCase()}S" + + override def nullable: Boolean = false + } + + sealed trait CalendarUnit extends TimeUnit + sealed trait FixedUnit extends TimeUnit + + object TimeUnit { + case object Year extends Expr("year") with CalendarUnit { + override def script: String = "y" + } + case object Month extends Expr("month") with CalendarUnit { + override def script: String = "M" + } + case object Quarter extends Expr("quarter") with CalendarUnit { + override def script: String = throw new IllegalArgumentException( + "Quarter must be converted to months (value * 3) before creating date-math" + ) + } + case object Week extends Expr("week") with CalendarUnit { + override def script: String = "w" + } + + case object Day extends Expr("day") with CalendarUnit with FixedUnit { + override def script: String = "d" + } + + case object Hour extends Expr("hour") with FixedUnit { + override def script: String = "H" + } + case object Minute extends Expr("minute") with FixedUnit { + override def script: String = "m" + } + case object Second extends Expr("second") with FixedUnit { + override def script: String = "s" + } + + } + + case object Interval extends Expr("interval") with TokenRegex + + sealed trait TimeInterval extends PainlessScript with MathScript { + def value: Int + def unit: TimeUnit + override def sql: String = s"$Interval $value ${unit.sql}" + + override def painless: String = s"$value, ${unit.painless}" + + override def script: String = TimeInterval.script(this) + + def checkType(in: SQLType): Either[String, SQLType] = { + import TimeUnit._ + in match { + case SQLTypes.Date => + unit match { + case Year | Month | Day => Right(SQLTypes.Date) + case Hour | Minute | Second => Right(SQLTypes.Timestamp) + case _ => Left(s"Invalid interval unit $unit for DATE") + } + case SQLTypes.Time => + unit match { + case Hour | Minute | Second => Right(SQLTypes.Time) + case _ => Left(s"Invalid interval unit $unit for TIME") + } + case SQLTypes.DateTime => + Right(SQLTypes.Timestamp) + case SQLTypes.Timestamp => + Right(SQLTypes.Timestamp) + case SQLTypes.Temporal => + Right(SQLTypes.Timestamp) + case _ => + Left(s"Intervals not supported for type $in") + } + } + + override def nullable: Boolean = false + } + + import TimeUnit._ + + case class CalendarInterval(value: Int, unit: CalendarUnit) extends TimeInterval + case class FixedInterval(value: Int, unit: FixedUnit) extends TimeInterval + + object TimeInterval { + def apply(value: Int, unit: TimeUnit): TimeInterval = unit match { + case cu: CalendarUnit => CalendarInterval(value, cu) + case fu: FixedUnit => FixedInterval(value, fu) + } + def script(interval: TimeInterval): String = interval match { + case CalendarInterval(v, Quarter) => s"${v * 3}M" + case CalendarInterval(v, u) => s"$v${u.script}" + case FixedInterval(v, u) => s"$v${u.script}" + } + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala similarity index 94% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLType.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index b0aea9da..2c91a959 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -1,4 +1,4 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.`type` sealed trait SQLType { def typeId: String } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala similarity index 96% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLTypeUtils.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 738ac852..6571bcba 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -1,5 +1,7 @@ -package app.softnetwork.elastic.sql -import SQLTypes._ +package app.softnetwork.elastic.sql.`type` + +import app.softnetwork.elastic.sql.PainlessScript +import app.softnetwork.elastic.sql.`type`.SQLTypes._ object SQLTypeUtils { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala similarity index 97% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLTypes.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index d067b650..afb0c4b1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -1,4 +1,4 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.`type` object SQLTypes { case object Any extends SQLAny { val typeId = "any" } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala index d7d37b55..95a8ac2f 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala @@ -1,7 +1,11 @@ package app.softnetwork.elastic.sql import org.scalatest.funsuite.AnyFunSuite +import app.softnetwork.elastic.sql.function._ +import app.softnetwork.elastic.sql.function.time._ +import app.softnetwork.elastic.sql.time._ import TimeUnit._ +import app.softnetwork.elastic.sql.`type`.SQLType class SQLDateTimeFunctionSuite extends AnyFunSuite { @@ -9,17 +13,17 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { val baseDate = "doc['createdAt'].value" // Liste de toutes les fonctions transformables avec leurs types - val transformFunctions: Seq[SQLTransformFunction[_, _]] = Seq( - ParseDate(SQLIdentifier(""), "yyyy-MM-dd"), - ParseDateTime(SQLIdentifier(""), "yyyy-MM-dd HH:mm:ss"), - DateAdd(SQLIdentifier(""), TimeInterval(1, Day)), - DateSub(SQLIdentifier(""), TimeInterval(2, Month)), - DateTimeAdd(SQLIdentifier(""), TimeInterval(3, Hour)), - DateTimeSub(SQLIdentifier(""), TimeInterval(30, Minute)), - DateTrunc(SQLIdentifier(""), Day), + val transformFunctions: Seq[TransformFunction[_, _]] = Seq( + ParseDate(GenericIdentifier(""), "yyyy-MM-dd"), + ParseDateTime(GenericIdentifier(""), "yyyy-MM-dd HH:mm:ss"), + DateAdd(GenericIdentifier(""), TimeInterval(1, Day)), + DateSub(GenericIdentifier(""), TimeInterval(2, Month)), + DateTimeAdd(GenericIdentifier(""), TimeInterval(3, Hour)), + DateTimeSub(GenericIdentifier(""), TimeInterval(30, Minute)), + DateTrunc(GenericIdentifier(""), Day), Extract(Day), - FormatDate(SQLIdentifier(""), "yyyy-MM-dd"), - FormatDateTime(SQLIdentifier(""), "yyyy-MM-dd HH:mm:ss"), + FormatDate(GenericIdentifier(""), "yyyy-MM-dd"), + FormatDateTime(GenericIdentifier(""), "yyyy-MM-dd HH:mm:ss"), YEAR, MONTH, DAY, @@ -31,7 +35,7 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { // Fonction pour chaîner une séquence de transformations en vérifiant les types def chainTransformsTyped( base: String, - transforms: Seq[SQLTransformFunction[_, _]] + transforms: Seq[TransformFunction[_, _]] ): String = { require(transforms.nonEmpty, "No transforms provided") @@ -39,7 +43,7 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { (transforms.head.toPainless(base, 0), transforms.head.outputType.asInstanceOf[SQLType]) val (finalExpr, _) = transforms.tail.foldLeft(initial) { - case ((expr, currentType), t: SQLFunctionN[_, _]) => + case ((expr, currentType), t: FunctionN[_, _]) => if (!currentType.getClass.isAssignableFrom(t.inputType.getClass)) { throw new IllegalArgumentException( s"Type mismatch: expected ${currentType.getClass.getSimpleName}, got ${t.inputType.getClass.getSimpleName}" @@ -53,9 +57,9 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { // Générer dynamiquement tous les chaînages valides jusqu'à N fonctions def generateChains( - functions: Seq[SQLTransformFunction[_, _]], + functions: Seq[TransformFunction[_, _]], maxLength: Int - ): Seq[Seq[SQLTransformFunction[_, _]]] = { + ): Seq[Seq[TransformFunction[_, _]]] = { if (maxLength <= 1) functions.map(Seq(_)) else { val shorter = generateChains(functions, maxLength - 1) @@ -68,9 +72,9 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { } // Tester tous les chaînages pour N=2 et N=3 - val chains2: Seq[Seq[SQLTransformFunction[_, _]]] = + val chains2: Seq[Seq[TransformFunction[_, _]]] = generateChains(transformFunctions, 2) - val chains3: Seq[Seq[SQLTransformFunction[_, _]]] = + val chains3: Seq[Seq[TransformFunction[_, _]]] = generateChains(transformFunctions, 3) (chains2 ++ chains3).zipWithIndex.foreach { case (chain, idx) => diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 3137cfcc..cced14c5 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -1,5 +1,7 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.parser._ + import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLStringValueSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala similarity index 81% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLStringValueSpec.scala rename to sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala index 7f524305..9bbf49cb 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLStringValueSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala @@ -5,10 +5,10 @@ import org.scalatest.matchers.should.Matchers /** Created by smanciot on 17/02/17. */ -class SQLStringValueSpec extends AnyFlatSpec with Matchers { +class StringValueSpec extends AnyFlatSpec with Matchers { "SQLLiteral" should "perform sql like" in { - val l = SQLStringValue("%dummy%") + val l = StringValue("%dummy%") l.like(Seq("dummy")) should ===(true) l.like(Seq("aa dummy")) should ===(true) l.like(Seq("dummy bbb")) should ===(true) From ded5100c11084ac2b41ca11dcb9dc6a37a9bdc02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 21 Sep 2025 13:19:23 +0200 Subject: [PATCH 02/48] add package query --- .../elastic/client/ElasticClientApi.scala | 2 +- .../elastic/persistence/query/ElasticProvider.scala | 2 +- .../elastic/client/ElasticClientSpec.scala | 2 +- .../elastic/client/MockElasticClientApi.scala | 2 +- .../elastic/client/jest/JestClientApi.scala | 2 +- .../elastic/client/rest/RestHighLevelClientApi.scala | 2 +- .../elastic/sql/bridge/ElasticAggregation.scala | 2 +- .../elastic/sql/bridge/ElasticCriteria.scala | 2 +- .../elastic/sql/bridge/ElasticQuery.scala | 2 +- .../elastic/sql/bridge/ElasticSearchRequest.scala | 2 +- .../app/softnetwork/elastic/sql/bridge/package.scala | 2 +- .../{CriteriaSpec.scala => SQLCriteriaSpec.scala} | 3 ++- .../app/softnetwork/elastic/sql/SQLQuerySpec.scala | 1 + .../elastic/client/rest/RestHighLevelClientApi.scala | 2 +- .../elastic/client/java/ElasticsearchClientApi.scala | 2 +- .../elastic/client/java/ElasticsearchClientApi.scala | 2 +- .../elastic/sql/bridge/ElasticAggregation.scala | 2 +- .../elastic/sql/bridge/ElasticCriteria.scala | 2 +- .../elastic/sql/bridge/ElasticQuery.scala | 2 +- .../elastic/sql/bridge/ElasticSearchRequest.scala | 2 +- .../app/softnetwork/elastic/sql/bridge/package.scala | 1 + .../softnetwork/elastic/sql/SQLCriteriaSpec.scala | 3 ++- .../app/softnetwork/elastic/sql/SQLQuerySpec.scala | 1 + .../app/softnetwork/elastic/sql/SQLImplicits.scala | 1 + .../elastic/sql/function/cond/package.scala | 11 ++--------- .../softnetwork/elastic/sql/function/package.scala | 3 ++- .../sql/operator/math/ArithmeticExpression.scala | 1 + .../scala/app/softnetwork/elastic/sql/package.scala | 1 + .../elastic/sql/{ => parser}/Delimiter.scala | 4 +++- .../softnetwork/elastic/sql/parser/SQLParser.scala | 11 ++++++----- .../softnetwork/elastic/sql/{ => query}/From.scala | 12 +++++++++++- .../elastic/sql/{ => query}/GroupBy.scala | 5 +++-- .../softnetwork/elastic/sql/{ => query}/Having.scala | 4 +++- .../softnetwork/elastic/sql/{ => query}/Limit.scala | 4 +++- .../elastic/sql/{ => query}/OrderBy.scala | 3 ++- .../sql/{ => query}/SQLMultiSearchRequest.scala | 4 +++- .../elastic/sql/{ => query}/SQLQuery.scala | 4 ++-- .../elastic/sql/{ => query}/SQLSearchRequest.scala | 4 +++- .../softnetwork/elastic/sql/{ => query}/Select.scala | 12 +++++++++++- .../elastic/sql/{ => query}/Validator.scala | 2 +- .../softnetwork/elastic/sql/{ => query}/Where.scala | 3 ++- 41 files changed, 85 insertions(+), 49 deletions(-) rename es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/{CriteriaSpec.scala => SQLCriteriaSpec.scala} (99%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => parser}/Delimiter.scala (85%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/From.scala (89%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/GroupBy.scala (96%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/Having.scala (80%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/Limit.scala (54%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/OrderBy.scala (86%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/SQLMultiSearchRequest.scala (81%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/SQLQuery.scala (71%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/SQLSearchRequest.scala (96%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/Select.scala (92%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/Validator.scala (96%) rename sql/src/main/scala/app/softnetwork/elastic/sql/{ => query}/Where.scala (99%) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index f6889bd0..418eca87 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -8,7 +8,7 @@ import _root_.akka.stream.{FlowShape, Materializer} import akka.stream.scaladsl._ import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization._ -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import com.google.gson.JsonParser import com.typesafe.config.{Config, ConfigFactory} import org.json4s.{DefaultFormats, Formats} diff --git a/core/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala b/core/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala index 557eab6e..bf45187b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala +++ b/core/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.persistence.query import app.softnetwork.elastic.client.ElasticClientApi -import app.softnetwork.elastic.sql.SQLQuery +import app.softnetwork.elastic.sql.query.SQLQuery import mustache.Mustache import org.json4s.Formats import app.softnetwork.persistence._ diff --git a/core/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/core/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 374cf575..8e744258 100644 --- a/core/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/core/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -4,7 +4,7 @@ import akka.actor.ActorSystem import app.softnetwork.elastic.model.{Binary, Child, Parent, Sample} import app.softnetwork.elastic.persistence.query.ElasticProvider import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -import app.softnetwork.elastic.sql.SQLQuery +import app.softnetwork.elastic.sql.query.SQLQuery import app.softnetwork.persistence._ import app.softnetwork.persistence.person.model.Person import com.fasterxml.jackson.core.JsonParseException diff --git a/core/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/core/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index e31da4fe..5a8fee1a 100644 --- a/core/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/core/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Flow -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import org.json4s.Formats import app.softnetwork.persistence.model.Timestamped import org.slf4j.{Logger, LoggerFactory} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 653c1e4b..13c0b393 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -5,7 +5,7 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import app.softnetwork.elastic.client._ import app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization._ diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 510040f6..f9392443 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -4,7 +4,7 @@ import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import app.softnetwork.elastic.client._ -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.{client, sql} import app.softnetwork.persistence.model.Timestamped diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 3ecac205..57627ab6 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{ +import app.softnetwork.elastic.sql.query.{ Asc, Bucket, BucketSelectorScript, diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 33428bbe..f117d0a9 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.Criteria +import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 1792cdac..8832b0b0 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{ +import app.softnetwork.elastic.sql.query.{ BetweenExpr, ElasticBoolQuery, ElasticChild, diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 60177e0d..569d392d 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{Bucket, Criteria, Except, Field} +import app.softnetwork.elastic.sql.query.{Bucket, Criteria, Except, Field} import com.sksamuel.elastic4s.searches.SearchRequest import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index e2683871..89723883 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.function.aggregate.Count import app.softnetwork.elastic.sql.operator._ - +import app.softnetwork.elastic.sql.query._ import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/CriteriaSpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala similarity index 99% rename from es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/CriteriaSpec.scala rename to es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 3e959641..bb3865f3 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/CriteriaSpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -1,6 +1,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.ElasticApi.matchAllQuery import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn import com.sksamuel.elastic4s.searches.SearchRequest @@ -9,7 +10,7 @@ import org.scalatest.matchers.should.Matchers /** Created by smanciot on 13/04/17. */ -class CriteriaSpec extends AnyFlatSpec with Matchers { +class SQLCriteriaSpec extends AnyFlatSpec with Matchers { import Queries._ diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index a3bcaac2..ad39b30c 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.Queries._ +import app.softnetwork.elastic.sql.query.SQLQuery import com.google.gson.{JsonArray, JsonObject, JsonParser, JsonPrimitive} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 82889d09..9ed055dc 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -5,7 +5,7 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import app.softnetwork.elastic.client._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.{client, sql} import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization.serialization diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 52e1607f..4288b8ec 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -5,7 +5,7 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import app.softnetwork.elastic.client._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.{client, sql} import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization.serialization diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 96cf2c3c..04bdd5df 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -5,7 +5,7 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import app.softnetwork.elastic.client._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.{client, sql} import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization.serialization diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 2835af2b..96a805e5 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{ +import app.softnetwork.elastic.sql.query.{ Asc, BucketSelectorScript, ElasticBoolQuery, diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 6abfb525..b5fd1acf 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.Criteria +import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.requests.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index b485973d..20d556d3 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{ +import app.softnetwork.elastic.sql.query.{ ElasticBoolQuery, ElasticChild, ElasticFilter, diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index fac950c9..dc5e6cc4 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{Bucket, Criteria, Except, Field} +import app.softnetwork.elastic.sql.query.{Bucket, Criteria, Except, Field} import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequest} case class ElasticSearchRequest( diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index a04d34cf..e1d3803c 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.function.aggregate.Count import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.query._ import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index d8c85e3c..a25955d0 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -1,6 +1,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.ElasticApi.matchAllQuery import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequest} import org.scalatest.flatspec.AnyFlatSpec @@ -18,7 +19,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { import SQLImplicits._ val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( - SQLSearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) + SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) ).string println(result) result diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 24551fa0..4f82720a 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.Queries._ +import app.softnetwork.elastic.sql.query._ import com.google.gson.{JsonArray, JsonObject, JsonParser, JsonPrimitive} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala index 272eb37b..b55ea7f0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala @@ -1,6 +1,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.parser.SQLParser +import app.softnetwork.elastic.sql.query.{Criteria, SQLMultiSearchRequest, SQLSearchRequest} import scala.util.matching.Regex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index efacba5d..6446372d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -1,15 +1,8 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{ - Expr, - Expression, - GenericIdentifier, - Identifier, - PainlessScript, - TokenRegex -} -import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, Identifier, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLBool, SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.query.Expression package object cond { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index b9b767b1..81f73625 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -3,6 +3,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression +import app.softnetwork.elastic.sql.query.Validator package object function { @@ -103,7 +104,7 @@ package object function { def arithmetic: Boolean = functions.nonEmpty && functions.forall { case _: ArithmeticExpression => true - case _ => false + case _ => false } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index b7398c40..96a89f92 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -3,6 +3,7 @@ package app.softnetwork.elastic.sql.operator.math import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} +import app.softnetwork.elastic.sql.query.Validator case class ArithmeticExpression( left: PainlessScript, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 9e4ed209..fd2a2ccb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic import app.softnetwork.elastic.sql.function.aggregate.{Max, Min} import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.query._ import java.security.MessageDigest import java.util.regex.Pattern diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Delimiter.scala similarity index 85% rename from sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/parser/Delimiter.scala index f3195f83..f8ffc49f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/Delimiter.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Delimiter.scala @@ -1,4 +1,6 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.{Expr, Token} sealed trait Delimiter extends Token diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala index 68860ba1..c988c886 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala @@ -14,6 +14,7 @@ import app.softnetwork.elastic.sql.operator.math._ import app.softnetwork.elastic.sql.time.TimeUnit._ import app.softnetwork.elastic.sql.time._ import app.softnetwork.elastic.sql._ +import app.softnetwork.elastic.sql.query._ import scala.language.implicitConversions import scala.util.parsing.combinator.{PackratParsers, RegexParsers} @@ -185,11 +186,11 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => def identifierWithArithmeticExpression: Parser[GenericIdentifier] = arithmeticExpressionLevel2 ^^ { - case af: ArithmeticExpression => GenericIdentifier("", functions = af :: Nil) - case id: GenericIdentifier => id - case f: FunctionWithIdentifier => f.identifier - case f: Function => GenericIdentifier("", functions = f :: Nil) - case other => throw new Exception(s"Unexpected expression $other") + case af: ArithmeticExpression => GenericIdentifier("", functions = af :: Nil) + case id: GenericIdentifier => id + case f: FunctionWithIdentifier => f.identifier + case f: Function => GenericIdentifier("", functions = f :: Nil) + case other => throw new Exception(s"Unexpected expression $other") } def interval: PackratParser[TimeInterval] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala similarity index 89% rename from sql/src/main/scala/app/softnetwork/elastic/sql/From.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 3b4efdd6..a4f197b1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -1,4 +1,14 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{ + asString, + Alias, + Expr, + GenericIdentifier, + Source, + TokenRegex, + Updateable +} case object From extends Expr("from") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala similarity index 96% rename from sql/src/main/scala/app/softnetwork/elastic/sql/GroupBy.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index ee73dc88..ec23faa5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -1,7 +1,8 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, TokenRegex, Updateable} case object GroupBy extends Expr("group by") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala similarity index 80% rename from sql/src/main/scala/app/softnetwork/elastic/sql/Having.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 0c7bda43..6459d8d2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -1,4 +1,6 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{Expr, TokenRegex, Updateable} case object Having extends Expr("having") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala similarity index 54% rename from sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala index 65ae276b..bf303cf1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/Limit.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala @@ -1,4 +1,6 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{Expr, TokenRegex} case object Limit extends Expr("limit") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala similarity index 86% rename from sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala index c41efa44..92ca6b91 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/OrderBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala @@ -1,6 +1,7 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.function.{Function, FunctionChain} +import app.softnetwork.elastic.sql.{Expr, Token, TokenRegex} case object OrderBy extends Expr("order by") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala similarity index 81% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala index 2d69b816..7b4d6ebb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala @@ -1,4 +1,6 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.Token case class SQLMultiSearchRequest(requests: Seq[SQLSearchRequest]) extends Token { override def sql: String = s"${requests.map(_.sql).mkString(" union ")}" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala similarity index 71% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala index 9d86903e..7fd44b24 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala @@ -1,7 +1,7 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query case class SQLQuery(query: String, score: Option[Double] = None) { - import SQLImplicits._ + import app.softnetwork.elastic.sql.SQLImplicits._ lazy val request: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { query } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala similarity index 96% rename from sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala index f3e3b75a..2f8a20b9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala @@ -1,4 +1,6 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{asString, GenericIdentifier, Token} case class SQLSearchRequest( select: Select = Select(), diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala similarity index 92% rename from sql/src/main/scala/app/softnetwork/elastic/sql/Select.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index b7564f80..29909be0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -1,6 +1,16 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.function.{Function, FunctionChain} +import app.softnetwork.elastic.sql.{ + asString, + Alias, + AliasUtils, + Expr, + GenericIdentifier, + PainlessScript, + TokenRegex, + Updateable +} case object Select extends Expr("select") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Validator.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Validator.scala similarity index 96% rename from sql/src/main/scala/app/softnetwork/elastic/sql/Validator.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/Validator.scala index 5064c068..1660ba22 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/Validator.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Validator.scala @@ -1,4 +1,4 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} import app.softnetwork.elastic.sql.function.{Function, FunctionN} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala similarity index 99% rename from sql/src/main/scala/app/softnetwork/elastic/sql/Where.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 627ebd6c..45bef723 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -1,4 +1,4 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function._ @@ -9,6 +9,7 @@ import app.softnetwork.elastic.sql.function.cond.{ } import app.softnetwork.elastic.sql.function.geo.Distance import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql._ import scala.annotation.tailrec From c40d69e82fc13bb5d308046eba5259d4f3e5b657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 21 Sep 2025 13:24:05 +0200 Subject: [PATCH 03/48] move validation to parser package --- .../scala/app/softnetwork/elastic/sql/function/package.scala | 2 +- .../elastic/sql/operator/math/ArithmeticExpression.scala | 2 +- sql/src/main/scala/app/softnetwork/elastic/sql/package.scala | 1 + .../softnetwork/elastic/sql/{query => parser}/Validator.scala | 2 +- .../main/scala/app/softnetwork/elastic/sql/query/Where.scala | 1 + 5 files changed, 5 insertions(+), 3 deletions(-) rename sql/src/main/scala/app/softnetwork/elastic/sql/{query => parser}/Validator.scala (96%) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 81f73625..267df3f5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression -import app.softnetwork.elastic.sql.query.Validator +import app.softnetwork.elastic.sql.parser.Validator package object function { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 96a89f92..9a571a91 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql.operator.math import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} -import app.softnetwork.elastic.sql.query.Validator +import app.softnetwork.elastic.sql.parser.Validator case class ArithmeticExpression( left: PainlessScript, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index fd2a2ccb..2f094829 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic import app.softnetwork.elastic.sql.function.aggregate.{Max, Min} import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.parser.Validation import app.softnetwork.elastic.sql.query._ import java.security.MessageDigest diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Validator.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala similarity index 96% rename from sql/src/main/scala/app/softnetwork/elastic/sql/query/Validator.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala index 1660ba22..947029e5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Validator.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala @@ -1,4 +1,4 @@ -package app.softnetwork.elastic.sql.query +package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} import app.softnetwork.elastic.sql.function.{Function, FunctionN} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 45bef723..5e67306c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -8,6 +8,7 @@ import app.softnetwork.elastic.sql.function.cond.{ IsNullFunction } import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql._ From d9d71dec5d2e96620d3aecabcec2409c3396e01a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 21 Sep 2025 21:09:09 +0200 Subject: [PATCH 04/48] add identifier object --- .../main/scala/app/softnetwork/elastic/sql/package.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 2f094829..39c7850c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -456,6 +456,14 @@ package object sql { } + object Identifier { + def apply(): Identifier = GenericIdentifier("") + def apply(function: Function): Identifier = GenericIdentifier("", functions = function :: Nil) + def apply(name: String): Identifier = GenericIdentifier(name) + def apply(name: String, function: Function): Identifier = + GenericIdentifier(name, functions = function :: Nil) + } + case class GenericIdentifier( name: String, tableAlias: Option[String] = None, From c691b9e01b94ac08a9c24dde95fe5b7c548e00c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Sep 2025 11:04:17 +0200 Subject: [PATCH 05/48] rename parsers --- .../elastic/sql/SQLImplicits.scala | 4 +- .../parser/{SQLParser.scala => Parser.scala} | 56 +++---- .../elastic/sql/SQLParserSpec.scala | 156 +++++++++--------- 3 files changed, 108 insertions(+), 108 deletions(-) rename sql/src/main/scala/app/softnetwork/elastic/sql/parser/{SQLParser.scala => Parser.scala} (97%) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala index b55ea7f0..377867e9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.parser.SQLParser +import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{Criteria, SQLMultiSearchRequest, SQLSearchRequest} import scala.util.matching.Regex @@ -21,7 +21,7 @@ object SQLImplicits { implicit def queryToSQLQuery( query: String ): Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { - SQLParser(query) match { + Parser(query) match { case Left(_) => None case Right(r) => Some(r) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala similarity index 97% rename from sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index c988c886..7f6183ac 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SQLParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -24,15 +24,15 @@ import scala.util.parsing.input.CharSequenceReader * * SQL Parser for ElasticSearch */ -object SQLParser - extends SQLParser - with SQLSelectParser - with SQLFromParser - with SQLWhereParser - with SQLGroupByParser - with SQLHavingParser - with SQLOrderByParser - with SQLLimitParser +object Parser + extends Parser + with SelectParser + with FromParser + with WhereParser + with GroupByParser + with HavingParser + with OrderByParser + with LimitParser with PackratParsers { def request: PackratParser[SQLSearchRequest] = { @@ -53,12 +53,12 @@ object SQLParser def apply( query: String - ): Either[SQLParserError, Either[SQLSearchRequest, SQLMultiSearchRequest]] = { + ): Either[ParserError, Either[SQLSearchRequest, SQLMultiSearchRequest]] = { val reader = new PackratReader(new CharSequenceReader(query)) parse(requests, reader) match { case NoSuccess(msg, _) => Console.err.println(msg) - Left(SQLParserError(msg)) + Left(ParserError(msg)) case Success(result, _) => result match { case x :: Nil => Right(Left(x)) @@ -69,11 +69,11 @@ object SQLParser } -trait SQLCompilationError +trait CompilationError -case class SQLParserError(msg: String) extends SQLCompilationError +case class ParserError(msg: String) extends CompilationError -trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => +trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def literal: PackratParser[StringValue] = """"[^"]*"|'[^']*'""".r ^^ (str => StringValue(str.substring(1, str.length - 1))) @@ -786,8 +786,8 @@ trait SQLParser extends RegexParsers with PackratParsers { _: SQLWhereParser => } -trait SQLSelectParser { - self: SQLParser with SQLWhereParser => +trait SelectParser { + self: Parser with WhereParser => def except: PackratParser[Except] = Except.regex ~ start ~ rep1sep(field, separator) ~ end ^^ { case _ ~ _ ~ e ~ _ => @@ -804,8 +804,8 @@ trait SQLSelectParser { } -trait SQLFromParser { - self: SQLParser with SQLLimitParser => +trait FromParser { + self: Parser with LimitParser => def unnest: PackratParser[Table] = Unnest.regex ~ start ~ identifier ~ limit.? ~ end ~ alias ^^ { case _ ~ _ ~ i ~ l ~ _ ~ a => @@ -821,8 +821,8 @@ trait SQLFromParser { } -trait SQLWhereParser { - self: SQLParser with SQLGroupByParser with SQLOrderByParser => +trait WhereParser { + self: Parser with GroupByParser with OrderByParser => def isNull: PackratParser[Criteria] = identifier ~ IsNull.regex ^^ { case i ~ _ => IsNullExpr(i) @@ -1137,8 +1137,8 @@ trait SQLWhereParser { } } -trait SQLGroupByParser { - self: SQLParser with SQLWhereParser => +trait GroupByParser { + self: Parser with WhereParser => def bucket: PackratParser[Bucket] = identifier ^^ { i => Bucket(i) @@ -1151,8 +1151,8 @@ trait SQLGroupByParser { } -trait SQLHavingParser { - self: SQLParser with SQLWhereParser => +trait HavingParser { + self: Parser with WhereParser => def having: PackratParser[Having] = Having.regex ~> whereCriteria ^^ { rawTokens => Having( @@ -1162,8 +1162,8 @@ trait SQLHavingParser { } -trait SQLOrderByParser { - self: SQLParser => +trait OrderByParser { + self: Parser => def asc: PackratParser[Asc.type] = Asc.regex ^^ (_ => Asc) @@ -1191,8 +1191,8 @@ trait SQLOrderByParser { } -trait SQLLimitParser { - self: SQLParser => +trait LimitParser { + self: Parser => def limit: PackratParser[Limit] = Limit.regex ~ long ^^ { case _ ~ i => Limit(i.value.toInt) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index cced14c5..88d65755 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -172,436 +172,436 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { import Queries._ "SQLParser" should "parse numerical eq" in { - val result = SQLParser(numericalEq) + val result = Parser(numericalEq) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalEq) } it should "parse numerical ne" in { - val result = SQLParser(numericalNe) + val result = Parser(numericalNe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalNe) } it should "parse numerical lt" in { - val result = SQLParser(numericalLt) + val result = Parser(numericalLt) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalLt) } it should "parse numerical le" in { - val result = SQLParser(numericalLe) + val result = Parser(numericalLe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalLe) } it should "parse numerical gt" in { - val result = SQLParser(numericalGt) + val result = Parser(numericalGt) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalGt) } it should "parse numerical ge" in { - val result = SQLParser(numericalGe) + val result = Parser(numericalGe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalGe) } it should "parse literal eq" in { - val result = SQLParser(literalEq) + val result = Parser(literalEq) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalEq) } it should "parse literal like" in { - val result = SQLParser(literalLike) + val result = Parser(literalLike) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalLike) } it should "parse literal not like" in { - val result = SQLParser(literalNotLike) + val result = Parser(literalNotLike) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalNotLike) } it should "parse literal ne" in { - val result = SQLParser(literalNe) + val result = Parser(literalNe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalNe) } it should "parse literal lt" in { - val result = SQLParser(literalLt) + val result = Parser(literalLt) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalLt) } it should "parse literal le" in { - val result = SQLParser(literalLe) + val result = Parser(literalLe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalLe) } it should "parse literal gt" in { - val result = SQLParser(literalGt) + val result = Parser(literalGt) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalGt) } it should "parse literal ge" in { - val result = SQLParser(literalGe) + val result = Parser(literalGe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalGe) } it should "parse boolean eq" in { - val result = SQLParser(boolEq) + val result = Parser(boolEq) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(boolEq) } it should "parse boolean ne" in { - val result = SQLParser(boolNe) + val result = Parser(boolNe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(boolNe) } it should "parse between" in { - val result = SQLParser(betweenExpression) + val result = Parser(betweenExpression) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(betweenExpression) } it should "parse and predicate" in { - val result = SQLParser(andPredicate) + val result = Parser(andPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(andPredicate) } it should "parse or predicate" in { - val result = SQLParser(orPredicate) + val result = Parser(orPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(orPredicate) } it should "parse left predicate with criteria" in { - val result = SQLParser(leftPredicate) + val result = Parser(leftPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(leftPredicate) } it should "parse right predicate with criteria" in { - val result = SQLParser(rightPredicate) + val result = Parser(rightPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(rightPredicate) } it should "parse multiple predicates" in { - val result = SQLParser(predicates) + val result = Parser(predicates) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(predicates) } it should "parse nested predicate" in { - val result = SQLParser(nestedPredicate) + val result = Parser(nestedPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(nestedPredicate) } it should "parse nested criteria" in { - val result = SQLParser(nestedCriteria) + val result = Parser(nestedCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(nestedCriteria) } it should "parse child predicate" in { - val result = SQLParser(childPredicate) + val result = Parser(childPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(childPredicate) } it should "parse child criteria" in { - val result = SQLParser(childCriteria) + val result = Parser(childCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(childCriteria) } it should "parse parent predicate" in { - val result = SQLParser(parentPredicate) + val result = Parser(parentPredicate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(parentPredicate) } it should "parse parent criteria" in { - val result = SQLParser(parentCriteria) + val result = Parser(parentCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(parentCriteria) } it should "parse in literal expression" in { - val result = SQLParser(inLiteralExpression) + val result = Parser(inLiteralExpression) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( inLiteralExpression ) } it should "parse in numerical expression with Int values" in { - val result = SQLParser(inNumericalExpressionWithIntValues) + val result = Parser(inNumericalExpressionWithIntValues) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( inNumericalExpressionWithIntValues ) } it should "parse in numerical expression with Double values" in { - val result = SQLParser(inNumericalExpressionWithDoubleValues) + val result = Parser(inNumericalExpressionWithDoubleValues) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( inNumericalExpressionWithDoubleValues ) } it should "parse not in literal expression" in { - val result = SQLParser(notInLiteralExpression) + val result = Parser(notInLiteralExpression) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( notInLiteralExpression ) } it should "parse not in numerical expression with Int values" in { - val result = SQLParser(notInNumericalExpressionWithIntValues) + val result = Parser(notInNumericalExpressionWithIntValues) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( notInNumericalExpressionWithIntValues ) } it should "parse not in numerical expression with Double values" in { - val result = SQLParser(notInNumericalExpressionWithDoubleValues) + val result = Parser(notInNumericalExpressionWithDoubleValues) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( notInNumericalExpressionWithDoubleValues ) } it should "parse nested with between" in { - val result = SQLParser(nestedWithBetween) + val result = Parser(nestedWithBetween) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(nestedWithBetween) } it should "parse count" in { - val result = SQLParser(count) + val result = Parser(count) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(count) } it should "parse distinct count" in { - val result = SQLParser(countDistinct) + val result = Parser(countDistinct) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(countDistinct) } it should "parse count with nested criteria" in { - val result = SQLParser(countNested) + val result = Parser(countNested) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(countNested) } it should "parse is null" in { - val result = SQLParser(isNull) + val result = Parser(isNull) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(isNull) } it should "parse is not null" in { - val result = SQLParser(isNotNull) + val result = Parser(isNotNull) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(isNotNull) } it should "parse geo distance criteria" in { - val result = SQLParser(geoDistanceCriteria) + val result = Parser(geoDistanceCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( geoDistanceCriteria ) } it should "parse except fields" in { - val result = SQLParser(except) + val result = Parser(except) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(except) } it should "parse match criteria" in { - val result = SQLParser(matchCriteria) + val result = Parser(matchCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(matchCriteria) } it should "parse group by" in { - val result = SQLParser(groupBy) + val result = Parser(groupBy) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(groupBy) } it should "parse order by" in { - val result = SQLParser(orderBy) + val result = Parser(orderBy) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(orderBy) } it should "parse limit" in { - val result = SQLParser(limit) + val result = Parser(limit) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(limit) } it should "parse group by with order by and limit" in { - val result = SQLParser(groupByWithOrderByAndLimit) + val result = Parser(groupByWithOrderByAndLimit) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( groupByWithOrderByAndLimit ) } it should "parse group by with having" in { - val result = SQLParser(groupByWithHaving) + val result = Parser(groupByWithHaving) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(groupByWithHaving) } it should "parse date time fields" in { - val result = SQLParser(dateTimeWithIntervalFields) + val result = Parser(dateTimeWithIntervalFields) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( dateTimeWithIntervalFields ) } it should "parse fields with interval" in { - val result = SQLParser(fieldsWithInterval) + val result = Parser(fieldsWithInterval) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(fieldsWithInterval) } it should "parse filter with date time and interval" in { - val result = SQLParser(filterWithDateTimeAndInterval) + val result = Parser(filterWithDateTimeAndInterval) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( filterWithDateTimeAndInterval ) } it should "parse filter with date and interval" in { - val result = SQLParser(filterWithDateAndInterval) + val result = Parser(filterWithDateAndInterval) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( filterWithDateAndInterval ) } it should "parse filter with time and interval" in { - val result = SQLParser(filterWithTimeAndInterval) + val result = Parser(filterWithTimeAndInterval) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( filterWithTimeAndInterval ) } it should "parse group by with having and date time functions" in { - val result = SQLParser(groupByWithHavingAndDateTimeFunctions) + val result = Parser(groupByWithHavingAndDateTimeFunctions) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( groupByWithHavingAndDateTimeFunctions ) } it should "parse parse_date function" in { - val result = SQLParser(parseDate) + val result = Parser(parseDate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( parseDate ) } it should "parse parse_date_time function" in { - val result = SQLParser(parseDateTime) + val result = Parser(parseDateTime) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( parseDateTime ) } it should "parse date_diff function" in { - val result = SQLParser(dateDiff) + val result = Parser(dateDiff) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( dateDiff ) } it should "parse date_diff function with aggregation" in { - val result = SQLParser(aggregationWithDateDiff) + val result = Parser(aggregationWithDateDiff) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( aggregationWithDateDiff ) } it should "parse format_date function" in { - val result = SQLParser(formatDate) + val result = Parser(formatDate) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( formatDate ) } it should "parse format_datetime function" in { - val result = SQLParser(formatDateTime) + val result = Parser(formatDateTime) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( formatDateTime ) } it should "parse date_add function" in { - val result = SQLParser(dateAdd) + val result = Parser(dateAdd) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( dateAdd ) } it should "parse date_sub function" in { - val result = SQLParser(dateSub) + val result = Parser(dateSub) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( dateSub ) } it should "parse datetime_add function" in { - val result = SQLParser(dateTimeAdd) + val result = Parser(dateTimeAdd) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( dateTimeAdd ) } it should "parse datetime_sub function" in { - val result = SQLParser(dateTimeSub) + val result = Parser(dateTimeSub) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( dateTimeSub ) } it should "parse isnull function" in { - val result = SQLParser(isnull) + val result = Parser(isnull) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( isnull ) } it should "parse isnotnull function" in { - val result = SQLParser(isnotnull) + val result = Parser(isnotnull) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( isnotnull ) } it should "parse isnull criteria" in { - val result = SQLParser(isNullCriteria) + val result = Parser(isNullCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( isNullCriteria ) } it should "parse isnotnull criteria" in { - val result = SQLParser(isNotNullCriteria) + val result = Parser(isNotNullCriteria) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( isNotNullCriteria ) } it should "parse coalesce function" in { - val result = SQLParser(coalesce) + val result = Parser(coalesce) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( coalesce ) } it should "parse nullif function" in { - val result = SQLParser(nullif) + val result = Parser(nullif) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( nullif ) } it should "parse cast function" in { - val result = SQLParser(cast) + val result = Parser(cast) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( cast ) } it should "parse all casts function" in { - val result = SQLParser(allCasts) + val result = Parser(allCasts) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( allCasts ) } it should "parse case when expression" in { - val result = SQLParser(caseWhen) + val result = Parser(caseWhen) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( caseWhen ) } it should "parse case when with expression" in { - val result = SQLParser(caseWhenExpr) + val result = Parser(caseWhenExpr) result.toOption .flatMap(_.left.toOption.map(_.sql)) .getOrElse("") @@ -609,28 +609,28 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { } it should "parse extract function" in { - val result = SQLParser(extract) + val result = Parser(extract) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( extract ) } it should "parse arithmetic expressions" in { - val result = SQLParser(arithmetic) + val result = Parser(arithmetic) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( arithmetic ) } it should "parse mathematical functions" in { - val result = SQLParser(mathematical) + val result = Parser(mathematical) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( mathematical ) } it should "parse string functions" in { - val result = SQLParser(string) + val result = Parser(string) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( string ) From 2269cca02a9f8c9802e66c9c6b8b137e5e972c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Sep 2025 12:17:13 +0200 Subject: [PATCH 06/48] updates related to identifier --- .../elastic/sql/bridge/package.scala | 2 +- .../elastic/sql/bridge/package.scala | 2 +- .../elastic/sql/function/cond/package.scala | 10 +++--- .../elastic/sql/function/math/package.scala | 4 +-- .../elastic/sql/function/package.scala | 2 +- .../elastic/sql/function/string/package.scala | 4 +-- .../elastic/sql/function/time/package.scala | 20 +++++------ .../app/softnetwork/elastic/sql/package.scala | 7 ++-- .../elastic/sql/parser/Parser.scala | 11 ++++--- .../softnetwork/elastic/sql/query/From.scala | 4 +-- .../elastic/sql/query/GroupBy.scala | 6 ++-- .../elastic/sql/query/SQLSearchRequest.scala | 4 +-- .../elastic/sql/query/Select.scala | 6 ++-- .../softnetwork/elastic/sql/query/Where.scala | 33 +++++++++---------- .../sql/SQLDateTimeFunctionSuite.scala | 18 +++++----- 15 files changed, 68 insertions(+), 65 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 89723883..b9b3dc4c 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -305,7 +305,7 @@ package object bridge { } case _ => matchAllQuery() } - case i: GenericIdentifier => + case i: Identifier => operator match { case op: ComparisonOperator => i.toScript match { diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index e1d3803c..ed6d917d 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -307,7 +307,7 @@ package object bridge { } case _ => matchAllQuery() } - case i: GenericIdentifier => + case i: Identifier => operator match { case op: ComparisonOperator => i.toScript match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 6446372d..59f0ed06 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, Identifier, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{Expr, Identifier, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLBool, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.query.Expression @@ -35,7 +35,7 @@ package object cond { override def toPainless(base: String, idx: Int): String = s"($base$painless)" } - case class IsNullFunction(identifier: GenericIdentifier) extends ConditionalFunction[SQLAny] { + case class IsNullFunction(identifier: Identifier) extends ConditionalFunction[SQLAny] { override def conditionalOp: ConditionalOp = IsNullFunction override def args: List[PainlessScript] = List(identifier) @@ -53,7 +53,7 @@ package object cond { } } - case class IsNotNullFunction(identifier: GenericIdentifier) extends ConditionalFunction[SQLAny] { + case class IsNotNullFunction(identifier: Identifier) extends ConditionalFunction[SQLAny] { override def conditionalOp: ConditionalOp = IsNotNullFunction override def args: List[PainlessScript] = List(identifier) @@ -82,7 +82,7 @@ package object cond { override def outputType: SQLType = SQLTypeUtils.leastCommonSuperType(args.map(_.out)) - override def identifier: GenericIdentifier = GenericIdentifier("") + override def identifier: Identifier = Identifier() override def inputType: SQLAny = SQLTypes.Any @@ -125,7 +125,7 @@ package object cond { override def args: List[PainlessScript] = List(expr1, expr2) - override def identifier: GenericIdentifier = GenericIdentifier("") + override def identifier: Identifier = Identifier() override def inputType: SQLAny = SQLTypes.Any diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 1d156766..2470f507 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, IntValue, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{Expr, Identifier, IntValue, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLTypes} package object math { @@ -45,7 +45,7 @@ package object math { override def fun: Option[PainlessScript] = Some(mathOp) - override def identifier: GenericIdentifier = GenericIdentifier("", functions = this :: Nil) + override def identifier: Identifier = Identifier(this) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 267df3f5..df688762 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -19,7 +19,7 @@ package object function { } trait FunctionWithIdentifier extends Function { - def identifier: GenericIdentifier //= SQLIdentifier("", functions = this :: Nil) + def identifier: Identifier } trait FunctionWithValue[+T] extends Function { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 6ebfa488..5cccc63c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, IntValue, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{Expr, Identifier, IntValue, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLType, SQLTypeUtils, SQLTypes, SQLVarchar} package object string { @@ -34,7 +34,7 @@ package object string { override def fun: Option[PainlessScript] = Some(stringOp) - override def identifier: GenericIdentifier = GenericIdentifier("", functions = this :: Nil) + override def identifier: Identifier = Identifier(this) override def toSQL(base: String): String = s"$sql($base)" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 1f5493ac..ce3d372a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, MathScript, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{Expr, Identifier, MathScript, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.operator.time._ import app.softnetwork.elastic.sql.`type`.{ SQLDate, @@ -129,7 +129,7 @@ package object time { override def painless: String = ".truncatedTo" } - case class DateTrunc(identifier: GenericIdentifier, unit: TimeUnit) + case class DateTrunc(identifier: Identifier, unit: TimeUnit) extends DateTimeFunction with TransformFunction[SQLTemporal, SQLTemporal] with FunctionWithIdentifier { @@ -216,7 +216,7 @@ package object time { case object DateAdd extends Expr("date_add") with TokenRegex - case class DateAdd(identifier: GenericIdentifier, interval: TimeInterval) + case class DateAdd(identifier: Identifier, interval: TimeInterval) extends DateFunction with AddInterval[SQLDate] with TransformFunction[SQLDate, SQLDate] @@ -231,7 +231,7 @@ package object time { case object DateSub extends Expr("date_sub") with TokenRegex - case class DateSub(identifier: GenericIdentifier, interval: TimeInterval) + case class DateSub(identifier: Identifier, interval: TimeInterval) extends DateFunction with SubtractInterval[SQLDate] with TransformFunction[SQLDate, SQLDate] @@ -248,7 +248,7 @@ package object time { override def painless: String = ".parse" } - case class ParseDate(identifier: GenericIdentifier, format: String) + case class ParseDate(identifier: Identifier, format: String) extends DateFunction with TransformFunction[SQLVarchar, SQLDate] with FunctionWithIdentifier { @@ -276,7 +276,7 @@ package object time { override def painless: String = ".format" } - case class FormatDate(identifier: GenericIdentifier, format: String) + case class FormatDate(identifier: Identifier, format: String) extends DateFunction with TransformFunction[SQLDate, SQLVarchar] with FunctionWithIdentifier { @@ -302,7 +302,7 @@ package object time { case object DateTimeAdd extends Expr("datetime_add") with TokenRegex - case class DateTimeAdd(identifier: GenericIdentifier, interval: TimeInterval) + case class DateTimeAdd(identifier: Identifier, interval: TimeInterval) extends DateTimeFunction with AddInterval[SQLDateTime] with TransformFunction[SQLDateTime, SQLDateTime] @@ -317,7 +317,7 @@ package object time { case object DateTimeSub extends Expr("datetime_sub") with TokenRegex - case class DateTimeSub(identifier: GenericIdentifier, interval: TimeInterval) + case class DateTimeSub(identifier: Identifier, interval: TimeInterval) extends DateTimeFunction with SubtractInterval[SQLDateTime] with TransformFunction[SQLDateTime, SQLDateTime] @@ -334,7 +334,7 @@ package object time { override def painless: String = ".parse" } - case class ParseDateTime(identifier: GenericIdentifier, format: String) + case class ParseDateTime(identifier: Identifier, format: String) extends DateTimeFunction with TransformFunction[SQLVarchar, SQLDateTime] with FunctionWithIdentifier { @@ -362,7 +362,7 @@ package object time { override def painless: String = ".format" } - case class FormatDateTime(identifier: GenericIdentifier, format: String) + case class FormatDateTime(identifier: Identifier, format: String) extends DateTimeFunction with TransformFunction[SQLDateTime, SQLVarchar] with FunctionWithIdentifier { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 39c7850c..b43078b0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -375,12 +375,15 @@ package object sql { def update(request: SQLSearchRequest): Source } - trait Identifier extends Token with Source with FunctionChain with PainlessScript { + sealed trait Identifier extends Token with Source with FunctionChain with PainlessScript { def name: String + def update(request: SQLSearchRequest): Identifier + def tableAlias: Option[String] def distinct: Boolean def nested: Boolean + def limit: Option[Limit] def fieldAlias: Option[String] def bucket: Option[Bucket] override def sql: String = { @@ -475,7 +478,7 @@ package object sql { bucket: Option[Bucket] = None ) extends Identifier { - def update(request: SQLSearchRequest): GenericIdentifier = { + def update(request: SQLSearchRequest): Identifier = { val parts: Seq[String] = name.split("\\.").toSeq if (request.tableAliases.values.toSeq.contains(parts.head)) { request.unnests.find(_._1 == parts.head) match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 7f6183ac..bdcf63ad 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -17,6 +17,7 @@ import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.query._ import scala.language.implicitConversions +import scala.language.existentials import scala.util.parsing.combinator.{PackratParsers, RegexParsers} import scala.util.parsing.input.CharSequenceReader @@ -444,7 +445,7 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => implicit def functionAsIdentifier(mf: Function): GenericIdentifier = mf match { case id: GenericIdentifier => id case fid: FunctionWithIdentifier => fid.identifier - case _ => GenericIdentifier("", functions = mf :: Nil) + case _ => Identifier(mf) } def arithmeticFunction: PackratParser[MathematicalFunction] = @@ -498,7 +499,7 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def mathematicalFunction: PackratParser[MathematicalFunction] = arithmeticFunction | trigonometricFunction | roundFunction | powFunction | signFunction - def mathematicalFunctionWithIdentifier: PackratParser[GenericIdentifier] = + def mathematicalFunctionWithIdentifier: PackratParser[Identifier] = mathematicalFunction ^^ { mf => mf.identifier } @@ -514,7 +515,7 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) } - def stringFunctionWithIdentifier: PackratParser[GenericIdentifier] = + def stringFunctionWithIdentifier: PackratParser[Identifier] = (concatFunction | substringFunction) ^^ { sf => sf.identifier } @@ -746,7 +747,7 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => t.identifier.copy(functions = t +: t.identifier.functions) } - def identifierWithTransformation: PackratParser[GenericIdentifier] = + def identifierWithTransformation: PackratParser[Identifier] = mathematicalFunctionWithIdentifier | castFunctionWithIdentifier | conditionalFunctionWithIdentifier | dateFunctionWithIdentifier | dateTimeFunctionWithIdentifier | stringFunctionWithIdentifier def identifierWithAggregation: PackratParser[GenericIdentifier] = @@ -838,7 +839,7 @@ trait WhereParser { private def diff: PackratParser[ComparisonOperator] = Diff.sql ^^ (_ => Diff) - private def any_identifier: PackratParser[GenericIdentifier] = + private def any_identifier: PackratParser[Identifier] = identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | date_diff_identifier | extract_identifier | identifier private def equality: PackratParser[GenericExpression] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index a4f197b1..6424ec9d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -4,7 +4,7 @@ import app.softnetwork.elastic.sql.{ asString, Alias, Expr, - GenericIdentifier, + Identifier, Source, TokenRegex, Updateable @@ -14,7 +14,7 @@ case object From extends Expr("from") with TokenRegex case object Unnest extends Expr("unnest") with TokenRegex -case class Unnest(identifier: GenericIdentifier, limit: Option[Limit]) extends Source { +case class Unnest(identifier: Identifier, limit: Option[Limit]) extends Source { override def sql: String = s"$Unnest($identifier${asString(limit)})" def update(request: SQLSearchRequest): Unnest = this.copy(identifier = identifier.update(request)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index ec23faa5..7f9dc01b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.operator._ -import app.softnetwork.elastic.sql.{Expr, GenericIdentifier, TokenRegex, Updateable} +import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} case object GroupBy extends Expr("group by") with TokenRegex @@ -24,7 +24,7 @@ case class GroupBy(buckets: Seq[Bucket]) extends Updateable { } case class Bucket( - identifier: GenericIdentifier + identifier: Identifier ) extends Updateable { override def sql: String = s"$identifier" def update(request: SQLSearchRequest): Bucket = @@ -53,7 +53,7 @@ object BucketSelectorScript { case e: Expression if e.aggregation => import e._ maybeValue match { - case Some(v: GenericIdentifier) if v.aggregation => + case Some(v: Identifier) if v.aggregation => Map(identifier.aliasOrName -> identifier.aliasOrName, v.aliasOrName -> v.aliasOrName) case _ => Map(identifier.aliasOrName -> identifier.aliasOrName) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala index 2f8a20b9..c3595606 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.{asString, GenericIdentifier, Token} +import app.softnetwork.elastic.sql.{asString, Identifier, Token} case class SQLSearchRequest( select: Select = Select(), @@ -48,7 +48,7 @@ case class SQLSearchRequest( lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) - lazy val sources: Seq[String] = from.tables.collect { case Table(source: GenericIdentifier, _) => + lazy val sources: Seq[String] = from.tables.collect { case Table(source: Identifier, _) => source.sql } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 29909be0..81f51d8e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -6,7 +6,7 @@ import app.softnetwork.elastic.sql.{ Alias, AliasUtils, Expr, - GenericIdentifier, + Identifier, PainlessScript, TokenRegex, Updateable @@ -15,7 +15,7 @@ import app.softnetwork.elastic.sql.{ case object Select extends Expr("select") with TokenRegex case class Field( - identifier: GenericIdentifier, + identifier: Identifier, fieldAlias: Option[Alias] = None ) extends Updateable with FunctionChain @@ -63,7 +63,7 @@ case class Except(fields: Seq[Field]) extends Updateable { } case class Select( - fields: Seq[Field] = Seq(Field(identifier = GenericIdentifier("*"))), + fields: Seq[Field] = Seq(Field(identifier = Identifier("*"))), except: Option[Except] = None ) extends Updateable { override def sql: String = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 5e67306c..72e6608c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -185,7 +185,7 @@ case class ElasticBoolQuery( } sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // to fix output type as Boolean - def identifier: GenericIdentifier + def identifier: Identifier override def nested: Boolean = identifier.nested override def group: Boolean = false override lazy val limit: Option[Limit] = identifier.limit @@ -213,10 +213,10 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def painlessValue: String = maybeValue .map { - case v: Value[_] => v.painless - case v: Values[_, _] => v.painless - case v: GenericIdentifier => v.painless - case v => v.sql + case v: Value[_] => v.painless + case v: Values[_, _] => v.painless + case v: Identifier => v.painless + case v => v.sql } .getOrElse("") /*{ operator match { @@ -274,7 +274,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { } case class GenericExpression( - identifier: GenericIdentifier, + identifier: Identifier, operator: ExpressionOperator, value: Token, maybeNot: Option[Not.type] = None @@ -284,7 +284,7 @@ case class GenericExpression( override def update(request: SQLSearchRequest): Criteria = { val updated = value match { - case id: GenericIdentifier => + case id: Identifier => this.copy(identifier = identifier.update(request), value = id.update(request)) case _ => this.copy(identifier = identifier.update(request)) } @@ -297,7 +297,7 @@ case class GenericExpression( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class IsNullExpr(identifier: GenericIdentifier) extends Expression { +case class IsNullExpr(identifier: Identifier) extends Expression { override val operator: Operator = IsNull override def maybeValue: Option[Token] = None @@ -315,7 +315,7 @@ case class IsNullExpr(identifier: GenericIdentifier) extends Expression { override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class IsNotNullExpr(identifier: GenericIdentifier) extends Expression { +case class IsNotNullExpr(identifier: Identifier) extends Expression { override val operator: Operator = IsNotNull override def maybeValue: Option[Token] = None @@ -350,8 +350,7 @@ object ConditionalFunctionAsCriteria { } } -case class IsNullCriteria(identifier: GenericIdentifier) - extends CriteriaWithConditionalFunction[SQLAny] { +case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { override val conditionalFunction: ConditionalFunction[SQLAny] = IsNullFunction(identifier) override val operator: Operator = IsNull override def update(request: SQLSearchRequest): Criteria = { @@ -370,7 +369,7 @@ case class IsNullCriteria(identifier: GenericIdentifier) } -case class IsNotNullCriteria(identifier: GenericIdentifier) +case class IsNotNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { override val conditionalFunction: ConditionalFunction[SQLAny] = IsNotNullFunction( identifier @@ -394,7 +393,7 @@ case class IsNotNullCriteria(identifier: GenericIdentifier) } case class InExpr[R, +T <: Value[R]]( - identifier: GenericIdentifier, + identifier: Identifier, values: Values[R, T], maybeNot: Option[Not.type] = None ) extends Expression { this: InExpr[R, T] => @@ -421,7 +420,7 @@ case class InExpr[R, +T <: Value[R]]( } case class BetweenExpr[+T]( - identifier: GenericIdentifier, + identifier: Identifier, fromTo: FromTo[T], maybeNot: Option[Not.type] ) extends Expression { @@ -446,7 +445,7 @@ case class BetweenExpr[+T]( } case class ElasticGeoDistance( - identifier: GenericIdentifier, + identifier: Identifier, distance: StringValue, lat: DoubleValue, lon: DoubleValue @@ -465,7 +464,7 @@ case class ElasticGeoDistance( } case class MatchCriteria( - identifiers: Seq[GenericIdentifier], + identifiers: Seq[Identifier], value: StringValue ) extends Criteria { override def sql: String = @@ -505,7 +504,7 @@ case class MatchCriteria( } case class ElasticMatch( - identifier: GenericIdentifier, + identifier: Identifier, value: StringValue, options: Option[String] ) extends Expression { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala index 95a8ac2f..5ed67b1b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala @@ -14,16 +14,16 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { // Liste de toutes les fonctions transformables avec leurs types val transformFunctions: Seq[TransformFunction[_, _]] = Seq( - ParseDate(GenericIdentifier(""), "yyyy-MM-dd"), - ParseDateTime(GenericIdentifier(""), "yyyy-MM-dd HH:mm:ss"), - DateAdd(GenericIdentifier(""), TimeInterval(1, Day)), - DateSub(GenericIdentifier(""), TimeInterval(2, Month)), - DateTimeAdd(GenericIdentifier(""), TimeInterval(3, Hour)), - DateTimeSub(GenericIdentifier(""), TimeInterval(30, Minute)), - DateTrunc(GenericIdentifier(""), Day), + ParseDate(Identifier(), "yyyy-MM-dd"), + ParseDateTime(Identifier(), "yyyy-MM-dd HH:mm:ss"), + DateAdd(Identifier(), TimeInterval(1, Day)), + DateSub(Identifier(), TimeInterval(2, Month)), + DateTimeAdd(Identifier(), TimeInterval(3, Hour)), + DateTimeSub(Identifier(), TimeInterval(30, Minute)), + DateTrunc(Identifier(), Day), Extract(Day), - FormatDate(GenericIdentifier(""), "yyyy-MM-dd"), - FormatDateTime(GenericIdentifier(""), "yyyy-MM-dd HH:mm:ss"), + FormatDate(Identifier(), "yyyy-MM-dd"), + FormatDateTime(Identifier(), "yyyy-MM-dd HH:mm:ss"), YEAR, MONTH, DAY, From dc41be891d52f1ee2af8470e7fbcbc9151f63f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Sep 2025 13:04:26 +0200 Subject: [PATCH 07/48] finalize updates related to identifier --- build.sbt | 2 +- .../app/softnetwork/elastic/sql/package.scala | 5 ++ .../elastic/sql/parser/Parser.scala | 88 +++++++++---------- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/build.sbt b/build.sbt index b8cd33b4..413871ee 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.8.0" +ThisBuild / version := "0.9.0" ThisBuild / scalaVersion := scala213 diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index b43078b0..424ceb73 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -378,6 +378,8 @@ package object sql { sealed trait Identifier extends Token with Source with FunctionChain with PainlessScript { def name: String + def withFunctions(functions: List[Function]): Identifier + def update(request: SQLSearchRequest): Identifier def tableAlias: Option[String] @@ -462,6 +464,7 @@ package object sql { object Identifier { def apply(): Identifier = GenericIdentifier("") def apply(function: Function): Identifier = GenericIdentifier("", functions = function :: Nil) + def apply(functions: List[Function]): Identifier = apply().withFunctions(functions) def apply(name: String): Identifier = GenericIdentifier(name) def apply(name: String, function: Function): Identifier = GenericIdentifier(name, functions = function :: Nil) @@ -478,6 +481,8 @@ package object sql { bucket: Option[Bucket] = None ) extends Identifier { + def withFunctions(functions: List[Function]): Identifier = this.copy(functions = functions) + def update(request: SQLSearchRequest): Identifier = { val parts: Seq[String] = name.split("\\.").toSeq if (request.tableAliases.values.toSeq.contains(parts.head)) { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index bdcf63ad..3f616a6c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -91,9 +91,9 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def boolean: PackratParser[BooleanValue] = """(true|false)""".r ^^ (bool => BooleanValue(bool.toBoolean)) - def value_identifier: PackratParser[GenericIdentifier] = + def value_identifier: PackratParser[Identifier] = (literal | long | double | pi | boolean) ^^ { v => - GenericIdentifier("", functions = v :: Nil) + Identifier(v) } def start: PackratParser[Delimiter] = "(" ^^ (_ => StartPredicate) @@ -185,12 +185,12 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => } } - def identifierWithArithmeticExpression: Parser[GenericIdentifier] = + def identifierWithArithmeticExpression: Parser[Identifier] = arithmeticExpressionLevel2 ^^ { - case af: ArithmeticExpression => GenericIdentifier("", functions = af :: Nil) - case id: GenericIdentifier => id + case af: ArithmeticExpression => Identifier(af) + case id: Identifier => id case f: FunctionWithIdentifier => f.identifier - case f: Function => GenericIdentifier("", functions = f :: Nil) + case f: Function => Identifier(f) case other => throw new Exception(s"Unexpected expression $other") } @@ -212,17 +212,17 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def intervalFunction: PackratParser[TransformFunction[SQLTemporal, SQLTemporal]] = add_interval | substract_interval - def identifierWithIntervalFunction: PackratParser[GenericIdentifier] = + def identifierWithIntervalFunction: PackratParser[Identifier] = (identifierWithFunction | identifier) ~ intervalFunction ^^ { case i ~ f => - i.copy(functions = f +: i.functions) + i.withFunctions(f +: i.functions) } - def identifierWithSystemFunction: PackratParser[GenericIdentifier] = + def identifierWithSystemFunction: PackratParser[Identifier] = (current_date | current_time | current_timestamp | now) ~ intervalFunction.? ^^ { case f1 ~ f2 => f2 match { - case Some(f) => GenericIdentifier("", functions = List(f, f1)) - case None => GenericIdentifier("", functions = List(f1)) + case Some(f) => Identifier(List(f, f1)) + case None => Identifier(f1) } } @@ -232,10 +232,10 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => DateTrunc(i, u) } - def extract_identifier: PackratParser[GenericIdentifier] = + def extract_identifier: PackratParser[Identifier] = "(?i)extract".r ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => - i.copy(functions = Extract(u) +: i.functions) + i.withFunctions(Extract(u) +: i.functions) } def extract_year: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = @@ -276,8 +276,8 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: StringValue => - ParseDate(GenericIdentifier("", functions = l :: Nil), f.value) - case i: GenericIdentifier => + ParseDate(Identifier(l), f.value) + case i: Identifier => ParseDate(i, f.value) } } @@ -307,8 +307,8 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: SQLLiteral => - ParseDateTime(GenericIdentifier("", functions = l :: Nil), f.value) - case i: GenericIdentifier => + ParseDateTime(Identifier(l), f.value) + case i: Identifier => ParseDateTime(i, f.value) } } @@ -326,7 +326,7 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def distance: PackratParser[Function] = Distance.regex ^^ (_ => Distance) - def identifierWithTemporalFunction: PackratParser[GenericIdentifier] = + def identifierWithTemporalFunction: PackratParser[Identifier] = rep1sep( date_trunc | extractors | date_functions | datetime_functions, start @@ -334,12 +334,12 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => end ) ^^ { case f ~ _ ~ i ~ _ => i match { - case Some(id) => id.copy(functions = id.functions ++ f) + case Some(id) => id.withFunctions(id.functions ++ f) case None => f.lastOption match { case Some(fi: FunctionWithIdentifier) => - fi.identifier.copy(functions = f ++ fi.identifier.functions) - case _ => GenericIdentifier("", functions = f) + fi.identifier.withFunctions(f ++ fi.identifier.functions) + case _ => Identifier(f) } } } @@ -349,8 +349,8 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) } - def date_diff_identifier: PackratParser[GenericIdentifier] = date_diff ^^ { dd => - GenericIdentifier("", functions = dd :: Nil) + def date_diff_identifier: PackratParser[Identifier] = date_diff ^^ { dd => + Identifier(dd) } def is_null: PackratParser[ConditionalFunction[_]] = @@ -421,8 +421,8 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => case _ ~ e ~ c ~ r ~ _ => Case(e, c, r) } - def case_when_identifier: Parser[GenericIdentifier] = case_when ^^ { cw => - GenericIdentifier("", functions = cw :: Nil) + def case_when_identifier: Parser[Identifier] = case_when ^^ { cw => + Identifier(cw) } def logical_functions: PackratParser[TransformFunction[_, _]] = @@ -442,8 +442,8 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => private[this] def log10: PackratParser[MathOp] = Log10.regex ^^ (_ => Log10) - implicit def functionAsIdentifier(mf: Function): GenericIdentifier = mf match { - case id: GenericIdentifier => id + implicit def functionAsIdentifier(mf: Function): Identifier = mf match { + case id: Identifier => id case fid: FunctionWithIdentifier => fid.identifier case _ => Identifier(mf) } @@ -674,7 +674,7 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => private val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex - def identifier: PackratParser[GenericIdentifier] = + def identifier: PackratParser[Identifier] = Distinct.regex.? ~ identifierRegex ^^ { case d ~ i => GenericIdentifier( i, @@ -719,44 +719,44 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def sql_type: PackratParser[SQLType] = char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type - private[this] def castFunctionWithIdentifier: PackratParser[GenericIdentifier] = + private[this] def castFunctionWithIdentifier: PackratParser[Identifier] = "(?i)cast".r ~ start ~ (identifierWithTransformation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => - i.copy(functions = a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) + i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) } - private[this] def dateFunctionWithIdentifier: PackratParser[GenericIdentifier] = + private[this] def dateFunctionWithIdentifier: PackratParser[Identifier] = (parse_date | format_date | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => af match { - case Some(f) => t.identifier.copy(functions = f +: t +: t.identifier.functions) - case None => t.identifier.copy(functions = t +: t.identifier.functions) + case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) + case None => t.identifier.withFunctions(t +: t.identifier.functions) } } - private[this] def dateTimeFunctionWithIdentifier: PackratParser[GenericIdentifier] = + private[this] def dateTimeFunctionWithIdentifier: PackratParser[Identifier] = (date_trunc | parse_datetime | format_datetime | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { case t ~ af => af match { - case Some(f) => t.identifier.copy(functions = f +: t +: t.identifier.functions) - case None => t.identifier.copy(functions = t +: t.identifier.functions) + case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) + case None => t.identifier.withFunctions(t +: t.identifier.functions) } } - private[this] def conditionalFunctionWithIdentifier: PackratParser[GenericIdentifier] = + private[this] def conditionalFunctionWithIdentifier: PackratParser[Identifier] = (is_null | is_notnull | coalesce | nullif) ^^ { t => - t.identifier.copy(functions = t +: t.identifier.functions) + t.identifier.withFunctions(t +: t.identifier.functions) } def identifierWithTransformation: PackratParser[Identifier] = mathematicalFunctionWithIdentifier | castFunctionWithIdentifier | conditionalFunctionWithIdentifier | dateFunctionWithIdentifier | dateTimeFunctionWithIdentifier | stringFunctionWithIdentifier - def identifierWithAggregation: PackratParser[GenericIdentifier] = + def identifierWithAggregation: PackratParser[Identifier] = aggregates ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case a ~ _ ~ i ~ _ => - i.copy(functions = a +: i.functions) + i.withFunctions(a +: i.functions) } - def identifierWithFunction: PackratParser[GenericIdentifier] = + def identifierWithFunction: PackratParser[Identifier] = rep1sep( sql_functions, start @@ -767,10 +767,10 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => case None => f.lastOption match { case Some(fi: FunctionWithIdentifier) => - fi.identifier.copy(functions = f ++ fi.identifier.functions) - case _ => GenericIdentifier("", functions = f) + fi.identifier.withFunctions(f ++ fi.identifier.functions) + case _ => Identifier(f) } - case Some(id) => id.copy(functions = id.functions ++ f) + case Some(id) => id.withFunctions(id.functions ++ f) } } From 3010f9ca567f000d6903b5a5d623d99db9d19be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Sep 2025 16:55:03 +0200 Subject: [PATCH 08/48] add parser packages --- .../elastic/sql/operator/time/package.scala | 1 + .../elastic/sql/parser/FromParser.scala | 20 + .../elastic/sql/parser/GroupByParser.scala | 17 + .../elastic/sql/parser/HavingParser.scala | 14 + .../elastic/sql/parser/LimitParser.scala | 12 + .../elastic/sql/parser/OrderByParser.scala | 33 + .../elastic/sql/parser/Parser.scala | 972 +----------------- .../elastic/sql/parser/SelectParser.scala | 27 + .../elastic/sql/parser/WhereParser.scala | 370 +++++++ .../parser/function/aggregate/package.scala | 31 + .../sql/parser/function/cond/package.scala | 95 ++ .../sql/parser/function/convert/package.scala | 18 + .../sql/parser/function/geo/package.scala | 14 + .../sql/parser/function/math/package.scala | 104 ++ .../sql/parser/function/string/package.scala | 65 ++ .../sql/parser/function/time/package.scala | 234 +++++ .../sql/parser/operator/math/package.scala | 61 ++ .../elastic/sql/parser/time/package.scala | 66 ++ .../elastic/sql/parser/type/package.scala | 79 ++ 19 files changed, 1285 insertions(+), 948 deletions(-) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/HavingParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala index e824e90f..b7086499 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala @@ -17,6 +17,7 @@ package object time { case object Plus extends Expr("+") with IntervalOperator { override def painless: String = ".plus" } + case object Minus extends Expr("-") with IntervalOperator { override def painless: String = ".minus" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala new file mode 100644 index 00000000..1a3ac23b --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/FromParser.scala @@ -0,0 +1,20 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.{From, Table, Unnest} + +trait FromParser { + self: Parser with LimitParser => + + def unnest: PackratParser[Table] = + Unnest.regex ~ start ~ identifier ~ limit.? ~ end ~ alias ^^ { case _ ~ _ ~ i ~ l ~ _ ~ a => + Table(Unnest(i, l), Some(a)) + } + + def table: PackratParser[Table] = identifier ~ alias.? ^^ { case i ~ a => Table(i, a) } + + def from: PackratParser[From] = From.regex ~ rep1sep(unnest | table, separator) ^^ { + case _ ~ tables => + From(tables) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala new file mode 100644 index 00000000..4e4f683b --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala @@ -0,0 +1,17 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.{Bucket, GroupBy} + +trait GroupByParser { + self: Parser with WhereParser => + + def bucket: PackratParser[Bucket] = identifier ^^ { i => + Bucket(i) + } + + def groupBy: PackratParser[GroupBy] = + GroupBy.regex ~ rep1sep(bucket, separator) ^^ { case _ ~ buckets => + GroupBy(buckets) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/HavingParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/HavingParser.scala new file mode 100644 index 00000000..59e3588e --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/HavingParser.scala @@ -0,0 +1,14 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.Having + +trait HavingParser { + self: Parser with WhereParser => + + def having: PackratParser[Having] = Having.regex ~> whereCriteria ^^ { rawTokens => + Having( + processTokens(rawTokens) + ) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala new file mode 100644 index 00000000..043000e4 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala @@ -0,0 +1,12 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.Limit + +trait LimitParser { + self: Parser => + + def limit: PackratParser[Limit] = Limit.regex ~ long ^^ { case _ ~ i => + Limit(i.value.toInt) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala new file mode 100644 index 00000000..bdfb2889 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala @@ -0,0 +1,33 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.function.Function +import app.softnetwork.elastic.sql.query.{Asc, Desc, FieldSort, OrderBy} + +trait OrderByParser { + self: Parser => + + def asc: PackratParser[Asc.type] = Asc.regex ^^ (_ => Asc) + + def desc: PackratParser[Desc.type] = Desc.regex ^^ (_ => Desc) + + private def fieldName: PackratParser[String] = + """\b(?!(?i)limit\b)[a-zA-Z_][a-zA-Z0-9_]*""".r ^^ (f => f) + + def fieldWithFunction: PackratParser[(String, List[Function])] = + rep1sep(sql_functions, start) ~ start.? ~ fieldName ~ rep1(end) ^^ { case f ~ _ ~ n ~ _ => + (n, f) + } + + def sort: PackratParser[FieldSort] = + (fieldWithFunction | fieldName) ~ (asc | desc).? ^^ { case f ~ o => + f match { + case i: (String, List[Function]) => FieldSort(i._1, o, i._2) + case s: String => FieldSort(s, o, List.empty) + } + } + + def orderBy: PackratParser[OrderBy] = OrderBy.regex ~ rep1sep(sort, separator) ^^ { case _ ~ s => + OrderBy(s) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 3f616a6c..00e5308c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -1,19 +1,17 @@ package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.sql.`type`._ +import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.function._ -import app.softnetwork.elastic.sql.function.aggregate._ -import app.softnetwork.elastic.sql.function.cond._ -import app.softnetwork.elastic.sql.function.convert._ -import app.softnetwork.elastic.sql.function.geo.Distance -import app.softnetwork.elastic.sql.function.math._ -import app.softnetwork.elastic.sql.function.string._ -import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.operator._ -import app.softnetwork.elastic.sql.operator.math._ -import app.softnetwork.elastic.sql.time.TimeUnit._ -import app.softnetwork.elastic.sql.time._ -import app.softnetwork.elastic.sql._ +import app.softnetwork.elastic.sql.parser.`type`.TypeParser +import app.softnetwork.elastic.sql.parser.function.aggregate.AggregateParser +import app.softnetwork.elastic.sql.parser.function.cond.CondParser +import app.softnetwork.elastic.sql.parser.function.convert.ConvertParser +import app.softnetwork.elastic.sql.parser.function.geo.GeoParser +import app.softnetwork.elastic.sql.parser.function.math.MathParser +import app.softnetwork.elastic.sql.parser.function.string.StringParser +import app.softnetwork.elastic.sql.parser.function.time.TemporalParser +import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ import scala.language.implicitConversions @@ -33,8 +31,7 @@ object Parser with GroupByParser with HavingParser with OrderByParser - with LimitParser - with PackratParsers { + with LimitParser { def request: PackratParser[SQLSearchRequest] = { phrase(select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.?) ^^ { @@ -74,27 +71,18 @@ trait CompilationError case class ParserError(msg: String) extends CompilationError -trait Parser extends RegexParsers with PackratParsers { _: WhereParser => - - def literal: PackratParser[StringValue] = - """"[^"]*"|'[^']*'""".r ^^ (str => StringValue(str.substring(1, str.length - 1))) - - def long: PackratParser[LongValue] = - """(-)?(0|[1-9]\d*)""".r ^^ (str => LongValue(str.toLong)) - - def double: PackratParser[DoubleValue] = - """(-)?(\d+\.\d+)""".r ^^ (str => DoubleValue(str.toDouble)) - - def pi: PackratParser[Value[Double]] = - Pi.regex ^^ (_ => PiValue) - - def boolean: PackratParser[BooleanValue] = - """(true|false)""".r ^^ (bool => BooleanValue(bool.toBoolean)) - - def value_identifier: PackratParser[Identifier] = - (literal | long | double | pi | boolean) ^^ { v => - Identifier(v) - } +trait Parser + extends RegexParsers + with PackratParsers + with AggregateParser + with ArithmeticParser + with CondParser + with ConvertParser + with GeoParser + with MathParser + with StringParser + with TemporalParser + with TypeParser { _: WhereParser => def start: PackratParser[Delimiter] = "(" ^^ (_ => StartPredicate) @@ -102,267 +90,6 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def separator: PackratParser[Delimiter] = "," ^^ (_ => Separator) - def count: PackratParser[AggregateFunction] = Count.regex ^^ (_ => Count) - - def min: PackratParser[AggregateFunction] = Min.regex ^^ (_ => Min) - - def max: PackratParser[AggregateFunction] = Max.regex ^^ (_ => Max) - - def avg: PackratParser[AggregateFunction] = Avg.regex ^^ (_ => Avg) - - def sum: PackratParser[AggregateFunction] = Sum.regex ^^ (_ => Sum) - - def year: PackratParser[TimeUnit] = Year.regex ^^ (_ => Year) - - def month: PackratParser[TimeUnit] = Month.regex ^^ (_ => Month) - - def quarter: PackratParser[TimeUnit] = Quarter.regex ^^ (_ => Quarter) - - def week: PackratParser[TimeUnit] = Week.regex ^^ (_ => Week) - - def day: PackratParser[TimeUnit] = Day.regex ^^ (_ => Day) - - def hour: PackratParser[TimeUnit] = Hour.regex ^^ (_ => Hour) - - def minute: PackratParser[TimeUnit] = Minute.regex ^^ (_ => Minute) - - def second: PackratParser[TimeUnit] = Second.regex ^^ (_ => Second) - - def time_unit: PackratParser[TimeUnit] = - year | month | quarter | week | day | hour | minute | second - - def parens: PackratParser[List[Delimiter]] = - start ~ end ^^ { case s ~ e => s :: e :: Nil } - - def current_date: PackratParser[CurrentFunction] = - CurrentDate.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) CurentDateWithParens else CurrentDate - } - - def current_time: PackratParser[CurrentFunction] = - CurrentTime.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) CurrentTimeWithParens else CurrentTime - } - - def current_timestamp: PackratParser[CurrentFunction] = - CurrentTimestamp.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) CurrentTimestampWithParens else CurrentTimestamp - } - - def now: PackratParser[CurrentFunction] = Now.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) NowWithParens else Now - } - - def add: PackratParser[ArithmeticOperator] = Add.sql ^^ (_ => Add) - - def subtract: PackratParser[ArithmeticOperator] = Subtract.sql ^^ (_ => Subtract) - - def multiply: PackratParser[ArithmeticOperator] = Multiply.sql ^^ (_ => Multiply) - - def divide: PackratParser[ArithmeticOperator] = Divide.sql ^^ (_ => Divide) - - def modulo: PackratParser[ArithmeticOperator] = Modulo.sql ^^ (_ => Modulo) - - def factor: PackratParser[PainlessScript] = - "(" ~> arithmeticExpressionLevel2 <~ ")" ^^ { - case expr: ArithmeticExpression => - expr.copy(group = true) - case other => other - } | valueExpr - - def arithmeticExpressionLevel1: Parser[PainlessScript] = - factor ~ rep((multiply | divide | modulo) ~ factor) ^^ { case left ~ list => - list.foldLeft(left) { case (acc, op ~ right) => - ArithmeticExpression(acc, op, right) - } - } - - def arithmeticExpressionLevel2: Parser[PainlessScript] = - arithmeticExpressionLevel1 ~ rep((add | subtract) ~ arithmeticExpressionLevel1) ^^ { - case left ~ list => - list.foldLeft(left) { case (acc, op ~ right) => - ArithmeticExpression(acc, op, right) - } - } - - def identifierWithArithmeticExpression: Parser[Identifier] = - arithmeticExpressionLevel2 ^^ { - case af: ArithmeticExpression => Identifier(af) - case id: Identifier => id - case f: FunctionWithIdentifier => f.identifier - case f: Function => Identifier(f) - case other => throw new Exception(s"Unexpected expression $other") - } - - def interval: PackratParser[TimeInterval] = - Interval.regex ~ long ~ time_unit ^^ { case _ ~ l ~ u => - TimeInterval(l.value.toInt, u) - } - - def add_interval: PackratParser[SQLAddInterval] = - add ~ interval ^^ { case _ ~ it => - SQLAddInterval(it) - } - - def substract_interval: PackratParser[SQLSubtractInterval] = - subtract ~ interval ^^ { case _ ~ it => - SQLSubtractInterval(it) - } - - def intervalFunction: PackratParser[TransformFunction[SQLTemporal, SQLTemporal]] = - add_interval | substract_interval - - def identifierWithIntervalFunction: PackratParser[Identifier] = - (identifierWithFunction | identifier) ~ intervalFunction ^^ { case i ~ f => - i.withFunctions(f +: i.functions) - } - - def identifierWithSystemFunction: PackratParser[Identifier] = - (current_date | current_time | current_timestamp | now) ~ intervalFunction.? ^^ { - case f1 ~ f2 => - f2 match { - case Some(f) => Identifier(List(f, f1)) - case None => Identifier(f1) - } - } - - def date_trunc: PackratParser[FunctionWithIdentifier] = - "(?i)date_trunc".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ u ~ _ => - DateTrunc(i, u) - } - - def extract_identifier: PackratParser[Identifier] = - "(?i)extract".r ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ u ~ _ ~ i ~ _ => - i.withFunctions(Extract(u) +: i.functions) - } - - def extract_year: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Year.regex ^^ (_ => YEAR) - - def extract_month: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Month.regex ^^ (_ => MONTH) - - def extract_day: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Day.regex ^^ (_ => DAY) - - def extract_hour: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Hour.regex ^^ (_ => HOUR) - - def extract_minute: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Minute.regex ^^ (_ => MINUTE) - - def extract_second: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Second.regex ^^ (_ => SECOND) - - def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - extract_year | extract_month | extract_day | extract_hour | extract_minute | extract_second - - def date_add: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)date_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ t ~ _ => - DateAdd(i, t) - } - - def date_sub: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)date_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ t ~ _ => - DateSub(i, t) - } - - def parse_date: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)parse_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { - case _ ~ _ ~ li ~ _ ~ f ~ _ => - li match { - case l: StringValue => - ParseDate(Identifier(l), f.value) - case i: Identifier => - ParseDate(i, f.value) - } - } - - def format_date: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)format_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ f ~ _ => - FormatDate(i, f.value) - } - - def date_functions: PackratParser[DateFunction] = date_add | date_sub | parse_date | format_date - - def datetime_add: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)datetime_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ t ~ _ => - DateTimeAdd(i, t) - } - - def datetime_sub: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)datetime_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ t ~ _ => - DateTimeSub(i, t) - } - - def parse_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)parse_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { - case _ ~ _ ~ li ~ _ ~ f ~ _ => - li match { - case l: SQLLiteral => - ParseDateTime(Identifier(l), f.value) - case i: Identifier => - ParseDateTime(i, f.value) - } - } - - def format_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)format_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { - case _ ~ _ ~ i ~ _ ~ f ~ _ => - FormatDateTime(i, f.value) - } - - def datetime_functions: PackratParser[DateTimeFunction] = - datetime_add | datetime_sub | parse_datetime | format_datetime - - def aggregates: PackratParser[AggregateFunction] = count | min | max | avg | sum - - def distance: PackratParser[Function] = Distance.regex ^^ (_ => Distance) - - def identifierWithTemporalFunction: PackratParser[Identifier] = - rep1sep( - date_trunc | extractors | date_functions | datetime_functions, - start - ) ~ start.? ~ (identifierWithSystemFunction | identifier).? ~ rep( - end - ) ^^ { case f ~ _ ~ i ~ _ => - i match { - case Some(id) => id.withFunctions(id.functions ++ f) - case None => - f.lastOption match { - case Some(fi: FunctionWithIdentifier) => - fi.identifier.withFunctions(f ++ fi.identifier.functions) - case _ => Identifier(f) - } - } - } - - def date_diff: PackratParser[BinaryFunction[_, _, _]] = - "(?i)date_diff".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { - case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) - } - - def date_diff_identifier: PackratParser[Identifier] = date_diff ^^ { dd => - Identifier(dd) - } - - def is_null: PackratParser[ConditionalFunction[_]] = - "(?i)isnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => IsNullFunction(i) - } - - def is_notnull: PackratParser[ConditionalFunction[_]] = - "(?i)isnotnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => IsNotNullFunction(i) - } - def valueExpr: PackratParser[PainlessScript] = // les plus spécifiques en premier identifierWithTransformation | // transformations appliquées à un identifier @@ -379,175 +106,14 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => boolean | identifier - def coalesce: PackratParser[Coalesce] = - Coalesce.regex ~ start ~ rep1sep( - valueExpr, - separator - ) ~ end ^^ { case _ ~ _ ~ ids ~ _ => - Coalesce(ids) - } - - def nullif: PackratParser[NullIf] = - NullIf.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ end ^^ { - case _ ~ _ ~ id1 ~ _ ~ id2 ~ _ => NullIf(id1, id2) - } - - def start_case: PackratParser[StartCase.type] = Case.regex ^^ (_ => StartCase) - - def when_case: PackratParser[WhenCase.type] = When.regex ^^ (_ => WhenCase) - - def then_case: PackratParser[ThenCase.type] = Then.regex ^^ (_ => ThenCase) - - def else_case: PackratParser[Else.type] = Else.regex ^^ (_ => Else) - - def end_case: PackratParser[EndCase.type] = End.regex ^^ (_ => EndCase) - - def case_condition: Parser[(PainlessScript, PainlessScript)] = - when_case ~ (whereCriteria | valueExpr) ~ then_case.? ~ valueExpr ^^ { case _ ~ c ~ _ ~ r => - c match { - case p: PainlessScript => p -> r - case rawTokens: List[Token] => - processTokens(rawTokens) match { - case Some(criteria) => criteria -> r - case _ => Null -> r - } - } - } - - def case_else: Parser[PainlessScript] = else_case ~ valueExpr ^^ { case _ ~ r => r } - - def case_when: PackratParser[Case] = - start_case ~ valueExpr.? ~ rep1(case_condition) ~ case_else.? ~ end_case ^^ { - case _ ~ e ~ c ~ r ~ _ => Case(e, c, r) - } - - def case_when_identifier: Parser[Identifier] = case_when ^^ { cw => - Identifier(cw) - } - - def logical_functions: PackratParser[TransformFunction[_, _]] = - is_null | is_notnull | coalesce | nullif | case_when - - private[this] def abs: PackratParser[MathOp] = Abs.regex ^^ (_ => Abs) - - private[this] def ceil: PackratParser[MathOp] = Ceil.regex ^^ (_ => Ceil) - - private[this] def floor: PackratParser[MathOp] = Floor.regex ^^ (_ => Floor) - - private[this] def exp: PackratParser[MathOp] = Exp.regex ^^ (_ => Exp) - - private[this] def sqrt: PackratParser[MathOp] = Sqrt.regex ^^ (_ => Sqrt) - - private[this] def log: PackratParser[MathOp] = Log.regex ^^ (_ => Log) - - private[this] def log10: PackratParser[MathOp] = Log10.regex ^^ (_ => Log10) - implicit def functionAsIdentifier(mf: Function): Identifier = mf match { case id: Identifier => id case fid: FunctionWithIdentifier => fid.identifier case _ => Identifier(mf) } - def arithmeticFunction: PackratParser[MathematicalFunction] = - (abs | ceil | exp | floor | log | log10 | sqrt) ~ start ~ valueExpr ~ end ^^ { - case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) - } - - private[this] def sin: PackratParser[Trigonometric] = Sin.regex ^^ (_ => Sin) - - private[this] def asin: PackratParser[Trigonometric] = Asin.regex ^^ (_ => Asin) - - private[this] def cos: PackratParser[Trigonometric] = Cos.regex ^^ (_ => Cos) - - private[this] def acos: PackratParser[Trigonometric] = Acos.regex ^^ (_ => Acos) - - private[this] def tan: PackratParser[Trigonometric] = Tan.regex ^^ (_ => Tan) - - private[this] def atan: PackratParser[Trigonometric] = Atan.regex ^^ (_ => Atan) - - private[this] def atan2: PackratParser[Trigonometric] = Atan2.regex ^^ (_ => Atan2) - - def atan2Function: PackratParser[MathematicalFunction] = - atan2 ~ start ~ (double | valueExpr) ~ separator ~ (double | valueExpr) ~ end ^^ { - case _ ~ _ ~ y ~ _ ~ x ~ _ => Atan2(y, x) - } - - def trigonometricFunction: PackratParser[MathematicalFunction] = - atan2Function | ((sin | asin | cos | acos | tan | atan) ~ start ~ valueExpr ~ end ^^ { - case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) - }) - - private[this] def round: PackratParser[MathOp] = Round.regex ^^ (_ => Round) - - def roundFunction: PackratParser[MathematicalFunction] = - round ~ start ~ valueExpr ~ separator.? ~ long.? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ _ => - Round(v, s.map(_.value.toInt)) - } - - private[this] def pow: PackratParser[MathOp] = Pow.regex ^^ (_ => Pow) - - def powFunction: PackratParser[MathematicalFunction] = - pow ~ start ~ valueExpr ~ separator ~ long ~ end ^^ { case _ ~ _ ~ v1 ~ _ ~ e ~ _ => - Pow(v1, e.value.toInt) - } - - private[this] def sign: PackratParser[MathOp] = Sign.regex ^^ (_ => Sign) - - def signFunction: PackratParser[MathematicalFunction] = - sign ~ start ~ valueExpr ~ end ^^ { case _ ~ _ ~ v ~ _ => Sign(v) } - - def mathematicalFunction: PackratParser[MathematicalFunction] = - arithmeticFunction | trigonometricFunction | roundFunction | powFunction | signFunction - - def mathematicalFunctionWithIdentifier: PackratParser[Identifier] = - mathematicalFunction ^^ { mf => - mf.identifier - } - - def concatFunction: PackratParser[StringFunction[SQLVarchar]] = - Concat.regex ~ start ~ rep1sep(valueExpr, separator) ~ end ^^ { case _ ~ _ ~ vs ~ _ => - Concat(vs) - } - - def substringFunction: PackratParser[StringFunction[SQLVarchar]] = - Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((To.regex | separator) ~ long).? ~ end ^^ { - case _ ~ _ ~ v ~ _ ~ s ~ eOpt ~ _ => - Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) - } - - def stringFunctionWithIdentifier: PackratParser[Identifier] = - (concatFunction | substringFunction) ^^ { sf => - sf.identifier - } - - def length: PackratParser[StringFunction[SQLBigInt]] = - Length.regex ^^ { _ => - SQLLength - } - - def lower: PackratParser[StringFunction[SQLVarchar]] = - Lower.regex ^^ { _ => - StringFunctionWithOp(Lower) - } - - def upper: PackratParser[StringFunction[SQLVarchar]] = - Upper.regex ^^ { _ => - StringFunctionWithOp(Upper) - } - - def trim: PackratParser[StringFunction[SQLVarchar]] = - Trim.regex ^^ { _ => - StringFunctionWithOp(Trim) - } - - def string_functions: Parser[ - StringFunction[_] - ] = /*concatFunction | substringFunction |*/ length | lower | upper | trim - def sql_functions: PackratParser[Function] = - aggregates | distance | date_diff | date_trunc | extractors | date_functions | datetime_functions | logical_functions | string_functions - - //private val regexIdentifier = """[\*a-zA-Z_\-][a-zA-Z0-9_\-\.\[\]\*]*""" + aggregates | distance | date_diff | date_trunc | extractors | date_functions | datetime_functions | conditional_functions | string_functions private val reservedKeywords = Seq( "select", @@ -683,79 +249,9 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => ) } - def char_type: PackratParser[SQLTypes.Char.type] = - "(?i)char".r ^^ (_ => SQLTypes.Char) - - def string_type: PackratParser[SQLTypes.Varchar.type] = - "(?i)varchar|string".r ^^ (_ => SQLTypes.Varchar) - - def date_type: PackratParser[SQLTypes.Date.type] = "(?i)date".r ^^ (_ => SQLTypes.Date) - - def time_type: PackratParser[SQLTypes.Time.type] = "(?i)time".r ^^ (_ => SQLTypes.Time) - - def datetime_type: PackratParser[SQLTypes.DateTime.type] = - "(?i)(datetime)".r ^^ (_ => SQLTypes.DateTime) - - def timestamp_type: PackratParser[SQLTypes.Timestamp.type] = - "(?i)(timestamp)".r ^^ (_ => SQLTypes.Timestamp) - - def boolean_type: PackratParser[SQLTypes.Boolean.type] = - "(?i)boolean".r ^^ (_ => SQLTypes.Boolean) - - def byte_type: PackratParser[SQLTypes.TinyInt.type] = - "(?i)(byte|tinyint)".r ^^ (_ => SQLTypes.TinyInt) - - def short_type: PackratParser[SQLTypes.SmallInt.type] = - "(?i)(short|smallint)".r ^^ (_ => SQLTypes.SmallInt) - - def int_type: PackratParser[SQLTypes.Int.type] = "(?i)(int|integer)".r ^^ (_ => SQLTypes.Int) - - def long_type: PackratParser[SQLTypes.BigInt.type] = "(?i)long|bigint".r ^^ (_ => SQLTypes.BigInt) - - def double_type: PackratParser[SQLTypes.Double.type] = "(?i)double".r ^^ (_ => SQLTypes.Double) - - def float_type: PackratParser[SQLTypes.Real.type] = "(?i)float|real".r ^^ (_ => SQLTypes.Real) - - def sql_type: PackratParser[SQLType] = - char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type - - private[this] def castFunctionWithIdentifier: PackratParser[Identifier] = - "(?i)cast".r ~ start ~ (identifierWithTransformation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { - case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => - i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) - } - - private[this] def dateFunctionWithIdentifier: PackratParser[Identifier] = - (parse_date | format_date | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => - af match { - case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) - case None => t.identifier.withFunctions(t +: t.identifier.functions) - } - } - - private[this] def dateTimeFunctionWithIdentifier: PackratParser[Identifier] = - (date_trunc | parse_datetime | format_datetime | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { - case t ~ af => - af match { - case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) - case None => t.identifier.withFunctions(t +: t.identifier.functions) - } - } - - private[this] def conditionalFunctionWithIdentifier: PackratParser[Identifier] = - (is_null | is_notnull | coalesce | nullif) ^^ { t => - t.identifier.withFunctions(t +: t.identifier.functions) - } - def identifierWithTransformation: PackratParser[Identifier] = mathematicalFunctionWithIdentifier | castFunctionWithIdentifier | conditionalFunctionWithIdentifier | dateFunctionWithIdentifier | dateTimeFunctionWithIdentifier | stringFunctionWithIdentifier - def identifierWithAggregation: PackratParser[Identifier] = - aggregates ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { - case a ~ _ ~ i ~ _ => - i.withFunctions(a +: i.functions) - } - def identifierWithFunction: PackratParser[Identifier] = rep1sep( sql_functions, @@ -779,424 +275,4 @@ trait Parser extends RegexParsers with PackratParsers { _: WhereParser => def alias: PackratParser[Alias] = Alias.regex.? ~ regexAlias.r ^^ { case _ ~ b => Alias(b) } - def field: PackratParser[Field] = - (identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | case_when_identifier | identifier) ~ alias.? ^^ { - case i ~ a => - Field(i, a) - } - -} - -trait SelectParser { - self: Parser with WhereParser => - - def except: PackratParser[Except] = Except.regex ~ start ~ rep1sep(field, separator) ~ end ^^ { - case _ ~ _ ~ e ~ _ => - Except(e) - } - - def select: PackratParser[Select] = - Select.regex ~ rep1sep( - field, - separator - ) ~ except.? ^^ { case _ ~ fields ~ e => - Select(fields, e) - } - -} - -trait FromParser { - self: Parser with LimitParser => - - def unnest: PackratParser[Table] = - Unnest.regex ~ start ~ identifier ~ limit.? ~ end ~ alias ^^ { case _ ~ _ ~ i ~ l ~ _ ~ a => - Table(Unnest(i, l), Some(a)) - } - - def table: PackratParser[Table] = identifier ~ alias.? ^^ { case i ~ a => Table(i, a) } - - def from: PackratParser[From] = From.regex ~ rep1sep(unnest | table, separator) ^^ { - case _ ~ tables => - From(tables) - } - -} - -trait WhereParser { - self: Parser with GroupByParser with OrderByParser => - - def isNull: PackratParser[Criteria] = identifier ~ IsNull.regex ^^ { case i ~ _ => - IsNullExpr(i) - } - - def isNotNull: PackratParser[Criteria] = identifier ~ IsNotNull.regex ^^ { case i ~ _ => - IsNotNullExpr(i) - } - - private def eq: PackratParser[ComparisonOperator] = Eq.sql ^^ (_ => Eq) - - private def ne: PackratParser[ComparisonOperator] = Ne.sql ^^ (_ => Ne) - - private def diff: PackratParser[ComparisonOperator] = Diff.sql ^^ (_ => Diff) - - private def any_identifier: PackratParser[Identifier] = - identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | date_diff_identifier | extract_identifier | identifier - - private def equality: PackratParser[GenericExpression] = - not.? ~ any_identifier ~ (eq | ne | diff) ~ (boolean | literal | double | pi | long | any_identifier) ^^ { - case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) - } - - def like: PackratParser[GenericExpression] = - any_identifier ~ not.? ~ Like.regex ~ literal ^^ { case i ~ n ~ _ ~ v => - GenericExpression(i, Like, v, n) - } - - private def ge: PackratParser[ComparisonOperator] = Ge.sql ^^ (_ => Ge) - - def gt: PackratParser[ComparisonOperator] = Gt.sql ^^ (_ => Gt) - - private def le: PackratParser[ComparisonOperator] = Le.sql ^^ (_ => Le) - - def lt: PackratParser[ComparisonOperator] = Lt.sql ^^ (_ => Lt) - - private def comparison: PackratParser[GenericExpression] = - not.? ~ any_identifier ~ (ge | gt | le | lt) ~ (double | pi | long | literal | any_identifier) ^^ { - case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) - } - - def in: PackratParser[ExpressionOperator] = In.regex ^^ (_ => In) - - private def inLiteral: PackratParser[Criteria] = - any_identifier ~ not.? ~ in ~ start ~ rep1sep(literal, separator) ~ end ^^ { - case i ~ n ~ _ ~ _ ~ v ~ _ => - InExpr( - i, - StringValues(v), - n - ) - } - - private def inDoubles: PackratParser[Criteria] = - any_identifier ~ not.? ~ in ~ start ~ rep1sep( - double, - separator - ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => - InExpr( - i, - DoubleValues(v), - n - ) - } - - private def inLongs: PackratParser[Criteria] = - any_identifier ~ not.? ~ in ~ start ~ rep1sep( - long, - separator - ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => - InExpr( - i, - LongValues(v), - n - ) - } - - def between: PackratParser[Criteria] = - any_identifier ~ not.? ~ Between.regex ~ literal ~ and ~ literal ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LiteralFromTo(from, to), n) - } - - def betweenLongs: PackratParser[Criteria] = - any_identifier ~ not.? ~ Between.regex ~ long ~ and ~ long ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LongFromTo(from, to), n) - } - - def betweenDoubles: PackratParser[Criteria] = - any_identifier ~ not.? ~ Between.regex ~ double ~ and ~ double ^^ { - case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) - } - - def sql_distance: PackratParser[Criteria] = - distance ~ start ~ identifier ~ separator ~ start ~ double ~ separator ~ double ~ end ~ end ~ le ~ literal ^^ { - case _ ~ _ ~ i ~ _ ~ _ ~ lat ~ _ ~ lon ~ _ ~ _ ~ _ ~ d => ElasticGeoDistance(i, d, lat, lon) - } - - def matchCriteria: PackratParser[MatchCriteria] = - Match.regex ~ start ~ rep1sep( - any_identifier, - separator - ) ~ end ~ Against.regex ~ start ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ _ ~ _ ~ l ~ _ => - MatchCriteria(i, l) - } - - def and: PackratParser[PredicateOperator] = And.regex ^^ (_ => And) - - def or: PackratParser[PredicateOperator] = Or.regex ^^ (_ => Or) - - def not: PackratParser[Not.type] = Not.regex ^^ (_ => Not) - - def logical_criteria: PackratParser[Criteria] = - (is_null | is_notnull) ^^ { case ConditionalFunctionAsCriteria(c) => - c - } - - def criteria: PackratParser[Criteria] = - (equality | like | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ sql_distance | matchCriteria | logical_criteria) ^^ ( - c => c - ) - - def predicate: PackratParser[Predicate] = criteria ~ (and | or) ~ not.? ~ criteria ^^ { - case l ~ o ~ n ~ r => Predicate(l, o, r, n) - } - - def nestedCriteria: PackratParser[ElasticRelation] = - Nested.regex ~ start.? ~ criteria ~ end.? ^^ { case _ ~ _ ~ c ~ _ => - ElasticNested(c, None) - } - - def nestedPredicate: PackratParser[ElasticRelation] = Nested.regex ~ start ~ predicate ~ end ^^ { - case _ ~ _ ~ p ~ _ => ElasticNested(p, None) - } - - def childCriteria: PackratParser[ElasticRelation] = Child.regex ~ start.? ~ criteria ~ end.? ^^ { - case _ ~ _ ~ c ~ _ => ElasticChild(c) - } - - def childPredicate: PackratParser[ElasticRelation] = Child.regex ~ start ~ predicate ~ end ^^ { - case _ ~ _ ~ p ~ _ => ElasticChild(p) - } - - def parentCriteria: PackratParser[ElasticRelation] = - Parent.regex ~ start.? ~ criteria ~ end.? ^^ { case _ ~ _ ~ c ~ _ => - ElasticParent(c) - } - - def parentPredicate: PackratParser[ElasticRelation] = Parent.regex ~ start ~ predicate ~ end ^^ { - case _ ~ _ ~ p ~ _ => ElasticParent(p) - } - - private def allPredicate: PackratParser[Criteria] = - nestedPredicate | childPredicate | parentPredicate | predicate - - private def allCriteria: PackratParser[Token] = - nestedCriteria | childCriteria | parentCriteria | criteria - - def whereCriteria: PackratParser[List[Token]] = rep1( - allPredicate | allCriteria | start | or | and | end | then_case - ) - - def where: PackratParser[Where] = - Where.regex ~ whereCriteria ^^ { case _ ~ rawTokens => - Where(processTokens(rawTokens)) - } - - import scala.annotation.tailrec - - /** This method is used to recursively process a list of SQL tokens and construct SQL criteria and - * predicates from these tokens. Here are the key points: - * - * Base case (Nil): If the list of tokens is empty (Nil), we check the contents of the stack to - * determine the final result. - * - * If the stack contains an operator, a left criterion and a right criterion, we create a - * SQLPredicate predicate. Otherwise, we return the first criterion (SQLCriteria) of the stack if - * it exists. Case of criteria (SQLCriteria): If the first token is a criterion, we treat it - * according to the content of the stack: - * - * If the stack contains a predicate operator, we create a predicate with the left and right - * criteria and update the stack. Otherwise, we simply add the criterion to the stack. Case of - * operators (SQLPredicateOperator): If the first token is a predicate operator, we treat it - * according to the contents of the stack: - * - * If the stack contains at least two elements, we create a predicate with the left and right - * criterion and update the stack. If the stack contains only one element (a single operator), we - * simply add the operator to the stack. Otherwise, it's a battery status error. Case of - * delimiters (StartDelimiter and EndDelimiter): If the first token is a start delimiter - * (StartDelimiter), we extract the tokens up to the corresponding end delimiter (EndDelimiter), - * we recursively process the extracted sub-tokens, then we continue with the rest of the tokens. - * - * Other cases: If none of the previous cases match, an IllegalStateException is thrown to - * indicate an unexpected token type. - * - * @param tokens - * - liste des tokens SQL - * @param stack - * - stack de tokens - * @return - */ - @tailrec - private def processTokensHelper( - tokens: List[Token], - stack: List[Token] - ): Option[Criteria] = { - tokens match { - case Nil => - stack match { - case (right: Criteria) :: (op: PredicateOperator) :: (left: Criteria) :: Nil => - Option( - Predicate(left, op, right) - ) - case _ => - stack.headOption.collect { case c: Criteria => c } - } - case (_: StartDelimiter) :: rest => - val (subTokens, remainingTokens) = extractSubTokens(rest, 1) - val subCriteria = processSubTokens(subTokens) match { - case p: Predicate => p.copy(group = true) - case c => c - } - processTokensHelper(remainingTokens, subCriteria :: stack) - case (c: Criteria) :: rest => - stack match { - case (op: PredicateOperator) :: (left: Criteria) :: tail => - val predicate = Predicate(left, op, c) - processTokensHelper(rest, predicate :: tail) - case _ => - processTokensHelper(rest, c :: stack) - } - case (op: PredicateOperator) :: rest => - stack match { - case (right: Criteria) :: (left: Criteria) :: tail => - val predicate = Predicate(left, op, right) - processTokensHelper(rest, predicate :: tail) - case (right: Criteria) :: (o: PredicateOperator) :: tail => - tail match { - case (left: Criteria) :: tt => - val predicate = Predicate(left, op, right) - processTokensHelper(rest, o :: predicate :: tt) - case _ => - processTokensHelper(rest, op :: stack) - } - case _ :: Nil => - processTokensHelper(rest, op :: stack) - case _ => - throw ValidationError("Invalid stack state for predicate creation") - } - case ThenCase :: _ => - processTokensHelper(Nil, stack) // exit processing on THEN - case (_: EndDelimiter) :: rest => - processTokensHelper(rest, stack) // Ignore and move on - case _ => processTokensHelper(Nil, stack) - } - } - - /** This method calls processTokensHelper with an empty stack (Nil) to begin processing primary - * tokens. - * - * @param tokens - * - list of SQL tokens - * @return - */ - protected def processTokens( - tokens: List[Token] - ): Option[Criteria] = { - processTokensHelper(tokens, Nil) - } - - /** This method is used to process subtokens extracted between delimiters. It calls - * processTokensHelper and returns the result as a SQLCriteria, or throws an exception if no - * criteria is found. - * - * @param tokens - * - list of SQL tokens - * @return - */ - private def processSubTokens(tokens: List[Token]): Criteria = { - processTokensHelper(tokens, Nil).getOrElse( - throw ValidationError("Empty sub-expression") - ) - } - - /** This method is used to extract subtokens between a start delimiter (StartDelimiter) and its - * corresponding end delimiter (EndDelimiter). It uses a recursive approach to maintain the count - * of open and closed delimiters and correctly construct the list of extracted subtokens. - * - * @param tokens - * - list of SQL tokens - * @param openCount - * - count of open delimiters - * @param subTokens - * - list of extracted subtokens - * @return - */ - @tailrec - private def extractSubTokens( - tokens: List[Token], - openCount: Int, - subTokens: List[Token] = Nil - ): (List[Token], List[Token]) = { - tokens match { - case Nil => throw ValidationError("Unbalanced parentheses") - case (start: StartDelimiter) :: rest => - extractSubTokens(rest, openCount + 1, start :: subTokens) - case (end: EndDelimiter) :: rest => - if (openCount - 1 == 0) { - (subTokens.reverse, rest) - } else extractSubTokens(rest, openCount - 1, end :: subTokens) - case head :: rest => extractSubTokens(rest, openCount, head :: subTokens) - } - } -} - -trait GroupByParser { - self: Parser with WhereParser => - - def bucket: PackratParser[Bucket] = identifier ^^ { i => - Bucket(i) - } - - def groupBy: PackratParser[GroupBy] = - GroupBy.regex ~ rep1sep(bucket, separator) ^^ { case _ ~ buckets => - GroupBy(buckets) - } - -} - -trait HavingParser { - self: Parser with WhereParser => - - def having: PackratParser[Having] = Having.regex ~> whereCriteria ^^ { rawTokens => - Having( - processTokens(rawTokens) - ) - } - -} - -trait OrderByParser { - self: Parser => - - def asc: PackratParser[Asc.type] = Asc.regex ^^ (_ => Asc) - - def desc: PackratParser[Desc.type] = Desc.regex ^^ (_ => Desc) - - private def fieldName: PackratParser[String] = - """\b(?!(?i)limit\b)[a-zA-Z_][a-zA-Z0-9_]*""".r ^^ (f => f) - - def fieldWithFunction: PackratParser[(String, List[Function])] = - rep1sep(sql_functions, start) ~ start.? ~ fieldName ~ rep1(end) ^^ { case f ~ _ ~ n ~ _ => - (n, f) - } - - def sort: PackratParser[FieldSort] = - (fieldWithFunction | fieldName) ~ (asc | desc).? ^^ { case f ~ o => - f match { - case i: (String, List[Function]) => FieldSort(i._1, o, i._2) - case s: String => FieldSort(s, o, List.empty) - } - } - - def orderBy: PackratParser[OrderBy] = OrderBy.regex ~ rep1sep(sort, separator) ^^ { case _ ~ s => - OrderBy(s) - } - -} - -trait LimitParser { - self: Parser => - - def limit: PackratParser[Limit] = Limit.regex ~ long ^^ { case _ ~ i => - Limit(i.value.toInt) - } - } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala new file mode 100644 index 00000000..765d8510 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -0,0 +1,27 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.{Except, Field, Select} + +trait SelectParser { + self: Parser with WhereParser => + + def field: PackratParser[Field] = + (identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | case_when_identifier | identifier) ~ alias.? ^^ { + case i ~ a => + Field(i, a) + } + + def except: PackratParser[Except] = Except.regex ~ start ~ rep1sep(field, separator) ~ end ^^ { + case _ ~ _ ~ e ~ _ => + Except(e) + } + + def select: PackratParser[Select] = + Select.regex ~ rep1sep( + field, + separator + ) ~ except.? ^^ { case _ ~ fields ~ e => + Select(fields, e) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala new file mode 100644 index 00000000..cc76aabf --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -0,0 +1,370 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.{ + DoubleFromTo, + DoubleValues, + Identifier, + LiteralFromTo, + LongFromTo, + LongValues, + StringValues, + Token +} +import app.softnetwork.elastic.sql.operator.{ + Against, + And, + Between, + Child, + ComparisonOperator, + Diff, + Eq, + ExpressionOperator, + Ge, + Gt, + In, + IsNotNull, + IsNull, + Le, + Like, + Lt, + Match, + Ne, + Nested, + Not, + Or, + Parent, + PredicateOperator +} +import app.softnetwork.elastic.sql.query.{ + BetweenExpr, + ConditionalFunctionAsCriteria, + Criteria, + ElasticChild, + ElasticGeoDistance, + ElasticNested, + ElasticParent, + ElasticRelation, + GenericExpression, + InExpr, + IsNotNullExpr, + IsNullExpr, + MatchCriteria, + Predicate, + Where +} + +trait WhereParser { + self: Parser with GroupByParser with OrderByParser => + + def isNull: PackratParser[Criteria] = identifier ~ IsNull.regex ^^ { case i ~ _ => + IsNullExpr(i) + } + + def isNotNull: PackratParser[Criteria] = identifier ~ IsNotNull.regex ^^ { case i ~ _ => + IsNotNullExpr(i) + } + + private def eq: PackratParser[ComparisonOperator] = Eq.sql ^^ (_ => Eq) + + private def ne: PackratParser[ComparisonOperator] = Ne.sql ^^ (_ => Ne) + + private def diff: PackratParser[ComparisonOperator] = Diff.sql ^^ (_ => Diff) + + private def any_identifier: PackratParser[Identifier] = + identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | date_diff_identifier | extract_identifier | identifier + + private def equality: PackratParser[GenericExpression] = + not.? ~ any_identifier ~ (eq | ne | diff) ~ (boolean | literal | double | pi | long | any_identifier) ^^ { + case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) + } + + def like: PackratParser[GenericExpression] = + any_identifier ~ not.? ~ Like.regex ~ literal ^^ { case i ~ n ~ _ ~ v => + GenericExpression(i, Like, v, n) + } + + private def ge: PackratParser[ComparisonOperator] = Ge.sql ^^ (_ => Ge) + + def gt: PackratParser[ComparisonOperator] = Gt.sql ^^ (_ => Gt) + + private def le: PackratParser[ComparisonOperator] = Le.sql ^^ (_ => Le) + + def lt: PackratParser[ComparisonOperator] = Lt.sql ^^ (_ => Lt) + + private def comparison: PackratParser[GenericExpression] = + not.? ~ any_identifier ~ (ge | gt | le | lt) ~ (double | pi | long | literal | any_identifier) ^^ { + case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) + } + + def in: PackratParser[ExpressionOperator] = In.regex ^^ (_ => In) + + private def inLiteral: PackratParser[Criteria] = + any_identifier ~ not.? ~ in ~ start ~ rep1sep(literal, separator) ~ end ^^ { + case i ~ n ~ _ ~ _ ~ v ~ _ => + InExpr( + i, + StringValues(v), + n + ) + } + + private def inDoubles: PackratParser[Criteria] = + any_identifier ~ not.? ~ in ~ start ~ rep1sep( + double, + separator + ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => + InExpr( + i, + DoubleValues(v), + n + ) + } + + private def inLongs: PackratParser[Criteria] = + any_identifier ~ not.? ~ in ~ start ~ rep1sep( + long, + separator + ) ~ end ^^ { case i ~ n ~ _ ~ _ ~ v ~ _ => + InExpr( + i, + LongValues(v), + n + ) + } + + def between: PackratParser[Criteria] = + any_identifier ~ not.? ~ Between.regex ~ literal ~ and ~ literal ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LiteralFromTo(from, to), n) + } + + def betweenLongs: PackratParser[Criteria] = + any_identifier ~ not.? ~ Between.regex ~ long ~ and ~ long ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LongFromTo(from, to), n) + } + + def betweenDoubles: PackratParser[Criteria] = + any_identifier ~ not.? ~ Between.regex ~ double ~ and ~ double ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) + } + + def sql_distance: PackratParser[Criteria] = + distance ~ start ~ identifier ~ separator ~ start ~ double ~ separator ~ double ~ end ~ end ~ le ~ literal ^^ { + case _ ~ _ ~ i ~ _ ~ _ ~ lat ~ _ ~ lon ~ _ ~ _ ~ _ ~ d => ElasticGeoDistance(i, d, lat, lon) + } + + def matchCriteria: PackratParser[MatchCriteria] = + Match.regex ~ start ~ rep1sep( + any_identifier, + separator + ) ~ end ~ Against.regex ~ start ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ _ ~ _ ~ l ~ _ => + MatchCriteria(i, l) + } + + def and: PackratParser[PredicateOperator] = And.regex ^^ (_ => And) + + def or: PackratParser[PredicateOperator] = Or.regex ^^ (_ => Or) + + def not: PackratParser[Not.type] = Not.regex ^^ (_ => Not) + + def logical_criteria: PackratParser[Criteria] = + (is_null | is_notnull) ^^ { case ConditionalFunctionAsCriteria(c) => + c + } + + def criteria: PackratParser[Criteria] = + (equality | like | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ sql_distance | matchCriteria | logical_criteria) ^^ ( + c => c + ) + + def predicate: PackratParser[Predicate] = criteria ~ (and | or) ~ not.? ~ criteria ^^ { + case l ~ o ~ n ~ r => Predicate(l, o, r, n) + } + + def nestedCriteria: PackratParser[ElasticRelation] = + Nested.regex ~ start.? ~ criteria ~ end.? ^^ { case _ ~ _ ~ c ~ _ => + ElasticNested(c, None) + } + + def nestedPredicate: PackratParser[ElasticRelation] = Nested.regex ~ start ~ predicate ~ end ^^ { + case _ ~ _ ~ p ~ _ => ElasticNested(p, None) + } + + def childCriteria: PackratParser[ElasticRelation] = Child.regex ~ start.? ~ criteria ~ end.? ^^ { + case _ ~ _ ~ c ~ _ => ElasticChild(c) + } + + def childPredicate: PackratParser[ElasticRelation] = Child.regex ~ start ~ predicate ~ end ^^ { + case _ ~ _ ~ p ~ _ => ElasticChild(p) + } + + def parentCriteria: PackratParser[ElasticRelation] = + Parent.regex ~ start.? ~ criteria ~ end.? ^^ { case _ ~ _ ~ c ~ _ => + ElasticParent(c) + } + + def parentPredicate: PackratParser[ElasticRelation] = Parent.regex ~ start ~ predicate ~ end ^^ { + case _ ~ _ ~ p ~ _ => ElasticParent(p) + } + + private def allPredicate: PackratParser[Criteria] = + nestedPredicate | childPredicate | parentPredicate | predicate + + private def allCriteria: PackratParser[Token] = + nestedCriteria | childCriteria | parentCriteria | criteria + + def whereCriteria: PackratParser[List[Token]] = rep1( + allPredicate | allCriteria | start | or | and | end | then_case + ) + + def where: PackratParser[Where] = + Where.regex ~ whereCriteria ^^ { case _ ~ rawTokens => + Where(processTokens(rawTokens)) + } + + import scala.annotation.tailrec + + /** This method is used to recursively process a list of SQL tokens and construct SQL criteria and + * predicates from these tokens. Here are the key points: + * + * Base case (Nil): If the list of tokens is empty (Nil), we check the contents of the stack to + * determine the final result. + * + * If the stack contains an operator, a left criterion and a right criterion, we create a + * SQLPredicate predicate. Otherwise, we return the first criterion (SQLCriteria) of the stack if + * it exists. Case of criteria (SQLCriteria): If the first token is a criterion, we treat it + * according to the content of the stack: + * + * If the stack contains a predicate operator, we create a predicate with the left and right + * criteria and update the stack. Otherwise, we simply add the criterion to the stack. Case of + * operators (SQLPredicateOperator): If the first token is a predicate operator, we treat it + * according to the contents of the stack: + * + * If the stack contains at least two elements, we create a predicate with the left and right + * criterion and update the stack. If the stack contains only one element (a single operator), we + * simply add the operator to the stack. Otherwise, it's a battery status error. Case of + * delimiters (StartDelimiter and EndDelimiter): If the first token is a start delimiter + * (StartDelimiter), we extract the tokens up to the corresponding end delimiter (EndDelimiter), + * we recursively process the extracted sub-tokens, then we continue with the rest of the tokens. + * + * Other cases: If none of the previous cases match, an IllegalStateException is thrown to + * indicate an unexpected token type. + * + * @param tokens + * - liste des tokens SQL + * @param stack + * - stack de tokens + * @return + */ + @tailrec + private def processTokensHelper( + tokens: List[Token], + stack: List[Token] + ): Option[Criteria] = { + tokens match { + case Nil => + stack match { + case (right: Criteria) :: (op: PredicateOperator) :: (left: Criteria) :: Nil => + Option( + Predicate(left, op, right) + ) + case _ => + stack.headOption.collect { case c: Criteria => c } + } + case (_: StartDelimiter) :: rest => + val (subTokens, remainingTokens) = extractSubTokens(rest, 1) + val subCriteria = processSubTokens(subTokens) match { + case p: Predicate => p.copy(group = true) + case c => c + } + processTokensHelper(remainingTokens, subCriteria :: stack) + case (c: Criteria) :: rest => + stack match { + case (op: PredicateOperator) :: (left: Criteria) :: tail => + val predicate = Predicate(left, op, c) + processTokensHelper(rest, predicate :: tail) + case _ => + processTokensHelper(rest, c :: stack) + } + case (op: PredicateOperator) :: rest => + stack match { + case (right: Criteria) :: (left: Criteria) :: tail => + val predicate = Predicate(left, op, right) + processTokensHelper(rest, predicate :: tail) + case (right: Criteria) :: (o: PredicateOperator) :: tail => + tail match { + case (left: Criteria) :: tt => + val predicate = Predicate(left, op, right) + processTokensHelper(rest, o :: predicate :: tt) + case _ => + processTokensHelper(rest, op :: stack) + } + case _ :: Nil => + processTokensHelper(rest, op :: stack) + case _ => + throw ValidationError("Invalid stack state for predicate creation") + } + case ThenCase :: _ => + processTokensHelper(Nil, stack) // exit processing on THEN + case (_: EndDelimiter) :: rest => + processTokensHelper(rest, stack) // Ignore and move on + case _ => processTokensHelper(Nil, stack) + } + } + + /** This method calls processTokensHelper with an empty stack (Nil) to begin processing primary + * tokens. + * + * @param tokens + * - list of SQL tokens + * @return + */ + protected def processTokens( + tokens: List[Token] + ): Option[Criteria] = { + processTokensHelper(tokens, Nil) + } + + /** This method is used to process subtokens extracted between delimiters. It calls + * processTokensHelper and returns the result as a SQLCriteria, or throws an exception if no + * criteria is found. + * + * @param tokens + * - list of SQL tokens + * @return + */ + private def processSubTokens(tokens: List[Token]): Criteria = { + processTokensHelper(tokens, Nil).getOrElse( + throw ValidationError("Empty sub-expression") + ) + } + + /** This method is used to extract subtokens between a start delimiter (StartDelimiter) and its + * corresponding end delimiter (EndDelimiter). It uses a recursive approach to maintain the count + * of open and closed delimiters and correctly construct the list of extracted subtokens. + * + * @param tokens + * - list of SQL tokens + * @param openCount + * - count of open delimiters + * @param subTokens + * - list of extracted subtokens + * @return + */ + @tailrec + private def extractSubTokens( + tokens: List[Token], + openCount: Int, + subTokens: List[Token] = Nil + ): (List[Token], List[Token]) = { + tokens match { + case Nil => throw ValidationError("Unbalanced parentheses") + case (start: StartDelimiter) :: rest => + extractSubTokens(rest, openCount + 1, start :: subTokens) + case (end: EndDelimiter) :: rest => + if (openCount - 1 == 0) { + (subTokens.reverse, rest) + } else extractSubTokens(rest, openCount - 1, end :: subTokens) + case head :: rest => extractSubTokens(rest, openCount, head :: subTokens) + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala new file mode 100644 index 00000000..a20bac13 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -0,0 +1,31 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, Avg, Count, Max, Min, Sum} +import app.softnetwork.elastic.sql.parser.Parser + +package object aggregate { + + trait AggregateParser { self: Parser => + + def count: PackratParser[AggregateFunction] = Count.regex ^^ (_ => Count) + + def min: PackratParser[AggregateFunction] = Min.regex ^^ (_ => Min) + + def max: PackratParser[AggregateFunction] = Max.regex ^^ (_ => Max) + + def avg: PackratParser[AggregateFunction] = Avg.regex ^^ (_ => Avg) + + def sum: PackratParser[AggregateFunction] = Sum.regex ^^ (_ => Sum) + + def aggregates: PackratParser[AggregateFunction] = count | min | max | avg | sum + + def identifierWithAggregation: PackratParser[Identifier] = + aggregates ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { + case a ~ _ ~ i ~ _ => + i.withFunctions(a +: i.functions) + } + + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala new file mode 100644 index 00000000..4afe71ea --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala @@ -0,0 +1,95 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.function.TransformFunction +import app.softnetwork.elastic.sql.function.cond.{ + Case, + Coalesce, + ConditionalFunction, + Else, + End, + IsNotNullFunction, + IsNullFunction, + NullIf, + Then, + When +} +import app.softnetwork.elastic.sql.{Identifier, Null, PainlessScript, Token} +import app.softnetwork.elastic.sql.parser.{ + EndCase, + Parser, + StartCase, + ThenCase, + WhenCase, + WhereParser +} + +package object cond { + + trait CondParser { self: Parser with WhereParser => + + def is_null: PackratParser[ConditionalFunction[_]] = + "(?i)isnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { + case _ ~ _ ~ i ~ _ => IsNullFunction(i) + } + + def is_notnull: PackratParser[ConditionalFunction[_]] = + "(?i)isnotnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { + case _ ~ _ ~ i ~ _ => IsNotNullFunction(i) + } + + def coalesce: PackratParser[Coalesce] = + Coalesce.regex ~ start ~ rep1sep( + valueExpr, + separator + ) ~ end ^^ { case _ ~ _ ~ ids ~ _ => + Coalesce(ids) + } + + def nullif: PackratParser[NullIf] = + NullIf.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ end ^^ { + case _ ~ _ ~ id1 ~ _ ~ id2 ~ _ => NullIf(id1, id2) + } + + def start_case: PackratParser[StartCase.type] = Case.regex ^^ (_ => StartCase) + + def when_case: PackratParser[WhenCase.type] = When.regex ^^ (_ => WhenCase) + + def then_case: PackratParser[ThenCase.type] = Then.regex ^^ (_ => ThenCase) + + def else_case: PackratParser[Else.type] = Else.regex ^^ (_ => Else) + + def end_case: PackratParser[EndCase.type] = End.regex ^^ (_ => EndCase) + + def case_condition: Parser[(PainlessScript, PainlessScript)] = + when_case ~ (whereCriteria | valueExpr) ~ then_case.? ~ valueExpr ^^ { case _ ~ c ~ _ ~ r => + c match { + case p: PainlessScript => p -> r + case rawTokens: List[Token] => + processTokens(rawTokens) match { + case Some(criteria) => criteria -> r + case _ => Null -> r + } + } + } + + def case_else: Parser[PainlessScript] = else_case ~ valueExpr ^^ { case _ ~ r => r } + + def case_when: PackratParser[Case] = + start_case ~ valueExpr.? ~ rep1(case_condition) ~ case_else.? ~ end_case ^^ { + case _ ~ e ~ c ~ r ~ _ => Case(e, c, r) + } + + def case_when_identifier: Parser[Identifier] = case_when ^^ { cw => + Identifier(cw) + } + + def conditional_functions: PackratParser[TransformFunction[_, _]] = + is_null | is_notnull | coalesce | nullif | case_when + + def conditionalFunctionWithIdentifier: PackratParser[Identifier] = + (is_null | is_notnull | coalesce | nullif) ^^ { t => + t.identifier.withFunctions(t +: t.identifier.functions) + } + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala new file mode 100644 index 00000000..0724be4f --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala @@ -0,0 +1,18 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.function.convert.Cast +import app.softnetwork.elastic.sql.{Alias, Identifier} +import app.softnetwork.elastic.sql.parser.Parser + +package object convert { + + trait ConvertParser { self: Parser => + + def castFunctionWithIdentifier: PackratParser[Identifier] = + "(?i)cast".r ~ start ~ (identifierWithTransformation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { + case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => + i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) + } + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala new file mode 100644 index 00000000..afe2af03 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala @@ -0,0 +1,14 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.function.Function +import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.parser.Parser + +package object geo { + + trait GeoParser { self: Parser => + + def distance: PackratParser[Function] = Distance.regex ^^ (_ => Distance) + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala new file mode 100644 index 00000000..b6383e8b --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala @@ -0,0 +1,104 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.function.math.{ + Abs, + Acos, + Asin, + Atan, + Atan2, + Ceil, + Cos, + Exp, + Floor, + Log, + Log10, + MathOp, + MathematicalFunction, + MathematicalFunctionWithOp, + Pow, + Round, + Sign, + Sin, + Sqrt, + Tan, + Trigonometric +} +import app.softnetwork.elastic.sql.parser.Parser + +package object math { + + trait MathParser { self: Parser => + + private[this] def abs: PackratParser[MathOp] = Abs.regex ^^ (_ => Abs) + + private[this] def ceil: PackratParser[MathOp] = Ceil.regex ^^ (_ => Ceil) + + private[this] def floor: PackratParser[MathOp] = Floor.regex ^^ (_ => Floor) + + private[this] def exp: PackratParser[MathOp] = Exp.regex ^^ (_ => Exp) + + private[this] def sqrt: PackratParser[MathOp] = Sqrt.regex ^^ (_ => Sqrt) + + private[this] def log: PackratParser[MathOp] = Log.regex ^^ (_ => Log) + + private[this] def log10: PackratParser[MathOp] = Log10.regex ^^ (_ => Log10) + + def arithmeticFunction: PackratParser[MathematicalFunction] = + (abs | ceil | exp | floor | log | log10 | sqrt) ~ start ~ valueExpr ~ end ^^ { + case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) + } + + private[this] def sin: PackratParser[Trigonometric] = Sin.regex ^^ (_ => Sin) + + private[this] def asin: PackratParser[Trigonometric] = Asin.regex ^^ (_ => Asin) + + private[this] def cos: PackratParser[Trigonometric] = Cos.regex ^^ (_ => Cos) + + private[this] def acos: PackratParser[Trigonometric] = Acos.regex ^^ (_ => Acos) + + private[this] def tan: PackratParser[Trigonometric] = Tan.regex ^^ (_ => Tan) + + private[this] def atan: PackratParser[Trigonometric] = Atan.regex ^^ (_ => Atan) + + private[this] def atan2: PackratParser[Trigonometric] = Atan2.regex ^^ (_ => Atan2) + + def atan2Function: PackratParser[MathematicalFunction] = + atan2 ~ start ~ (double | valueExpr) ~ separator ~ (double | valueExpr) ~ end ^^ { + case _ ~ _ ~ y ~ _ ~ x ~ _ => Atan2(y, x) + } + + def trigonometricFunction: PackratParser[MathematicalFunction] = + atan2Function | ((sin | asin | cos | acos | tan | atan) ~ start ~ valueExpr ~ end ^^ { + case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) + }) + + private[this] def round: PackratParser[MathOp] = Round.regex ^^ (_ => Round) + + def roundFunction: PackratParser[MathematicalFunction] = + round ~ start ~ valueExpr ~ separator.? ~ long.? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ _ => + Round(v, s.map(_.value.toInt)) + } + + private[this] def pow: PackratParser[MathOp] = Pow.regex ^^ (_ => Pow) + + def powFunction: PackratParser[MathematicalFunction] = + pow ~ start ~ valueExpr ~ separator ~ long ~ end ^^ { case _ ~ _ ~ v1 ~ _ ~ e ~ _ => + Pow(v1, e.value.toInt) + } + + private[this] def sign: PackratParser[MathOp] = Sign.regex ^^ (_ => Sign) + + def signFunction: PackratParser[MathematicalFunction] = + sign ~ start ~ valueExpr ~ end ^^ { case _ ~ _ ~ v ~ _ => Sign(v) } + + def mathematicalFunction: PackratParser[MathematicalFunction] = + arithmeticFunction | trigonometricFunction | roundFunction | powFunction | signFunction + + def mathematicalFunctionWithIdentifier: PackratParser[Identifier] = + mathematicalFunction ^^ { mf => + mf.identifier + } + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala new file mode 100644 index 00000000..536e50f7 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -0,0 +1,65 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLVarchar} +import app.softnetwork.elastic.sql.function.string.{ + Concat, + Length, + Lower, + SQLLength, + StringFunction, + StringFunctionWithOp, + Substring, + To, + Trim, + Upper +} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.From + +package object string { + + trait StringParser { self: Parser => + + def concatFunction: PackratParser[StringFunction[SQLVarchar]] = + Concat.regex ~ start ~ rep1sep(valueExpr, separator) ~ end ^^ { case _ ~ _ ~ vs ~ _ => + Concat(vs) + } + + def substringFunction: PackratParser[StringFunction[SQLVarchar]] = + Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((To.regex | separator) ~ long).? ~ end ^^ { + case _ ~ _ ~ v ~ _ ~ s ~ eOpt ~ _ => + Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) + } + + def stringFunctionWithIdentifier: PackratParser[Identifier] = + (concatFunction | substringFunction) ^^ { sf => + sf.identifier + } + + def length: PackratParser[StringFunction[SQLBigInt]] = + Length.regex ^^ { _ => + SQLLength + } + + def lower: PackratParser[StringFunction[SQLVarchar]] = + Lower.regex ^^ { _ => + StringFunctionWithOp(Lower) + } + + def upper: PackratParser[StringFunction[SQLVarchar]] = + Upper.regex ^^ { _ => + StringFunctionWithOp(Upper) + } + + def trim: PackratParser[StringFunction[SQLVarchar]] = + Trim.regex ^^ { _ => + StringFunctionWithOp(Trim) + } + + def string_functions: Parser[ + StringFunction[_] + ] = /*concatFunction | substringFunction |*/ length | lower | upper | trim + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala new file mode 100644 index 00000000..7f765ed0 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -0,0 +1,234 @@ +package app.softnetwork.elastic.sql.parser.function + +import app.softnetwork.elastic.sql.{Identifier, StringValue} +import app.softnetwork.elastic.sql.`type`.{SQLLiteral, SQLNumeric, SQLTemporal} +import app.softnetwork.elastic.sql.function.{ + BinaryFunction, + FunctionWithIdentifier, + TransformFunction +} +import app.softnetwork.elastic.sql.function.time.{ + CurentDateWithParens, + CurrentDate, + CurrentFunction, + CurrentTime, + CurrentTimeWithParens, + CurrentTimestamp, + CurrentTimestampWithParens, + DAY, + DateAdd, + DateDiff, + DateFunction, + DateSub, + DateTimeAdd, + DateTimeFunction, + DateTimeSub, + DateTrunc, + Extract, + FormatDate, + FormatDateTime, + HOUR, + MINUTE, + MONTH, + Now, + NowWithParens, + ParseDate, + ParseDateTime, + SECOND, + YEAR +} +import app.softnetwork.elastic.sql.parser.time.TimeParser +import app.softnetwork.elastic.sql.parser.{Delimiter, Parser} +import app.softnetwork.elastic.sql.time.TimeUnit.{Day, Hour, Minute, Month, Second, Year} + +package object time { + + trait SystemParser { self: Parser with TimeParser => + + def parens: PackratParser[List[Delimiter]] = + start ~ end ^^ { case s ~ e => s :: e :: Nil } + + def current_date: PackratParser[CurrentFunction] = + CurrentDate.regex ~ parens.? ^^ { case _ ~ p => + if (p.isDefined) CurentDateWithParens else CurrentDate + } + + def current_time: PackratParser[CurrentFunction] = + CurrentTime.regex ~ parens.? ^^ { case _ ~ p => + if (p.isDefined) CurrentTimeWithParens else CurrentTime + } + + def current_timestamp: PackratParser[CurrentFunction] = + CurrentTimestamp.regex ~ parens.? ^^ { case _ ~ p => + if (p.isDefined) CurrentTimestampWithParens else CurrentTimestamp + } + + def now: PackratParser[CurrentFunction] = Now.regex ~ parens.? ^^ { case _ ~ p => + if (p.isDefined) NowWithParens else Now + } + + def identifierWithSystemFunction: PackratParser[Identifier] = + (current_date | current_time | current_timestamp | now) ~ intervalFunction.? ^^ { + case f1 ~ f2 => + f2 match { + case Some(f) => Identifier(List(f, f1)) + case None => Identifier(f1) + } + } + + } + + trait DateParser { self: Parser with TemporalParser => + + def date_add: PackratParser[DateFunction with FunctionWithIdentifier] = + "(?i)date_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ t ~ _ => + DateAdd(i, t) + } + + def date_sub: PackratParser[DateFunction with FunctionWithIdentifier] = + "(?i)date_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ t ~ _ => + DateSub(i, t) + } + + def parse_date: PackratParser[DateFunction with FunctionWithIdentifier] = + "(?i)parse_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { + case _ ~ _ ~ li ~ _ ~ f ~ _ => + li match { + case l: StringValue => + ParseDate(Identifier(l), f.value) + case i: Identifier => + ParseDate(i, f.value) + } + } + + def format_date: PackratParser[DateFunction with FunctionWithIdentifier] = + "(?i)format_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ f ~ _ => + FormatDate(i, f.value) + } + + def date_functions: PackratParser[DateFunction] = date_add | date_sub | parse_date | format_date + + def dateFunctionWithIdentifier: PackratParser[Identifier] = + (parse_date | format_date | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => + af match { + case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) + case None => t.identifier.withFunctions(t +: t.identifier.functions) + } + } + + } + + trait DateTimeParser { self: Parser with TemporalParser => + + def datetime_add: PackratParser[DateTimeFunction with FunctionWithIdentifier] = + "(?i)datetime_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ t ~ _ => + DateTimeAdd(i, t) + } + + def datetime_sub: PackratParser[DateTimeFunction with FunctionWithIdentifier] = + "(?i)datetime_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ t ~ _ => + DateTimeSub(i, t) + } + + def parse_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = + "(?i)parse_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { + case _ ~ _ ~ li ~ _ ~ f ~ _ => + li match { + case l: SQLLiteral => + ParseDateTime(Identifier(l), f.value) + case i: Identifier => + ParseDateTime(i, f.value) + } + } + + def format_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = + "(?i)format_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ f ~ _ => + FormatDateTime(i, f.value) + } + + def datetime_functions: PackratParser[DateTimeFunction] = + datetime_add | datetime_sub | parse_datetime | format_datetime + + def dateTimeFunctionWithIdentifier: PackratParser[Identifier] = + (date_trunc | parse_datetime | format_datetime | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { + case t ~ af => + af match { + case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) + case None => t.identifier.withFunctions(t +: t.identifier.functions) + } + } + + } + + trait TemporalParser extends SystemParser with TimeParser with DateParser with DateTimeParser { + self: Parser => + + def identifierWithTemporalFunction: PackratParser[Identifier] = + rep1sep( + date_trunc | extractors | date_functions | datetime_functions, + start + ) ~ start.? ~ (identifierWithSystemFunction | identifier).? ~ rep( + end + ) ^^ { case f ~ _ ~ i ~ _ => + i match { + case Some(id) => id.withFunctions(id.functions ++ f) + case None => + f.lastOption match { + case Some(fi: FunctionWithIdentifier) => + fi.identifier.withFunctions(f ++ fi.identifier.functions) + case _ => Identifier(f) + } + } + } + + def date_diff: PackratParser[BinaryFunction[_, _, _]] = + "(?i)date_diff".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { + case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) + } + + def date_diff_identifier: PackratParser[Identifier] = date_diff ^^ { dd => + Identifier(dd) + } + + def date_trunc: PackratParser[FunctionWithIdentifier] = + "(?i)date_trunc".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { + case _ ~ _ ~ i ~ _ ~ u ~ _ => + DateTrunc(i, u) + } + + def extract_identifier: PackratParser[Identifier] = + "(?i)extract".r ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { + case _ ~ _ ~ u ~ _ ~ i ~ _ => + i.withFunctions(Extract(u) +: i.functions) + } + + def extract_year: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + Year.regex ^^ (_ => YEAR) + + def extract_month: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + Month.regex ^^ (_ => MONTH) + + def extract_day: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + Day.regex ^^ (_ => DAY) + + def extract_hour: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + Hour.regex ^^ (_ => HOUR) + + def extract_minute: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + Minute.regex ^^ (_ => MINUTE) + + def extract_second: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + Second.regex ^^ (_ => SECOND) + + def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + extract_year | extract_month | extract_day | extract_hour | extract_minute | extract_second + + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala new file mode 100644 index 00000000..6d8ae538 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala @@ -0,0 +1,61 @@ +package app.softnetwork.elastic.sql.parser.operator + +import app.softnetwork.elastic.sql.function.{Function, FunctionWithIdentifier} +import app.softnetwork.elastic.sql.{Identifier, PainlessScript} +import app.softnetwork.elastic.sql.operator.math.{ + Add, + ArithmeticExpression, + ArithmeticOperator, + Divide, + Modulo, + Multiply, + Subtract +} +import app.softnetwork.elastic.sql.parser.Parser + +package object math { + + trait ArithmeticParser { self: Parser => + def add: PackratParser[ArithmeticOperator] = Add.sql ^^ (_ => Add) + + def subtract: PackratParser[ArithmeticOperator] = Subtract.sql ^^ (_ => Subtract) + + def multiply: PackratParser[ArithmeticOperator] = Multiply.sql ^^ (_ => Multiply) + + def divide: PackratParser[ArithmeticOperator] = Divide.sql ^^ (_ => Divide) + + def modulo: PackratParser[ArithmeticOperator] = Modulo.sql ^^ (_ => Modulo) + + def factor: PackratParser[PainlessScript] = + "(" ~> arithmeticExpressionLevel2 <~ ")" ^^ { + case expr: ArithmeticExpression => + expr.copy(group = true) + case other => other + } | valueExpr + + def arithmeticExpressionLevel1: Parser[PainlessScript] = + factor ~ rep((multiply | divide | modulo) ~ factor) ^^ { case left ~ list => + list.foldLeft(left) { case (acc, op ~ right) => + ArithmeticExpression(acc, op, right) + } + } + + def arithmeticExpressionLevel2: Parser[PainlessScript] = + arithmeticExpressionLevel1 ~ rep((add | subtract) ~ arithmeticExpressionLevel1) ^^ { + case left ~ list => + list.foldLeft(left) { case (acc, op ~ right) => + ArithmeticExpression(acc, op, right) + } + } + + def identifierWithArithmeticExpression: Parser[Identifier] = + arithmeticExpressionLevel2 ^^ { + case af: ArithmeticExpression => Identifier(af) + case id: Identifier => id + case f: FunctionWithIdentifier => f.identifier + case f: Function => Identifier(f) + case other => throw new Exception(s"Unexpected expression $other") + } + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala new file mode 100644 index 00000000..79a54c37 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala @@ -0,0 +1,66 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.`type`.SQLTemporal +import app.softnetwork.elastic.sql.function.TransformFunction +import app.softnetwork.elastic.sql.function.time.{SQLAddInterval, SQLSubtractInterval} +import app.softnetwork.elastic.sql.time.{Interval, TimeInterval, TimeUnit} +import app.softnetwork.elastic.sql.time.TimeUnit.{ + Day, + Hour, + Minute, + Month, + Quarter, + Second, + Week, + Year +} + +package object time { + + trait TimeParser { self: Parser => + + def year: PackratParser[TimeUnit] = Year.regex ^^ (_ => Year) + + def month: PackratParser[TimeUnit] = Month.regex ^^ (_ => Month) + + def quarter: PackratParser[TimeUnit] = Quarter.regex ^^ (_ => Quarter) + + def week: PackratParser[TimeUnit] = Week.regex ^^ (_ => Week) + + def day: PackratParser[TimeUnit] = Day.regex ^^ (_ => Day) + + def hour: PackratParser[TimeUnit] = Hour.regex ^^ (_ => Hour) + + def minute: PackratParser[TimeUnit] = Minute.regex ^^ (_ => Minute) + + def second: PackratParser[TimeUnit] = Second.regex ^^ (_ => Second) + + def time_unit: PackratParser[TimeUnit] = + year | month | quarter | week | day | hour | minute | second + + def interval: PackratParser[TimeInterval] = + Interval.regex ~ long ~ time_unit ^^ { case _ ~ l ~ u => + TimeInterval(l.value.toInt, u) + } + + def add_interval: PackratParser[SQLAddInterval] = + add ~ interval ^^ { case _ ~ it => + SQLAddInterval(it) + } + + def substract_interval: PackratParser[SQLSubtractInterval] = + subtract ~ interval ^^ { case _ ~ it => + SQLSubtractInterval(it) + } + + def intervalFunction: PackratParser[TransformFunction[SQLTemporal, SQLTemporal]] = + add_interval | substract_interval + + def identifierWithIntervalFunction: PackratParser[Identifier] = + (identifierWithFunction | identifier) ~ intervalFunction ^^ { case i ~ f => + i.withFunctions(f +: i.functions) + } + + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala new file mode 100644 index 00000000..13d78aaa --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -0,0 +1,79 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.{ + BooleanValue, + DoubleValue, + Identifier, + LongValue, + PiValue, + StringValue, + Value +} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.function.math.Pi + +package object `type` { + + trait TypeParser { self: Parser => + + def literal: PackratParser[StringValue] = + (("\"" ~> """([^"\\]|\\.)*""".r <~ "\"") | ("'" ~> """([^'\\]|\\.)*""".r <~ "'")) ^^ { str => + StringValue(str) + } + + def long: PackratParser[LongValue] = + """(-)?(0|[1-9]\d*)""".r ^^ (str => LongValue(str.toLong)) + + def double: PackratParser[DoubleValue] = + """(-)?(\d+\.\d+)""".r ^^ (str => DoubleValue(str.toDouble)) + + def pi: PackratParser[Value[Double]] = + Pi.regex ^^ (_ => PiValue) + + def boolean: PackratParser[BooleanValue] = + """(true|false)""".r ^^ (bool => BooleanValue(bool.toBoolean)) + + def value_identifier: PackratParser[Identifier] = + (literal | long | double | pi | boolean) ^^ { v => + Identifier(v) + } + + def char_type: PackratParser[SQLTypes.Char.type] = + "(?i)char".r ^^ (_ => SQLTypes.Char) + + def string_type: PackratParser[SQLTypes.Varchar.type] = + "(?i)varchar|string".r ^^ (_ => SQLTypes.Varchar) + + def date_type: PackratParser[SQLTypes.Date.type] = "(?i)date".r ^^ (_ => SQLTypes.Date) + + def time_type: PackratParser[SQLTypes.Time.type] = "(?i)time".r ^^ (_ => SQLTypes.Time) + + def datetime_type: PackratParser[SQLTypes.DateTime.type] = + "(?i)(datetime)".r ^^ (_ => SQLTypes.DateTime) + + def timestamp_type: PackratParser[SQLTypes.Timestamp.type] = + "(?i)(timestamp)".r ^^ (_ => SQLTypes.Timestamp) + + def boolean_type: PackratParser[SQLTypes.Boolean.type] = + "(?i)boolean".r ^^ (_ => SQLTypes.Boolean) + + def byte_type: PackratParser[SQLTypes.TinyInt.type] = + "(?i)(byte|tinyint)".r ^^ (_ => SQLTypes.TinyInt) + + def short_type: PackratParser[SQLTypes.SmallInt.type] = + "(?i)(short|smallint)".r ^^ (_ => SQLTypes.SmallInt) + + def int_type: PackratParser[SQLTypes.Int.type] = "(?i)(integer|int)".r ^^ (_ => SQLTypes.Int) + + def long_type: PackratParser[SQLTypes.BigInt.type] = + "(?i)long|bigint".r ^^ (_ => SQLTypes.BigInt) + + def double_type: PackratParser[SQLTypes.Double.type] = "(?i)double".r ^^ (_ => SQLTypes.Double) + + def float_type: PackratParser[SQLTypes.Real.type] = "(?i)float|real".r ^^ (_ => SQLTypes.Real) + + def sql_type: PackratParser[SQLType] = + char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type + + } +} From 880412ab95a5bb85e4efe4c963001bcb6cb0af6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 09:33:06 +0200 Subject: [PATCH 09/48] update Tokens regex --- .../elastic/sql/SQLQuerySpec.scala | 4 +- .../sql/function/aggregate/package.scala | 10 +- .../elastic/sql/function/cond/package.scala | 24 +- .../sql/function/convert/package.scala | 2 +- .../elastic/sql/function/geo/package.scala | 2 +- .../elastic/sql/function/math/package.scala | 37 +- .../elastic/sql/function/string/package.scala | 19 +- .../elastic/sql/function/time/package.scala | 76 +-- .../elastic/sql/operator/package.scala | 28 +- .../app/softnetwork/elastic/sql/package.scala | 17 +- .../sql/parser/function/time/package.scala | 58 +-- .../elastic/sql/parser/type/package.scala | 5 +- .../softnetwork/elastic/sql/query/From.scala | 4 +- .../elastic/sql/query/GroupBy.scala | 2 +- .../elastic/sql/query/Having.scala | 2 +- .../softnetwork/elastic/sql/query/Limit.scala | 4 +- .../elastic/sql/query/OrderBy.scala | 6 +- .../elastic/sql/query/Select.scala | 2 +- .../softnetwork/elastic/sql/query/Where.scala | 2 +- .../elastic/sql/time/package.scala | 18 +- .../sql/SQLDateTimeFunctionSuite.scala | 8 +- .../elastic/sql/SQLParserSpec.scala | 488 ++++++++++++------ 22 files changed, 493 insertions(+), 325 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index ad39b30c..54399bdf 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1190,7 +1190,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle parse_date function" in { val select: ElasticSearchRequest = - SQLQuery(parseDate) + SQLQuery(dateParse) val query = select.query println(query) query shouldBe @@ -1256,7 +1256,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle parse_datetime function" in { val select: ElasticSearchRequest = - SQLQuery(parseDateTime) + SQLQuery(dateTimeParse) val query = select.query println(query) query shouldBe diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 58dda60a..a002f9f3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -6,14 +6,14 @@ package object aggregate { sealed trait AggregateFunction extends Function - case object Count extends Expr("count") with AggregateFunction + case object Count extends Expr("COUNT") with AggregateFunction - case object Min extends Expr("min") with AggregateFunction + case object Min extends Expr("MIN") with AggregateFunction - case object Max extends Expr("max") with AggregateFunction + case object Max extends Expr("MAX") with AggregateFunction - case object Avg extends Expr("avg") with AggregateFunction + case object Avg extends Expr("AVG") with AggregateFunction - case object Sum extends Expr("sum") with AggregateFunction + case object Sum extends Expr("SUM") with AggregateFunction } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 59f0ed06..346d0dc6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -10,18 +10,18 @@ package object cond { override def painless: String = sql } - case object Coalesce extends Expr("coalesce") with ConditionalOp - case object IsNullFunction extends Expr("isnull") with ConditionalOp - case object IsNotNullFunction extends Expr("isnotnull") with ConditionalOp - case object NullIf extends Expr("nullif") with ConditionalOp - case object Exists extends Expr("exists") with ConditionalOp - - case object Case extends Expr("case") with ConditionalOp - - case object When extends Expr("when") with TokenRegex - case object Then extends Expr("then") with TokenRegex - case object Else extends Expr("else") with TokenRegex - case object End extends Expr("end") with TokenRegex + case object Coalesce extends Expr("COALESCE") with ConditionalOp + case object IsNullFunction extends Expr("ISNULL") with ConditionalOp + case object IsNotNullFunction extends Expr("ISNOTNULL") with ConditionalOp + case object NullIf extends Expr("NULLIF") with ConditionalOp + case object Exists extends Expr("EXISTS") with ConditionalOp + + case object Case extends Expr("CASE") with ConditionalOp + + case object When extends Expr("WHEN") with TokenRegex + case object Then extends Expr("THEN") with TokenRegex + case object Else extends Expr("ELSE") with TokenRegex + case object End extends Expr("END") with TokenRegex sealed trait ConditionalFunction[In <: SQLType] extends TransformFunction[In, SQLBool] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index c9185020..aeda9728 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -5,7 +5,7 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} package object convert { - case object Cast extends Expr("cast") with TokenRegex + case object Cast extends Expr("CAST") with TokenRegex case class Cast(value: PainlessScript, targetType: SQLType, as: Boolean = true) extends TransformFunction[SQLType, SQLType] { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 308ac591..115ba8c4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -5,6 +5,6 @@ import app.softnetwork.elastic.sql.operator.Operator package object geo { - case object Distance extends Expr("distance") with Function with Operator + case object Distance extends Expr("DISTANCE") with Function with Operator } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 2470f507..18b21008 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -10,29 +10,26 @@ package object math { override def toString: String = s" $sql " } - case object Abs extends Expr("abs") with MathOp - case object Ceil extends Expr("ceil") with MathOp - case object Floor extends Expr("floor") with MathOp - case object Round extends Expr("round") with MathOp - case object Exp extends Expr("exp") with MathOp - case object Log extends Expr("log") with MathOp - case object Log10 extends Expr("log10") with MathOp - case object Pow extends Expr("pow") with MathOp - case object Sqrt extends Expr("sqrt") with MathOp - case object Sign extends Expr("sign") with MathOp - case object Pi extends Expr("pi") with MathOp { - override def painless: String = "Math.PI" - } + case object Abs extends Expr("ABS") with MathOp + case object Ceil extends Expr("CEIL") with MathOp + case object Floor extends Expr("FLOOR") with MathOp + case object Round extends Expr("ROUND") with MathOp + case object Exp extends Expr("EXP") with MathOp + case object Log extends Expr("LOG") with MathOp + case object Log10 extends Expr("LOG10") with MathOp + case object Pow extends Expr("POW") with MathOp + case object Sqrt extends Expr("SQRT") with MathOp + case object Sign extends Expr("SIGN") with MathOp sealed trait Trigonometric extends MathOp - case object Sin extends Expr("sin") with Trigonometric - case object Asin extends Expr("asin") with Trigonometric - case object Cos extends Expr("cos") with Trigonometric - case object Acos extends Expr("acos") with Trigonometric - case object Tan extends Expr("tan") with Trigonometric - case object Atan extends Expr("atan") with Trigonometric - case object Atan2 extends Expr("atan2") with Trigonometric + case object Sin extends Expr("SIN") with Trigonometric + case object Asin extends Expr("ASIN") with Trigonometric + case object Cos extends Expr("COS") with Trigonometric + case object Acos extends Expr("ACOS") with Trigonometric + case object Tan extends Expr("TAN") with Trigonometric + case object Atan extends Expr("ATAN") with Trigonometric + case object Atan2 extends Expr("ATAN2") with Trigonometric sealed trait MathematicalFunction extends TransformFunction[SQLNumeric, SQLNumeric] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 5cccc63c..6bd36bef 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -9,19 +9,20 @@ package object string { override def painless: String = s".${sql.toLowerCase()}()" } - case object Concat extends Expr("concat") with StringOp { + case object Concat extends Expr("CONCAT") with StringOp { override def painless: String = " + " } - case object Lower extends Expr("lower") with StringOp - case object Upper extends Expr("upper") with StringOp - case object Trim extends Expr("trim") with StringOp - //case object LTrim extends SQLExpr("ltrim") with SQLStringOperator - //case object RTrim extends SQLExpr("rtrim") with SQLStringOperator - case object Substring extends Expr("substring") with StringOp { + case object Lower extends Expr("LOWER") with StringOp + case object Upper extends Expr("UPPER") with StringOp + case object Trim extends Expr("TRIM") with StringOp + //case object LTrim extends SQLExpr("LTRIM") with SQLStringOperator + //case object RTrim extends SQLExpr("RTRIM") with SQLStringOperator + case object Substring extends Expr("SUBSTRING") with StringOp { override def painless: String = ".substring" + override lazy val words: List[String] = List(sql, "SUBSTR") } - case object To extends Expr("to") with TokenRegex - case object Length extends Expr("length") with StringOp + case object To extends Expr("TO") with TokenRegex + case object Length extends Expr("LENGTH") with StringOp sealed trait StringFunction[Out <: SQLType] extends TransformFunction[SQLVarchar, Out] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index ce3d372a..f23622cc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -107,26 +107,31 @@ package object time { override def painless: String = s"$now.toLocalTime()" } - case object CurrentDate extends Expr("current_date") with CurrentDateFunction + case object CurrentDate extends Expr("CURRENT_DATE") with CurrentDateFunction { + override lazy val words: List[String] = List(sql, "CURDATE") + } - case object CurentDateWithParens extends Expr("current_date()") with CurrentDateFunction + case object CurentDateWithParens extends Expr("CURRENT_DATE()") with CurrentDateFunction - case object CurrentTime extends Expr("current_time") with CurrentTimeFunction + case object CurrentTime extends Expr("CURRENT_TIME") with CurrentTimeFunction { + override lazy val words: List[String] = List(sql, "CURTIME") + } - case object CurrentTimeWithParens extends Expr("current_time()") with CurrentTimeFunction + case object CurrentTimeWithParens extends Expr("CURRENT_TIME()") with CurrentTimeFunction - case object CurrentTimestamp extends Expr("current_timestamp") with CurrentDateTimeFunction + case object CurrentTimestamp extends Expr("CURRENT_TIMESTAMP") with CurrentDateTimeFunction case object CurrentTimestampWithParens - extends Expr("current_timestamp()") + extends Expr("CURRENT_TIMESTAMP()") with CurrentDateTimeFunction - case object Now extends Expr("now") with CurrentDateTimeFunction + case object Now extends Expr("NOW") with CurrentDateTimeFunction - case object NowWithParens extends Expr("now()") with CurrentDateTimeFunction + case object NowWithParens extends Expr("NOW()") with CurrentDateTimeFunction - case object DateTrunc extends Expr("date_trunc") with TokenRegex with PainlessScript { + case object DateTrunc extends Expr("DATE_TRUNC") with TokenRegex with PainlessScript { override def painless: String = ".truncatedTo" + override lazy val words: List[String] = List(sql, "DATETRUNC") } case class DateTrunc(identifier: Identifier, unit: TimeUnit) @@ -146,7 +151,7 @@ package object time { } } - case object Extract extends Expr("extract") with TokenRegex with PainlessScript { + case object Extract extends Expr("EXTRACT") with TokenRegex with PainlessScript { override def painless: String = ".get" } @@ -190,8 +195,9 @@ package object time { override def toSQL(base: String): String = s"$sql($base)" } - case object DateDiff extends Expr("date_diff") with TokenRegex with PainlessScript { + case object DateDiff extends Expr("DATE_DIFF") with TokenRegex with PainlessScript { override def painless: String = ".between" + override lazy val words: List[String] = List(sql, "DATEDIFF") } case class DateDiff(end: PainlessScript, start: PainlessScript, unit: TimeUnit) @@ -214,7 +220,9 @@ package object time { s"${unit.painless}${DateDiff.painless}(${callArgs.mkString(", ")})" } - case object DateAdd extends Expr("date_add") with TokenRegex + case object DateAdd extends Expr("DATE_ADD") with TokenRegex { + override lazy val words: List[String] = List(sql, "DATEADD") + } case class DateAdd(identifier: Identifier, interval: TimeInterval) extends DateFunction @@ -229,7 +237,9 @@ package object time { } } - case object DateSub extends Expr("date_sub") with TokenRegex + case object DateSub extends Expr("DATE_SUB") with TokenRegex { + override lazy val words: List[String] = List(sql, "DATESUB") + } case class DateSub(identifier: Identifier, interval: TimeInterval) extends DateFunction @@ -244,22 +254,22 @@ package object time { } } - case object ParseDate extends Expr("parse_date") with TokenRegex with PainlessScript { + case object DateParse extends Expr("DATE_PARSE") with TokenRegex with PainlessScript { override def painless: String = ".parse" } - case class ParseDate(identifier: Identifier, format: String) + case class DateParse(identifier: Identifier, format: String) extends DateFunction with TransformFunction[SQLVarchar, SQLDate] with FunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(ParseDate) + override def fun: Option[PainlessScript] = Some(DateParse) override def args: List[PainlessScript] = List.empty override def inputType: SQLVarchar = SQLTypes.Varchar override def outputType: SQLDate = SQLTypes.Date - override def sql: String = ParseDate.sql + override def sql: String = DateParse.sql override def toSQL(base: String): String = { s"$sql($base, '$format')" } @@ -272,22 +282,22 @@ package object time { s"DateTimeFormatter.ofPattern('$format').parse($base, LocalDate::from)" } - case object FormatDate extends Expr("format_date") with TokenRegex with PainlessScript { + case object DateFormat extends Expr("DATE_FORMAT") with TokenRegex with PainlessScript { override def painless: String = ".format" } - case class FormatDate(identifier: Identifier, format: String) + case class DateFormat(identifier: Identifier, format: String) extends DateFunction with TransformFunction[SQLDate, SQLVarchar] with FunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(FormatDate) + override def fun: Option[PainlessScript] = Some(DateFormat) override def args: List[PainlessScript] = List.empty override def inputType: SQLDate = SQLTypes.Date override def outputType: SQLVarchar = SQLTypes.Varchar - override def sql: String = FormatDate.sql + override def sql: String = DateFormat.sql override def toSQL(base: String): String = { s"$sql($base, '$format')" } @@ -300,7 +310,9 @@ package object time { s"DateTimeFormatter.ofPattern('$format').format($base)" } - case object DateTimeAdd extends Expr("datetime_add") with TokenRegex + case object DateTimeAdd extends Expr("DATETIME_ADD") with TokenRegex { + override lazy val words: List[String] = List(sql, "DATETIMEADD") + } case class DateTimeAdd(identifier: Identifier, interval: TimeInterval) extends DateTimeFunction @@ -315,7 +327,9 @@ package object time { } } - case object DateTimeSub extends Expr("datetime_sub") with TokenRegex + case object DateTimeSub extends Expr("DATETIME_SUB") with TokenRegex { + override lazy val words: List[String] = List(sql, "DATETIMESUB") + } case class DateTimeSub(identifier: Identifier, interval: TimeInterval) extends DateTimeFunction @@ -330,22 +344,22 @@ package object time { } } - case object ParseDateTime extends Expr("parse_datetime") with TokenRegex with PainlessScript { + case object DateTimeParse extends Expr("DATETIME_PARSE") with TokenRegex with PainlessScript { override def painless: String = ".parse" } - case class ParseDateTime(identifier: Identifier, format: String) + case class DateTimeParse(identifier: Identifier, format: String) extends DateTimeFunction with TransformFunction[SQLVarchar, SQLDateTime] with FunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(ParseDateTime) + override def fun: Option[PainlessScript] = Some(DateTimeParse) override def args: List[PainlessScript] = List.empty override def inputType: SQLVarchar = SQLTypes.Varchar override def outputType: SQLDateTime = SQLTypes.DateTime - override def sql: String = ParseDateTime.sql + override def sql: String = DateTimeParse.sql override def toSQL(base: String): String = { s"$sql($base, '$format')" } @@ -358,22 +372,22 @@ package object time { s"DateTimeFormatter.ofPattern('$format').parse($base, ZonedDateTime::from)" } - case object FormatDateTime extends Expr("format_datetime") with TokenRegex with PainlessScript { + case object DateTimeFormat extends Expr("DATETIME_FORMAT") with TokenRegex with PainlessScript { override def painless: String = ".format" } - case class FormatDateTime(identifier: Identifier, format: String) + case class DateTimeFormat(identifier: Identifier, format: String) extends DateTimeFunction with TransformFunction[SQLDateTime, SQLVarchar] with FunctionWithIdentifier { - override def fun: Option[PainlessScript] = Some(FormatDateTime) + override def fun: Option[PainlessScript] = Some(DateTimeFormat) override def args: List[PainlessScript] = List.empty override def inputType: SQLDateTime = SQLTypes.DateTime override def outputType: SQLVarchar = SQLTypes.Varchar - override def sql: String = FormatDateTime.sql + override def sql: String = DateTimeFormat.sql override def toSQL(base: String): String = { s"$sql($base, '$format')" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala index 3c09b572..409aa985 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala @@ -39,29 +39,29 @@ package object operator { case object Gt extends Expr(">") with ComparisonOperator case object Le extends Expr("<=") with ComparisonOperator case object Lt extends Expr("<") with ComparisonOperator - case object In extends Expr("in") with ComparisonOperator - case object Like extends Expr("like") with ComparisonOperator - case object Between extends Expr("between") with ComparisonOperator - case object IsNull extends Expr("is null") with ComparisonOperator - case object IsNotNull extends Expr("is not null") with ComparisonOperator + case object In extends Expr("IN") with ComparisonOperator + case object Like extends Expr("LIKE") with ComparisonOperator + case object Between extends Expr("BETWEEN") with ComparisonOperator + case object IsNull extends Expr("IS NULL") with ComparisonOperator + case object IsNotNull extends Expr("IS NOT NULL") with ComparisonOperator - case object Match extends Expr("match") with ComparisonOperator - case object Against extends Expr("against") with TokenRegex + case object Match extends Expr("MATCH") with ComparisonOperator + case object Against extends Expr("AGAINST") with TokenRegex sealed trait LogicalOperator extends ExpressionOperator - case object Not extends Expr("not") with LogicalOperator + case object Not extends Expr("NOT") with LogicalOperator sealed trait PredicateOperator extends LogicalOperator - case object And extends Expr("and") with PredicateOperator - case object Or extends Expr("or") with PredicateOperator + case object And extends Expr("AND") with PredicateOperator + case object Or extends Expr("OR") with PredicateOperator - case object Union extends Expr("union") with Operator with TokenRegex + case object Union extends Expr("UNION") with Operator with TokenRegex sealed trait ElasticOperator extends Operator with TokenRegex - case object Nested extends Expr("nested") with ElasticOperator - case object Child extends Expr("child") with ElasticOperator - case object Parent extends Expr("parent") with ElasticOperator + case object Nested extends Expr("NESTED") with ElasticOperator + case object Child extends Expr("CHILD") with ElasticOperator + case object Parent extends Expr("PARENT") with ElasticOperator } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 424ceb73..b80f94c5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -84,8 +84,8 @@ package object sql { override def nullable: Boolean = false } - case object Null extends Value[Null](null) { - override def sql: String = "null" + case object Null extends Value[Null](null) with TokenRegex { + override def sql: String = "NULL" override def painless: String = "null" override def nullable: Boolean = true override def out: SQLType = SQLTypes.Null @@ -188,14 +188,14 @@ package object sql { override def out: SQLNumeric = SQLTypes.Double } - case object PiValue extends Value[Double](Math.PI) { - override def sql: String = "pi" + case object PiValue extends Value[Double](Math.PI) with TokenRegex { + override def sql: String = "PI" override def painless: String = "Math.PI" override def out: SQLNumeric = SQLTypes.Double } - case object EValue extends Value[Double](Math.E) { - override def sql: String = "e" + case object EValue extends Value[Double](Math.E) with TokenRegex { + override def sql: String = "E" override def painless: String = "Math.E" override def out: SQLNumeric = SQLTypes.Double } @@ -327,7 +327,7 @@ package object sql { s"""${if (startWith) ".*"}$v${if (endWith) ".*"}""" } - case object Alias extends Expr("as") with TokenRegex + case object Alias extends Expr("AS") with TokenRegex case class Alias(alias: String) extends Expr(s" ${Alias.sql} $alias") @@ -367,7 +367,8 @@ package object sql { } trait TokenRegex extends Token { - lazy val regex: Regex = s"\\b(?i)$sql\\b".r + def words: List[String] = List(sql) + lazy val regex: Regex = s"(?i)(${words.mkString("|")})\\b".r } trait Source extends Updateable { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index 7f765ed0..51759120 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -18,22 +18,22 @@ import app.softnetwork.elastic.sql.function.time.{ DAY, DateAdd, DateDiff, + DateFormat, DateFunction, + DateParse, DateSub, DateTimeAdd, + DateTimeFormat, DateTimeFunction, + DateTimeParse, DateTimeSub, DateTrunc, Extract, - FormatDate, - FormatDateTime, HOUR, MINUTE, MONTH, Now, NowWithParens, - ParseDate, - ParseDateTime, SECOND, YEAR } @@ -81,38 +81,38 @@ package object time { trait DateParser { self: Parser with TemporalParser => def date_add: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)date_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateAdd.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateAdd(i, t) } def date_sub: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)date_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateSub.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateSub(i, t) } - def parse_date: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)parse_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { + def date_parse: PackratParser[DateFunction with FunctionWithIdentifier] = + DateParse.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: StringValue => - ParseDate(Identifier(l), f.value) + DateParse(Identifier(l), f.value) case i: Identifier => - ParseDate(i, f.value) + DateParse(i, f.value) } } - def format_date: PackratParser[DateFunction with FunctionWithIdentifier] = - "(?i)format_date".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { + def date_format: PackratParser[DateFunction with FunctionWithIdentifier] = + DateFormat.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ f ~ _ => - FormatDate(i, f.value) + DateFormat(i, f.value) } - def date_functions: PackratParser[DateFunction] = date_add | date_sub | parse_date | format_date + def date_functions: PackratParser[DateFunction] = date_add | date_sub | date_parse | date_format def dateFunctionWithIdentifier: PackratParser[Identifier] = - (parse_date | format_date | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => + (date_parse | date_format | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => af match { case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) case None => t.identifier.withFunctions(t +: t.identifier.functions) @@ -124,39 +124,39 @@ package object time { trait DateTimeParser { self: Parser with TemporalParser => def datetime_add: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)datetime_add".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateTimeAdd.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateTimeAdd(i, t) } def datetime_sub: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)datetime_sub".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateTimeSub.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateTimeSub(i, t) } - def parse_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)parse_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { + def datetime_parse: PackratParser[DateTimeFunction with FunctionWithIdentifier] = + DateTimeParse.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: SQLLiteral => - ParseDateTime(Identifier(l), f.value) + DateTimeParse(Identifier(l), f.value) case i: Identifier => - ParseDateTime(i, f.value) + DateTimeParse(i, f.value) } } - def format_datetime: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - "(?i)format_datetime".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { + def datetime_format: PackratParser[DateTimeFunction with FunctionWithIdentifier] = + DateTimeFormat.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ f ~ _ => - FormatDateTime(i, f.value) + DateTimeFormat(i, f.value) } def datetime_functions: PackratParser[DateTimeFunction] = - datetime_add | datetime_sub | parse_datetime | format_datetime + datetime_add | datetime_sub | datetime_parse | datetime_format def dateTimeFunctionWithIdentifier: PackratParser[Identifier] = - (date_trunc | parse_datetime | format_datetime | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { + (date_trunc | datetime_parse | datetime_format | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { case t ~ af => af match { case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) @@ -188,7 +188,7 @@ package object time { } def date_diff: PackratParser[BinaryFunction[_, _, _]] = - "(?i)date_diff".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { + DateDiff.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) } @@ -197,13 +197,13 @@ package object time { } def date_trunc: PackratParser[FunctionWithIdentifier] = - "(?i)date_trunc".r ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { + DateTrunc.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { case _ ~ _ ~ i ~ _ ~ u ~ _ => DateTrunc(i, u) } def extract_identifier: PackratParser[Identifier] = - "(?i)extract".r ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { + Extract.regex ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => i.withFunctions(Extract(u) +: i.functions) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index 13d78aaa..e98c9822 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -10,7 +10,6 @@ import app.softnetwork.elastic.sql.{ Value } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} -import app.softnetwork.elastic.sql.function.math.Pi package object `type` { @@ -28,10 +27,10 @@ package object `type` { """(-)?(\d+\.\d+)""".r ^^ (str => DoubleValue(str.toDouble)) def pi: PackratParser[Value[Double]] = - Pi.regex ^^ (_ => PiValue) + PiValue.regex ^^ (_ => PiValue) def boolean: PackratParser[BooleanValue] = - """(true|false)""".r ^^ (bool => BooleanValue(bool.toBoolean)) + """(?i)(true|false)\\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) def value_identifier: PackratParser[Identifier] = (literal | long | double | pi | boolean) ^^ { v => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 6424ec9d..7b8651eb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -10,9 +10,9 @@ import app.softnetwork.elastic.sql.{ Updateable } -case object From extends Expr("from") with TokenRegex +case object From extends Expr("FROM") with TokenRegex -case object Unnest extends Expr("unnest") with TokenRegex +case object Unnest extends Expr("UNNEST") with TokenRegex case class Unnest(identifier: Identifier, limit: Option[Limit]) extends Source { override def sql: String = s"$Unnest($identifier${asString(limit)})" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 7f9dc01b..7504beaf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -4,7 +4,7 @@ import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} -case object GroupBy extends Expr("group by") with TokenRegex +case object GroupBy extends Expr("GROUP BY") with TokenRegex case class GroupBy(buckets: Seq[Bucket]) extends Updateable { override def sql: String = s" $GroupBy ${buckets.mkString(", ")}" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 6459d8d2..73b2cd1b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.{Expr, TokenRegex, Updateable} -case object Having extends Expr("having") with TokenRegex +case object Having extends Expr("HAVING") with TokenRegex case class Having(criteria: Option[Criteria]) extends Updateable { override def sql: String = criteria match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala index bf303cf1..f1e421ab 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala @@ -2,6 +2,6 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.{Expr, TokenRegex} -case object Limit extends Expr("limit") with TokenRegex +case object Limit extends Expr("LIMIT") with TokenRegex -case class Limit(limit: Int) extends Expr(s" limit $limit") +case class Limit(limit: Int) extends Expr(s" LIMIT $limit") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala index 92ca6b91..1cf69ef6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala @@ -3,13 +3,13 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.function.{Function, FunctionChain} import app.softnetwork.elastic.sql.{Expr, Token, TokenRegex} -case object OrderBy extends Expr("order by") with TokenRegex +case object OrderBy extends Expr("ORDER BY") with TokenRegex sealed trait SortOrder extends TokenRegex -case object Desc extends Expr("desc") with SortOrder +case object Desc extends Expr("DESC") with SortOrder -case object Asc extends Expr("asc") with SortOrder +case object Asc extends Expr("ASC") with SortOrder case class FieldSort( field: String, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 81f51d8e..80d7a6b9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -12,7 +12,7 @@ import app.softnetwork.elastic.sql.{ Updateable } -case object Select extends Expr("select") with TokenRegex +case object Select extends Expr("SELECT") with TokenRegex case class Field( identifier: Identifier, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 72e6608c..67ac671c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -14,7 +14,7 @@ import app.softnetwork.elastic.sql._ import scala.annotation.tailrec -case object Where extends Expr("where") with TokenRegex +case object Where extends Expr("WHERE") with TokenRegex sealed trait Criteria extends Updateable with PainlessScript { def operator: Operator diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index a92d9ca8..a8379b3e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -18,38 +18,38 @@ package object time { sealed trait FixedUnit extends TimeUnit object TimeUnit { - case object Year extends Expr("year") with CalendarUnit { + case object Year extends Expr("YEAR") with CalendarUnit { override def script: String = "y" } - case object Month extends Expr("month") with CalendarUnit { + case object Month extends Expr("MONTH") with CalendarUnit { override def script: String = "M" } - case object Quarter extends Expr("quarter") with CalendarUnit { + case object Quarter extends Expr("QUARTER") with CalendarUnit { override def script: String = throw new IllegalArgumentException( "Quarter must be converted to months (value * 3) before creating date-math" ) } - case object Week extends Expr("week") with CalendarUnit { + case object Week extends Expr("WEEK") with CalendarUnit { override def script: String = "w" } - case object Day extends Expr("day") with CalendarUnit with FixedUnit { + case object Day extends Expr("DAY") with CalendarUnit with FixedUnit { override def script: String = "d" } - case object Hour extends Expr("hour") with FixedUnit { + case object Hour extends Expr("HOUR") with FixedUnit { override def script: String = "H" } - case object Minute extends Expr("minute") with FixedUnit { + case object Minute extends Expr("MINUTE") with FixedUnit { override def script: String = "m" } - case object Second extends Expr("second") with FixedUnit { + case object Second extends Expr("SECOND") with FixedUnit { override def script: String = "s" } } - case object Interval extends Expr("interval") with TokenRegex + case object Interval extends Expr("INTERVAL") with TokenRegex sealed trait TimeInterval extends PainlessScript with MathScript { def value: Int diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala index 5ed67b1b..2fcd8e9f 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala @@ -14,16 +14,16 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { // Liste de toutes les fonctions transformables avec leurs types val transformFunctions: Seq[TransformFunction[_, _]] = Seq( - ParseDate(Identifier(), "yyyy-MM-dd"), - ParseDateTime(Identifier(), "yyyy-MM-dd HH:mm:ss"), + DateParse(Identifier(), "yyyy-MM-dd"), + DateTimeParse(Identifier(), "yyyy-MM-dd HH:mm:ss"), DateAdd(Identifier(), TimeInterval(1, Day)), DateSub(Identifier(), TimeInterval(2, Month)), DateTimeAdd(Identifier(), TimeInterval(3, Hour)), DateTimeSub(Identifier(), TimeInterval(30, Minute)), DateTrunc(Identifier(), Day), Extract(Day), - FormatDate(Identifier(), "yyyy-MM-dd"), - FormatDateTime(Identifier(), "yyyy-MM-dd HH:mm:ss"), + DateFormat(Identifier(), "yyyy-MM-dd"), + DateTimeFormat(Identifier(), "yyyy-MM-dd HH:mm:ss"), YEAR, MONTH, DAY, diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 88d65755..f5b03f60 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -98,14 +98,14 @@ object Queries { |having Country <> 'USA' and City != 'Berlin' and count(CustomerID) > 1 and lastSeen > now - interval 7 day |order by Country asc""".stripMargin .replaceAll("\n", " ") - val parseDate = - "select identifier, count(identifier2) as ct, max(parse_date(createdAt, 'yyyy-MM-dd')) as lastSeen from Table where identifier2 is not null group by identifier order by count(identifier2) desc" - val parseDateTime: String = + val dateParse = + "select identifier, count(identifier2) as ct, max(date_parse(createdAt, 'yyyy-MM-dd')) as lastSeen from Table where identifier2 is not null group by identifier order by count(identifier2) desc" + val dateTimeParse: String = """select identifier, count(identifier2) as ct, |max( |year( |date_trunc( - |parse_datetime( + |datetime_parse( |createdAt, |'yyyy-MM-ddTHH:mm:ssZ' |), minute))) as lastSeen @@ -120,12 +120,12 @@ object Queries { val dateDiff = "select date_diff(createdAt, updatedAt, day) as diff, identifier from Table" val aggregationWithDateDiff = - "select max(date_diff(parse_datetime(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, day)) as max_diff from Table group by identifier" + "select max(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, day)) as max_diff from Table group by identifier" - val formatDate = - "select identifier, format_date(date_trunc(lastUpdated, month), 'yyyy-MM-dd') as lastSeen from Table where identifier2 is not null" - val formatDateTime = - "select identifier, format_datetime(date_trunc(lastUpdated, month), 'yyyy-MM-ddThh:mm:ssZ') as lastSeen from Table where identifier2 is not null" + val dateFormat = + "select identifier, date_format(date_trunc(lastUpdated, month), 'yyyy-MM-dd') as lastSeen from Table where identifier2 is not null" + val dateTimeFormat = + "select identifier, datetime_format(date_trunc(lastUpdated, month), 'yyyy-MM-ddThh:mm:ssZ') as lastSeen from Table where identifier2 is not null" val dateAdd = "select identifier, date_add(lastUpdated, interval 10 day) as lastSeen from Table where identifier2 is not null" val dateSub = @@ -142,9 +142,9 @@ object Queries { val coalesce: String = "select coalesce(createdAt - interval 35 minute, current_date) as c, identifier from Table" val nullif: String = - "select coalesce(nullif(createdAt, parse_date('2025-09-11', 'yyyy-MM-dd') - interval 2 day), current_date) as c, identifier from Table" + "select coalesce(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd') - interval 2 day), current_date) as c, identifier from Table" val cast: String = - "select cast(coalesce(nullif(createdAt, parse_date('2025-09-11', 'yyyy-MM-dd')), current_date - interval 2 hour) bigint) as c, identifier from Table" + "select cast(coalesce(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd')), current_date - interval 2 hour) bigint) as c, identifier from Table" val allCasts = "select cast(identifier as int) as c1, cast(identifier as bigint) as c2, cast(identifier as double) as c3, cast(identifier as real) as c4, cast(identifier as boolean) as c5, cast(identifier as char) as c6, cast(identifier as varchar) as c7, cast(createdAt as date) as c8, cast(createdAt as time) as c9, cast(createdAt as datetime) as c10, cast(createdAt as timestamp) as c11, cast(identifier as smallint) as c12, cast(identifier as tinyint) as c13 from Table" val caseWhen: String = @@ -173,431 +173,583 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { "SQLParser" should "parse numerical eq" in { val result = Parser(numericalEq) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalEq) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(numericalEq) shouldBe true } it should "parse numerical ne" in { val result = Parser(numericalNe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalNe) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(numericalNe) shouldBe true } it should "parse numerical lt" in { val result = Parser(numericalLt) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalLt) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(numericalLt) shouldBe true } it should "parse numerical le" in { val result = Parser(numericalLe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalLe) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(numericalLe) shouldBe true } it should "parse numerical gt" in { val result = Parser(numericalGt) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalGt) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(numericalGt) shouldBe true } it should "parse numerical ge" in { val result = Parser(numericalGe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(numericalGe) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(numericalGe) shouldBe true } it should "parse literal eq" in { val result = Parser(literalEq) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalEq) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalEq) shouldBe true } it should "parse literal like" in { val result = Parser(literalLike) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalLike) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalLike) shouldBe true } it should "parse literal not like" in { val result = Parser(literalNotLike) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalNotLike) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalNotLike) shouldBe true } it should "parse literal ne" in { val result = Parser(literalNe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalNe) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalNe) shouldBe true } it should "parse literal lt" in { val result = Parser(literalLt) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalLt) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalLt) shouldBe true } it should "parse literal le" in { val result = Parser(literalLe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalLe) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalLe) shouldBe true } it should "parse literal gt" in { val result = Parser(literalGt) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalGt) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalGt) shouldBe true } it should "parse literal ge" in { val result = Parser(literalGe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(literalGe) + result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") equalsIgnoreCase literalGe } it should "parse boolean eq" in { val result = Parser(boolEq) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(boolEq) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(boolEq) shouldBe true } it should "parse boolean ne" in { val result = Parser(boolNe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(boolNe) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(boolNe) shouldBe true } it should "parse between" in { val result = Parser(betweenExpression) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(betweenExpression) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(betweenExpression) shouldBe true } it should "parse and predicate" in { val result = Parser(andPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(andPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(andPredicate) shouldBe true } it should "parse or predicate" in { val result = Parser(orPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(orPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(orPredicate) shouldBe true } it should "parse left predicate with criteria" in { val result = Parser(leftPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(leftPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(leftPredicate) shouldBe true } it should "parse right predicate with criteria" in { val result = Parser(rightPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(rightPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(rightPredicate) shouldBe true } it should "parse multiple predicates" in { val result = Parser(predicates) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(predicates) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(predicates) shouldBe true } it should "parse nested predicate" in { val result = Parser(nestedPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(nestedPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(nestedPredicate) shouldBe true } it should "parse nested criteria" in { val result = Parser(nestedCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(nestedCriteria) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(nestedCriteria) shouldBe true } it should "parse child predicate" in { val result = Parser(childPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(childPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(childPredicate) shouldBe true } it should "parse child criteria" in { val result = Parser(childCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(childCriteria) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(childCriteria) shouldBe true } it should "parse parent predicate" in { val result = Parser(parentPredicate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(parentPredicate) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(parentPredicate) shouldBe true } it should "parse parent criteria" in { val result = Parser(parentCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(parentCriteria) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(parentCriteria) shouldBe true } it should "parse in literal expression" in { val result = Parser(inLiteralExpression) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - inLiteralExpression - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(inLiteralExpression) shouldBe true } it should "parse in numerical expression with Int values" in { val result = Parser(inNumericalExpressionWithIntValues) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - inNumericalExpressionWithIntValues - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(inNumericalExpressionWithIntValues) shouldBe true } it should "parse in numerical expression with Double values" in { val result = Parser(inNumericalExpressionWithDoubleValues) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - inNumericalExpressionWithDoubleValues - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(inNumericalExpressionWithDoubleValues) shouldBe true } it should "parse not in literal expression" in { val result = Parser(notInLiteralExpression) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - notInLiteralExpression - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(notInLiteralExpression) shouldBe true } it should "parse not in numerical expression with Int values" in { val result = Parser(notInNumericalExpressionWithIntValues) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - notInNumericalExpressionWithIntValues - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(notInNumericalExpressionWithIntValues) shouldBe true } it should "parse not in numerical expression with Double values" in { val result = Parser(notInNumericalExpressionWithDoubleValues) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - notInNumericalExpressionWithDoubleValues - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(notInNumericalExpressionWithDoubleValues) shouldBe true } it should "parse nested with between" in { val result = Parser(nestedWithBetween) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(nestedWithBetween) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(nestedWithBetween) shouldBe true } it should "parse count" in { val result = Parser(count) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(count) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(count) shouldBe true } it should "parse distinct count" in { val result = Parser(countDistinct) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(countDistinct) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(countDistinct) shouldBe true } it should "parse count with nested criteria" in { val result = Parser(countNested) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(countNested) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(countNested) shouldBe true } it should "parse is null" in { val result = Parser(isNull) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(isNull) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(isNull) shouldBe true } it should "parse is not null" in { val result = Parser(isNotNull) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(isNotNull) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(isNotNull) shouldBe true } it should "parse geo distance criteria" in { val result = Parser(geoDistanceCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - geoDistanceCriteria - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(geoDistanceCriteria) shouldBe true } it should "parse except fields" in { val result = Parser(except) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(except) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(except) shouldBe true } it should "parse match criteria" in { val result = Parser(matchCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(matchCriteria) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(matchCriteria) shouldBe true } it should "parse group by" in { val result = Parser(groupBy) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(groupBy) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(groupBy) shouldBe true } it should "parse order by" in { val result = Parser(orderBy) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(orderBy) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(orderBy) shouldBe true } it should "parse limit" in { val result = Parser(limit) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(limit) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(limit) shouldBe true } it should "parse group by with order by and limit" in { val result = Parser(groupByWithOrderByAndLimit) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - groupByWithOrderByAndLimit - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(groupByWithOrderByAndLimit) shouldBe true } it should "parse group by with having" in { val result = Parser(groupByWithHaving) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(groupByWithHaving) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(groupByWithHaving) shouldBe true } it should "parse date time fields" in { val result = Parser(dateTimeWithIntervalFields) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - dateTimeWithIntervalFields - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateTimeWithIntervalFields) shouldBe true } it should "parse fields with interval" in { val result = Parser(fieldsWithInterval) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===(fieldsWithInterval) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(fieldsWithInterval) shouldBe true } it should "parse filter with date time and interval" in { val result = Parser(filterWithDateTimeAndInterval) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - filterWithDateTimeAndInterval - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(filterWithDateTimeAndInterval) shouldBe true } it should "parse filter with date and interval" in { val result = Parser(filterWithDateAndInterval) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - filterWithDateAndInterval - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(filterWithDateAndInterval) shouldBe true } it should "parse filter with time and interval" in { val result = Parser(filterWithTimeAndInterval) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - filterWithTimeAndInterval - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(filterWithTimeAndInterval) shouldBe true } it should "parse group by with having and date time functions" in { val result = Parser(groupByWithHavingAndDateTimeFunctions) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - groupByWithHavingAndDateTimeFunctions - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(groupByWithHavingAndDateTimeFunctions) shouldBe true } - it should "parse parse_date function" in { - val result = Parser(parseDate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - parseDate - ) + it should "parse date_parse function" in { + val result = Parser(dateParse) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateParse) shouldBe true } - it should "parse parse_date_time function" in { - val result = Parser(parseDateTime) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - parseDateTime - ) + it should "parse date_parse_time function" in { + val result = Parser(dateTimeParse) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateTimeParse) shouldBe true } it should "parse date_diff function" in { val result = Parser(dateDiff) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - dateDiff - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateDiff) shouldBe true } it should "parse date_diff function with aggregation" in { val result = Parser(aggregationWithDateDiff) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - aggregationWithDateDiff - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(aggregationWithDateDiff) shouldBe true } it should "parse format_date function" in { - val result = Parser(formatDate) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - formatDate - ) + val result = Parser(dateFormat) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateFormat) shouldBe true } it should "parse format_datetime function" in { - val result = Parser(formatDateTime) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - formatDateTime - ) + val result = Parser(dateTimeFormat) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateTimeFormat) shouldBe true } it should "parse date_add function" in { val result = Parser(dateAdd) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - dateAdd - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateAdd) shouldBe true } it should "parse date_sub function" in { val result = Parser(dateSub) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - dateSub - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateSub) shouldBe true } it should "parse datetime_add function" in { val result = Parser(dateTimeAdd) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - dateTimeAdd - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateTimeAdd) shouldBe true } it should "parse datetime_sub function" in { val result = Parser(dateTimeSub) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - dateTimeSub - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(dateTimeSub) shouldBe true } it should "parse isnull function" in { val result = Parser(isnull) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - isnull - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(isnull) shouldBe true } it should "parse isnotnull function" in { val result = Parser(isnotnull) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - isnotnull - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(isnotnull) shouldBe true } it should "parse isnull criteria" in { val result = Parser(isNullCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - isNullCriteria - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(isNullCriteria) shouldBe true } it should "parse isnotnull criteria" in { val result = Parser(isNotNullCriteria) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - isNotNullCriteria - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(isNotNullCriteria) shouldBe true } it should "parse coalesce function" in { val result = Parser(coalesce) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - coalesce - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(coalesce) shouldBe true } it should "parse nullif function" in { val result = Parser(nullif) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - nullif - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(nullif) shouldBe true } it should "parse cast function" in { val result = Parser(cast) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - cast - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(cast) shouldBe true } it should "parse all casts function" in { val result = Parser(allCasts) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - allCasts - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(allCasts) shouldBe true } it should "parse case when expression" in { val result = Parser(caseWhen) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - caseWhen - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(caseWhen) shouldBe true } it should "parse case when with expression" in { @@ -610,30 +762,34 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse extract function" in { val result = Parser(extract) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - extract - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(extract) shouldBe true } it should "parse arithmetic expressions" in { val result = Parser(arithmetic) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - arithmetic - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(arithmetic) shouldBe true } it should "parse mathematical functions" in { val result = Parser(mathematical) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - mathematical - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(mathematical) shouldBe true } it should "parse string functions" in { val result = Parser(string) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") should ===( - string - ) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(string) shouldBe true } } From d3c98f4531e4639a654ef4e3dd71cd9b2897ea2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 10:56:26 +0200 Subject: [PATCH 10/48] fix boolean regex --- .../elastic/client/MappingComparator.scala | 2 +- .../softnetwork/elastic/client/package.scala | 2 +- .../elastic/client/jest/JestClientApi.scala | 4 +- .../client/jest/JestClientCompanion.scala | 2 +- .../client/rest/RestHighLevelClientApi.scala | 2 +- .../rest/RestHighLevelClientCompanion.scala | 2 +- .../elastic/sql/bridge/ElasticQuery.scala | 4 +- .../elastic/sql/bridge/package.scala | 39 +++++++------------ .../rest/RestHighLevelClientCompanion.scala | 2 +- .../client/java/ElasticsearchClientApi.scala | 2 +- .../elastic/sql/bridge/ElasticQuery.scala | 16 ++++---- .../elastic/sql/bridge/package.scala | 37 ++++++------------ .../elastic/sql/parser/type/package.scala | 2 +- 13 files changed, 45 insertions(+), 71 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala index dc30810e..4495ef34 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.client import com.google.gson._ import com.typesafe.scalalogging.StrictLogging -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} object MappingComparator extends StrictLogging { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index 5141184b..b837a72d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -13,7 +13,7 @@ import scala.collection.mutable import scala.language.reflectiveCalls import scala.util.{Failure, Success, Try} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ /** Created by smanciot on 30/06/2018. */ diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 13c0b393..e042bb68 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -9,7 +9,7 @@ import app.softnetwork.elastic.sql.query.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization._ -import com.google.gson.{Gson, JsonParser} +import com.google.gson.JsonParser import io.searchbox.action.BulkableAction import io.searchbox.core._ import io.searchbox.core.search.aggregation.RootAggregation @@ -21,7 +21,7 @@ import io.searchbox.indices.settings.{GetSettings, UpdateSettings} import io.searchbox.params.Parameters import org.json4s.Formats -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.implicitConversions import scala.util.{Failure, Success, Try} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala index 37278153..3334a79b 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala @@ -10,7 +10,7 @@ import org.apache.http.HttpHost import java.io.IOException import java.util import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls import scala.util.{Failure, Success, Try} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index f9392443..ca4d2a1d 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -49,7 +49,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder import org.json4s.Formats import java.io.ByteArrayInputStream -import scala.collection.JavaConverters.mapAsScalaMapConverter +import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.implicitConversions import scala.util.{Failure, Success, Try} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala index 1dd158a2..b54eb1a1 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala @@ -19,7 +19,7 @@ trait RestHighLevelClientCompanion extends Logging { private var client: Option[RestHighLevelClient] = None lazy val namedXContentRegistry: NamedXContentRegistry = { - import scala.collection.JavaConverters._ + import scala.jdk.CollectionConverters._ val searchModule = new SearchModule(Settings.EMPTY, false, List.empty[SearchPlugin].asJava) new NamedXContentRegistry(searchModule.getNamedXContents) } diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 8832b0b0..4cfc2c76 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -66,9 +66,7 @@ case class ElasticQuery(filter: ElasticFilter) { case isNull: IsNullExpr => isNull case isNotNull: IsNotNullExpr => isNotNull case in: InExpr[_, _] => in - case between: BetweenExpr[String] => between - case between: BetweenExpr[Long] => between - case between: BetweenExpr[Double] => between + case between: BetweenExpr[_] => between case geoDistance: ElasticGeoDistance => geoDistance case matchExpression: ElasticMatch => matchExpression case isNull: IsNullCriteria => isNull diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index b9b3dc4c..321b48f8 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} import app.softnetwork.elastic.sql.function.aggregate.Count import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ @@ -377,32 +378,22 @@ package object bridge { } implicit def betweenToQuery( - between: BetweenExpr[String] + between: BetweenExpr[_] ): Query = { import between._ - val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value - maybeNot match { - case Some(_) => not(r) - case _ => r - } - } - - implicit def betweenLongsToQuery( - between: BetweenExpr[Long] - ): Query = { - import between._ - val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value - maybeNot match { - case Some(_) => not(r) - case _ => r - } - } - - implicit def betweenDoublesToQuery( - between: BetweenExpr[Double] - ): Query = { - import between._ - val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value + val r = + out match { + case _: SQLDouble => + rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Double] lte fromTo.to.value + .asInstanceOf[Double] + case _: SQLBigInt => + rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Long] lte fromTo.to.value + .asInstanceOf[Long] + case _ => + rangeQuery(identifier.name) gte String.valueOf(fromTo.from.value) lte String.valueOf( + fromTo.to.value + ) + } maybeNot match { case Some(_) => not(r) case _ => r diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala index cf6c40bd..39777559 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala @@ -21,7 +21,7 @@ trait RestHighLevelClientCompanion { private var client: Option[RestHighLevelClient] = None lazy val namedXContentRegistry: NamedXContentRegistry = { - import scala.collection.JavaConverters._ + import scala.jdk.CollectionConverters._ val searchModule = new SearchModule(Settings.EMPTY, false, List.empty[SearchPlugin].asJava) new NamedXContentRegistry(searchModule.getNamedXContents) } diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 4288b8ec..6dd5bf92 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -31,7 +31,7 @@ import com.google.gson.{Gson, JsonParser} import _root_.java.io.{StringReader, StringWriter} import _root_.java.util.{Map => JMap} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future, Promise} diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 20d556d3..f8308229 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -62,17 +62,15 @@ case class ElasticQuery(filter: ElasticFilter) { criteria.asQuery(group = group, innerHitsNames = innerHitsNames), score = false ) - case expression: GenericExpression => expression - case isNull: IsNullExpr => isNull - case isNotNull: IsNotNullExpr => isNotNull - case in: InExpr[_, _] => in - case between: BetweenExpr[String] => between - case between: BetweenExpr[Long] => between - case between: BetweenExpr[Double] => between + case expression: GenericExpression => expression + case isNull: IsNullExpr => isNull + case isNotNull: IsNotNullExpr => isNotNull + case in: InExpr[_, _] => in + case between: BetweenExpr[_] => between case geoDistance: ElasticGeoDistance => geoDistance case matchExpression: ElasticMatch => matchExpression - case isNull: IsNullCriteria => isNull - case isNotNull: IsNotNullCriteria => isNotNull + case isNull: IsNullCriteria => isNull + case isNotNull: IsNotNullCriteria => isNotNull case other => throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index ed6d917d..439c47aa 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} import app.softnetwork.elastic.sql.function.aggregate.Count import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ @@ -379,32 +380,18 @@ package object bridge { } implicit def betweenToQuery( - between: BetweenExpr[String] - ): Query = { - import between._ - val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value - maybeNot match { - case Some(_) => not(r) - case _ => r - } - } - - implicit def betweenLongsToQuery( - between: BetweenExpr[Long] - ): Query = { - import between._ - val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value - maybeNot match { - case Some(_) => not(r) - case _ => r - } - } - - implicit def betweenDoublesToQuery( - between: BetweenExpr[Double] - ): Query = { + between: BetweenExpr[_] + ): Query = { import between._ - val r = rangeQuery(identifier.name) gte fromTo.from.value lte fromTo.to.value + val r = + out match { + case _: SQLDouble => + rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Double] lte fromTo.to.value.asInstanceOf[Double] + case _: SQLBigInt => + rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Long] lte fromTo.to.value.asInstanceOf[Long] + case _ => + rangeQuery(identifier.name) gte String.valueOf(fromTo.from.value) lte String.valueOf(fromTo.to.value) + } maybeNot match { case Some(_) => not(r) case _ => r diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index e98c9822..44e92052 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -30,7 +30,7 @@ package object `type` { PiValue.regex ^^ (_ => PiValue) def boolean: PackratParser[BooleanValue] = - """(?i)(true|false)\\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) + """(?i)(true|false)\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) def value_identifier: PackratParser[Identifier] = (literal | long | double | pi | boolean) ^^ { v => From 33e3d3d0bd8afcee903a8f03b37dc53b93876628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 13:50:22 +0200 Subject: [PATCH 11/48] add time field, fix extractors --- .../elastic/sql/SQLQuerySpec.scala | 30 +++++--- .../elastic/sql/SQLQuerySpec.scala | 34 ++++++--- .../elastic/sql/function/time/package.scala | 30 +++++--- .../sql/parser/function/time/package.scala | 72 ++++++------------- .../elastic/sql/parser/time/package.scala | 67 ++++++++++------- .../elastic/sql/time/package.scala | 56 ++++++++++----- .../sql/SQLDateTimeFunctionSuite.scala | 26 +++---- .../elastic/sql/SQLParserSpec.scala | 2 +- 8 files changed, 179 insertions(+), 138 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 54399bdf..17e2952e 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1293,7 +1293,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoUnit.YEARS) : null)" + | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoField.YEAR) : null)" | } | } | } @@ -2004,40 +2004,52 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "script_fields": { - | "day": { + | "dom": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.DAYS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_MONTH) : null)" + | } + | }, + | "dow": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_WEEK) : null)" + | } + | }, + | "doy": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_YEAR) : null)" | } | }, | "month": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.MONTHS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MONTH_OF_YEAR) : null)" | } | }, | "year": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.YEARS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.YEAR) : null)" | } | }, | "hour": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.HOURS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.HOUR_OF_DAY) : null)" | } | }, | "minute": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.MINUTES) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MINUTE_OF_HOUR) : null)" | } | }, | "second": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.SECONDS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" | } | } | }, @@ -2077,7 +2089,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def lv0 = ((!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value)); ( lv0 == null ) ? null : (lv0 * (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoUnit.YEARS) - 10)) > 10000" + | "source": "def lv0 = ((!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value)); ( lv0 == null ) ? null : (lv0 * (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR) - 10)) > 10000" | } | } | } diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 4f82720a..bf8c9121 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1185,7 +1185,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle parse_date function" in { val select: ElasticSearchRequest = - SQLQuery(parseDate) + SQLQuery(dateParse) val query = select.query println(query) query shouldBe @@ -1251,7 +1251,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle parse_datetime function" in { val select: ElasticSearchRequest = - SQLQuery(parseDateTime) + SQLQuery(dateTimeParse) val query = select.query println(query) query shouldBe @@ -1288,7 +1288,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoUnit.YEARS) : null)" + | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoField.YEAR) : null)" | } | } | } @@ -1993,40 +1993,52 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "match_all": {} | }, | "script_fields": { - | "day": { + | "dom": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.DAYS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_MONTH) : null)" + | } + | }, + | "dow": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_WEEK) : null)" + | } + | }, + | "doy": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_YEAR) : null)" | } | }, | "month": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.MONTHS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MONTH_OF_YEAR) : null)" | } | }, | "year": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.YEARS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.YEAR) : null)" | } | }, | "hour": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.HOURS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.HOUR_OF_DAY) : null)" | } | }, | "minute": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.MINUTES) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MINUTE_OF_HOUR) : null)" | } | }, | "second": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoUnit.SECONDS) : null)" + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" | } | } | }, @@ -2066,7 +2078,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def lv0 = ((!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value)); ( lv0 == null ) ? null : (lv0 * (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoUnit.YEARS) - 10)) > 10000" + | "source": "def lv0 = ((!doc.containsKey('identifier') || doc['identifier'].empty ? null : doc['identifier'].value)); ( lv0 == null ) ? null : (lv0 * (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR) - 10)) > 10000" | } | } | } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index f23622cc..8806f616 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -12,7 +12,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTypes, SQLVarchar } -import app.softnetwork.elastic.sql.time.{TimeInterval, TimeUnit} +import app.softnetwork.elastic.sql.time.{TimeField, TimeInterval, TimeUnit} package object time { @@ -155,43 +155,51 @@ package object time { override def painless: String = ".get" } - case class Extract(unit: TimeUnit, override val sql: String = "extract") + case class Extract(field: TimeField, override val sql: String = Extract.sql) extends DateTimeFunction with TransformFunction[SQLTemporal, SQLNumeric] { override def fun: Option[PainlessScript] = Some(Extract) - override def args: List[PainlessScript] = List(unit) + override def args: List[PainlessScript] = List(field) override def inputType: SQLTemporal = SQLTypes.Temporal override def outputType: SQLNumeric = SQLTypes.Numeric - override def toSQL(base: String): String = s"$sql(${unit.sql} from $base)" + override def toSQL(base: String): String = s"$sql(${field.sql} FROM $base)" } - import TimeUnit._ + import TimeField._ + + object Year extends Extract(YEAR, YEAR.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } + + object MonthOfYear extends Extract(MONTH_OF_YEAR, MONTH_OF_YEAR.sql) { + override def toSQL(base: String): String = s"$sql($base)" + } - object YEAR extends Extract(Year, Year.sql) { + object DayOfMonth extends Extract(DAY_OF_MONTH, DAY_OF_MONTH.sql) { override def toSQL(base: String): String = s"$sql($base)" } - object MONTH extends Extract(Month, Month.sql) { + object DayOfWeek extends Extract(DAY_OF_WEEK, DAY_OF_WEEK.sql) { override def toSQL(base: String): String = s"$sql($base)" } - object DAY extends Extract(Day, Day.sql) { + object DayOfYear extends Extract(DAY_OF_YEAR, DAY_OF_YEAR.sql) { override def toSQL(base: String): String = s"$sql($base)" } - object HOUR extends Extract(Hour, Hour.sql) { + object HourOfDay extends Extract(HOUR_OF_DAY, HOUR_OF_DAY.sql) { override def toSQL(base: String): String = s"$sql($base)" } - object MINUTE extends Extract(Minute, Minute.sql) { + object MinuteOfHour extends Extract(MINUTE_OF_HOUR, MINUTE_OF_HOUR.sql) { override def toSQL(base: String): String = s"$sql($base)" } - object SECOND extends Extract(Second, Second.sql) { + object SecondOfMinute extends Extract(SECOND_OF_MINUTE, SECOND_OF_MINUTE.sql) { override def toSQL(base: String): String = s"$sql($base)" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index 51759120..e3097549 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -7,39 +7,10 @@ import app.softnetwork.elastic.sql.function.{ FunctionWithIdentifier, TransformFunction } -import app.softnetwork.elastic.sql.function.time.{ - CurentDateWithParens, - CurrentDate, - CurrentFunction, - CurrentTime, - CurrentTimeWithParens, - CurrentTimestamp, - CurrentTimestampWithParens, - DAY, - DateAdd, - DateDiff, - DateFormat, - DateFunction, - DateParse, - DateSub, - DateTimeAdd, - DateTimeFormat, - DateTimeFunction, - DateTimeParse, - DateTimeSub, - DateTrunc, - Extract, - HOUR, - MINUTE, - MONTH, - Now, - NowWithParens, - SECOND, - YEAR -} +import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.parser.time.TimeParser import app.softnetwork.elastic.sql.parser.{Delimiter, Parser} -import app.softnetwork.elastic.sql.time.TimeUnit.{Day, Hour, Minute, Month, Second, Year} +import app.softnetwork.elastic.sql.time.TimeField package object time { @@ -203,31 +174,32 @@ package object time { } def extract_identifier: PackratParser[Identifier] = - Extract.regex ~ start ~ time_unit ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { + Extract.regex ~ start ~ time_field ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => i.withFunctions(Extract(u) +: i.functions) } - def extract_year: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Year.regex ^^ (_ => YEAR) - - def extract_month: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Month.regex ^^ (_ => MONTH) - - def extract_day: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Day.regex ^^ (_ => DAY) - - def extract_hour: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Hour.regex ^^ (_ => HOUR) - - def extract_minute: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Minute.regex ^^ (_ => MINUTE) - - def extract_second: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - Second.regex ^^ (_ => SECOND) + import TimeField._ + + def year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + YEAR.regex ^^ (_ => Year) + def month_of_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + MONTH_OF_YEAR.regex ^^ (_ => MonthOfYear) + def day_of_month_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + DAY_OF_MONTH.regex ^^ (_ => DayOfMonth) + def day_of_week_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + DAY_OF_WEEK.regex ^^ (_ => DayOfWeek) + def day_of_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + DAY_OF_YEAR.regex ^^ (_ => DayOfYear) + def hour_of_day_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + HOUR_OF_DAY.regex ^^ (_ => HourOfDay) + def minute_of_hour_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + MINUTE_OF_HOUR.regex ^^ (_ => MinuteOfHour) + def second_of_minute_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + SECOND_OF_MINUTE.regex ^^ (_ => SecondOfMinute) def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - extract_year | extract_month | extract_day | extract_hour | extract_minute | extract_second + year_tr | month_of_year_tr | day_of_month_tr | day_of_week_tr | day_of_year_tr | hour_of_day_tr | minute_of_hour_tr | second_of_minute_tr } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala index 79a54c37..607e80a6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala @@ -4,40 +4,53 @@ import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.function.TransformFunction import app.softnetwork.elastic.sql.function.time.{SQLAddInterval, SQLSubtractInterval} -import app.softnetwork.elastic.sql.time.{Interval, TimeInterval, TimeUnit} -import app.softnetwork.elastic.sql.time.TimeUnit.{ - Day, - Hour, - Minute, - Month, - Quarter, - Second, - Week, - Year -} +import app.softnetwork.elastic.sql.time.{Interval, TimeField, TimeInterval, TimeUnit} package object time { trait TimeParser { self: Parser => - def year: PackratParser[TimeUnit] = Year.regex ^^ (_ => Year) - - def month: PackratParser[TimeUnit] = Month.regex ^^ (_ => Month) - - def quarter: PackratParser[TimeUnit] = Quarter.regex ^^ (_ => Quarter) - - def week: PackratParser[TimeUnit] = Week.regex ^^ (_ => Week) - - def day: PackratParser[TimeUnit] = Day.regex ^^ (_ => Day) - - def hour: PackratParser[TimeUnit] = Hour.regex ^^ (_ => Hour) - - def minute: PackratParser[TimeUnit] = Minute.regex ^^ (_ => Minute) - - def second: PackratParser[TimeUnit] = Second.regex ^^ (_ => Second) + import TimeField._ + + def year: PackratParser[TimeField] = YEAR.regex ^^ (_ => YEAR) + def month_of_year: PackratParser[TimeField] = MONTH_OF_YEAR.regex ^^ (_ => MONTH_OF_YEAR) + def day_of_month: PackratParser[TimeField] = + DAY_OF_MONTH.regex ^^ (_ => DAY_OF_MONTH) + def day_of_week: PackratParser[TimeField] = + DAY_OF_WEEK.regex ^^ (_ => DAY_OF_WEEK) + def day_of_year: PackratParser[TimeField] = + DAY_OF_YEAR.regex ^^ (_ => DAY_OF_YEAR) + def hour_of_day: PackratParser[TimeField] = HOUR_OF_DAY.regex ^^ (_ => HOUR_OF_DAY) + def minute_of_hour: PackratParser[TimeField] = MINUTE_OF_HOUR.regex ^^ (_ => MINUTE_OF_HOUR) + def second_of_minute: PackratParser[TimeField] = + SECOND_OF_MINUTE.regex ^^ (_ => SECOND_OF_MINUTE) + def nano_of_second: PackratParser[TimeField] = + NANO_OF_SECOND.regex ^^ (_ => NANO_OF_SECOND) + def micro_of_second: PackratParser[TimeField] = + MICRO_OF_SECOND.regex ^^ (_ => MICRO_OF_SECOND) + def milli_of_second: PackratParser[TimeField] = + MILLI_OF_SECOND.regex ^^ (_ => MILLI_OF_SECOND) + def epoch_day: PackratParser[TimeField] = + EPOCH_DAY.regex ^^ (_ => EPOCH_DAY) + def offset_seconds: PackratParser[TimeField] = + OFFSET_SECONDS.regex ^^ (_ => OFFSET_SECONDS) + + def time_field: PackratParser[TimeField] = + year | month_of_year | day_of_month | day_of_week | day_of_year | hour_of_day | minute_of_hour | second_of_minute | nano_of_second | micro_of_second | milli_of_second | epoch_day | offset_seconds + + import TimeUnit._ + + def years: PackratParser[TimeUnit] = YEARS.regex ^^ (_ => YEARS) + def months: PackratParser[TimeUnit] = MONTHS.regex ^^ (_ => MONTHS) + def quarters: PackratParser[TimeUnit] = QUARTERS.regex ^^ (_ => QUARTERS) + def weeks: PackratParser[TimeUnit] = WEEKS.regex ^^ (_ => WEEKS) + def days: PackratParser[TimeUnit] = DAYS.regex ^^ (_ => DAYS) + def hours: PackratParser[TimeUnit] = HOURS.regex ^^ (_ => HOURS) + def minutes: PackratParser[TimeUnit] = MINUTES.regex ^^ (_ => MINUTES) + def seconds: PackratParser[TimeUnit] = SECONDS.regex ^^ (_ => SECONDS) def time_unit: PackratParser[TimeUnit] = - year | month | quarter | week | day | hour | minute | second + years | months | quarters | weeks | days | hours | minutes | seconds def interval: PackratParser[TimeInterval] = Interval.regex ~ long ~ time_unit ^^ { case _ ~ l ~ u => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index a8379b3e..6a85414c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -6,6 +6,30 @@ import scala.util.matching.Regex package object time { + sealed trait TimeField extends PainlessScript { + lazy val regex: Regex = s"\\b(?i)$sql\\b".r + + override def painless: String = s"ChronoField.$sql" + + override def nullable: Boolean = false + } + + object TimeField { + case object YEAR extends Expr("YEAR") with TimeField + case object MONTH_OF_YEAR extends Expr("MONTH_OF_YEAR") with TimeField + case object DAY_OF_MONTH extends Expr("DAY_OF_MONTH") with TimeField + case object DAY_OF_WEEK extends Expr("DAY_OF_WEEK") with TimeField + case object DAY_OF_YEAR extends Expr("DAY_OF_YEAR") with TimeField + case object HOUR_OF_DAY extends Expr("HOUR_OF_DAY") with TimeField + case object MINUTE_OF_HOUR extends Expr("MINUTE_OF_HOUR") with TimeField + case object SECOND_OF_MINUTE extends Expr("SECOND_OF_MINUTE") with TimeField + case object NANO_OF_SECOND extends Expr("NANO_OF_SECOND") with TimeField + case object MICRO_OF_SECOND extends Expr("MICRO_OF_SECOND") with TimeField + case object MILLI_OF_SECOND extends Expr("MILLI_OF_SECOND") with TimeField + case object EPOCH_DAY extends Expr("EPOCH_DAY") with TimeField + case object OFFSET_SECONDS extends Expr("OFFSET_SECONDS") with TimeField + } + sealed trait TimeUnit extends PainlessScript with MathScript { lazy val regex: Regex = s"\\b(?i)$sql(s)?\\b".r @@ -18,32 +42,32 @@ package object time { sealed trait FixedUnit extends TimeUnit object TimeUnit { - case object Year extends Expr("YEAR") with CalendarUnit { + case object YEARS extends Expr("YEAR") with CalendarUnit { override def script: String = "y" } - case object Month extends Expr("MONTH") with CalendarUnit { + case object MONTHS extends Expr("MONTH") with CalendarUnit { override def script: String = "M" } - case object Quarter extends Expr("QUARTER") with CalendarUnit { + case object QUARTERS extends Expr("QUARTER") with CalendarUnit { override def script: String = throw new IllegalArgumentException( "Quarter must be converted to months (value * 3) before creating date-math" ) } - case object Week extends Expr("WEEK") with CalendarUnit { + case object WEEKS extends Expr("WEEK") with CalendarUnit { override def script: String = "w" } - case object Day extends Expr("DAY") with CalendarUnit with FixedUnit { + case object DAYS extends Expr("DAY") with CalendarUnit with FixedUnit { override def script: String = "d" } - case object Hour extends Expr("HOUR") with FixedUnit { + case object HOURS extends Expr("HOUR") with FixedUnit { override def script: String = "H" } - case object Minute extends Expr("MINUTE") with FixedUnit { + case object MINUTES extends Expr("MINUTE") with FixedUnit { override def script: String = "m" } - case object Second extends Expr("SECOND") with FixedUnit { + case object SECONDS extends Expr("SECOND") with FixedUnit { override def script: String = "s" } @@ -65,14 +89,14 @@ package object time { in match { case SQLTypes.Date => unit match { - case Year | Month | Day => Right(SQLTypes.Date) - case Hour | Minute | Second => Right(SQLTypes.Timestamp) - case _ => Left(s"Invalid interval unit $unit for DATE") + case YEARS | MONTHS | DAYS => Right(SQLTypes.Date) + case HOURS | MINUTES | SECONDS => Right(SQLTypes.Timestamp) + case _ => Left(s"Invalid interval unit $unit for DATE") } case SQLTypes.Time => unit match { - case Hour | Minute | Second => Right(SQLTypes.Time) - case _ => Left(s"Invalid interval unit $unit for TIME") + case HOURS | MINUTES | SECONDS => Right(SQLTypes.Time) + case _ => Left(s"Invalid interval unit $unit for TIME") } case SQLTypes.DateTime => Right(SQLTypes.Timestamp) @@ -99,9 +123,9 @@ package object time { case fu: FixedUnit => FixedInterval(value, fu) } def script(interval: TimeInterval): String = interval match { - case CalendarInterval(v, Quarter) => s"${v * 3}M" - case CalendarInterval(v, u) => s"$v${u.script}" - case FixedInterval(v, u) => s"$v${u.script}" + case CalendarInterval(v, QUARTERS) => s"${v * 3}M" + case CalendarInterval(v, u) => s"$v${u.script}" + case FixedInterval(v, u) => s"$v${u.script}" } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala index 2fcd8e9f..84335c8e 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala @@ -4,7 +4,7 @@ import org.scalatest.funsuite.AnyFunSuite import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.time._ -import TimeUnit._ +import TimeField._ import app.softnetwork.elastic.sql.`type`.SQLType class SQLDateTimeFunctionSuite extends AnyFunSuite { @@ -16,20 +16,20 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { val transformFunctions: Seq[TransformFunction[_, _]] = Seq( DateParse(Identifier(), "yyyy-MM-dd"), DateTimeParse(Identifier(), "yyyy-MM-dd HH:mm:ss"), - DateAdd(Identifier(), TimeInterval(1, Day)), - DateSub(Identifier(), TimeInterval(2, Month)), - DateTimeAdd(Identifier(), TimeInterval(3, Hour)), - DateTimeSub(Identifier(), TimeInterval(30, Minute)), - DateTrunc(Identifier(), Day), - Extract(Day), + DateAdd(Identifier(), TimeInterval(1, TimeUnit.DAYS)), + DateSub(Identifier(), TimeInterval(2, TimeUnit.MONTHS)), + DateTimeAdd(Identifier(), TimeInterval(3, TimeUnit.HOURS)), + DateTimeSub(Identifier(), TimeInterval(30, TimeUnit.MINUTES)), + DateTrunc(Identifier(), TimeUnit.DAYS), + Extract(TimeField.DAY_OF_MONTH), DateFormat(Identifier(), "yyyy-MM-dd"), DateTimeFormat(Identifier(), "yyyy-MM-dd HH:mm:ss"), - YEAR, - MONTH, - DAY, - HOUR, - MINUTE, - SECOND + Year, + MonthOfYear, + DayOfYear, + HourOfDay, + MinuteOfHour, + SecondOfMinute ) // Fonction pour chaîner une séquence de transformations en vérifiant les types diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index f5b03f60..3ca6d8b0 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -153,7 +153,7 @@ object Queries { "select case current_date - interval 7 day when cast(lastUpdated as date) - interval 3 day then lastUpdated when lastSeen then lastSeen + interval 2 day else createdAt end as c, identifier from Table" val extract: String = - "select extract(day from createdAt) as day, extract(month from createdAt) as month, extract(year from createdAt) as year, extract(hour from createdAt) as hour, extract(minute from createdAt) as minute, extract(second from createdAt) as second from Table" + "select extract(day_of_month from createdAt) as dom, extract(day_of_week from createdAt) as dow, extract(day_of_year from createdAt) as doy, extract(month_of_year from createdAt) as month, extract(year from createdAt) as year, extract(hour_of_day from createdAt) as hour, extract(minute_of_hour from createdAt) as minute, extract(second_of_minute from createdAt) as second from Table" val arithmetic: String = "select identifier, identifier + 1 as add, identifier - 1 as sub, identifier * 2 as mul, identifier / 2 as div, identifier % 2 as mod, (identifier * identifier2) - 10 as group1 from Table where identifier * (extract(year from current_date) - 10) > 10000" From 540da9f12b57a6cd03e51612273a960030f81d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 14:16:06 +0200 Subject: [PATCH 12/48] update regex for aliases --- .../elastic/sql/SQLQuerySpec.scala | 22 +++++++++---------- .../elastic/sql/SQLQuerySpec.scala | 22 +++++++++---------- .../elastic/sql/parser/Parser.scala | 6 ++--- .../elastic/sql/SQLParserSpec.scala | 4 ++-- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 17e2952e..e726b389 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2022,31 +2022,31 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_YEAR) : null)" | } | }, - | "month": { + | "m": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MONTH_OF_YEAR) : null)" | } | }, - | "year": { + | "y": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.YEAR) : null)" | } | }, - | "hour": { + | "h": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.HOUR_OF_DAY) : null)" | } | }, - | "minute": { + | "minutes": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MINUTE_OF_HOUR) : null)" | } | }, - | "second": { + | "s": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" @@ -2354,37 +2354,37 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "script_fields": { - | "len": { + | "l": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.length() : null)" | } | }, - | "lower": { + | "low": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.lower() : null)" | } | }, - | "upper": { + | "upp": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.upper() : null)" | } | }, - | "substr": { + | "sub": { | "script": { | "lang": "painless", | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : ((1 - 1) < 0 || (1 - 1 + 3) > arg0.length()) ? null : arg0.substring((1 - 1), (1 - 1 + 3)))" | } | }, - | "trim": { + | "tr": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.trim() : null)" | } | }, - | "concat": { + | "con": { | "script": { | "lang": "painless", | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : String.valueOf(arg0) + \"_test\" + String.valueOf(1))" diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index bf8c9121..4f526caf 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2011,31 +2011,31 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_YEAR) : null)" | } | }, - | "month": { + | "m": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MONTH_OF_YEAR) : null)" | } | }, - | "year": { + | "y": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.YEAR) : null)" | } | }, - | "hour": { + | "h": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.HOUR_OF_DAY) : null)" | } | }, - | "minute": { + | "minutes": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MINUTE_OF_HOUR) : null)" | } | }, - | "second": { + | "s": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" @@ -2343,37 +2343,37 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "script_fields": { - | "len": { + | "l": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.length() : null)" | } | }, - | "lower": { + | "low": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.lower() : null)" | } | }, - | "upper": { + | "upp": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.upper() : null)" | } | }, - | "substr": { + | "sub": { | "script": { | "lang": "painless", | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : ((1 - 1) < 0 || (1 - 1 + 3) > arg0.length()) ? null : arg0.substring((1 - 1), (1 - 1 + 3)))" | } | }, - | "trim": { + | "tr": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.trim() : null)" | } | }, - | "concat": { + | "con": { | "script": { | "lang": "painless", | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : String.valueOf(arg0) + \"_test\" + String.valueOf(1))" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 00e5308c..b85c1cb7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -234,9 +234,7 @@ trait Parser ) private val identifierRegexStr = - s"""(?i)(?!(?:${reservedKeywords.mkString( - "|" - )})\\b)[\\*a-zA-Z_\\-][a-zA-Z0-9_\\-.\\[\\]\\*]*""" + s"""(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[\\*a-zA-Z_\\-][a-zA-Z0-9_\\-.\\[\\]\\*]*""" private val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex @@ -271,7 +269,7 @@ trait Parser } private val regexAlias = - """\b(?!(?i)as\b)\b(?!(?i)except\b)\b(?!(?i)where\b)\b(?!(?i)filter\b)\b(?!(?i)from\b)\b(?!(?i)group\b)\b(?!(?i)having\b)\b(?!(?i)order\b)\b(?!(?i)limit\b)[a-zA-Z0-9_]*""" + s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_]*""".stripMargin def alias: PackratParser[Alias] = Alias.regex.? ~ regexAlias.r ^^ { case _ ~ b => Alias(b) } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 3ca6d8b0..b747fa8e 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -153,7 +153,7 @@ object Queries { "select case current_date - interval 7 day when cast(lastUpdated as date) - interval 3 day then lastUpdated when lastSeen then lastSeen + interval 2 day else createdAt end as c, identifier from Table" val extract: String = - "select extract(day_of_month from createdAt) as dom, extract(day_of_week from createdAt) as dow, extract(day_of_year from createdAt) as doy, extract(month_of_year from createdAt) as month, extract(year from createdAt) as year, extract(hour_of_day from createdAt) as hour, extract(minute_of_hour from createdAt) as minute, extract(second_of_minute from createdAt) as second from Table" + "select extract(day_of_month from createdAt) as dom, extract(day_of_week from createdAt) as dow, extract(day_of_year from createdAt) as doy, extract(month_of_year from createdAt) as m, extract(year from createdAt) as y, extract(hour_of_day from createdAt) as h, extract(minute_of_hour from createdAt) as minutes, extract(second_of_minute from createdAt) as s from Table" val arithmetic: String = "select identifier, identifier + 1 as add, identifier - 1 as sub, identifier * 2 as mul, identifier / 2 as div, identifier % 2 as mod, (identifier * identifier2) - 10 as group1 from Table where identifier * (extract(year from current_date) - 10) > 10000" @@ -162,7 +162,7 @@ object Queries { "select identifier, (abs(identifier) + 1.0) * 2, ceil(identifier), floor(identifier), sqrt(identifier), exp(identifier), log(identifier), log10(identifier), pow(identifier, 3), round(identifier), round(identifier, 2), sign(identifier), cos(identifier), acos(identifier), sin(identifier), asin(identifier), tan(identifier), atan(identifier), atan2(identifier, 3.0) from Table where sqrt(identifier) > 100.0" val string: String = - "select identifier, length(identifier2) as len, lower(identifier2) as lower, upper(identifier2) as upper, substring(identifier2, 1, 3) as substr, trim(identifier2) as trim, concat(identifier2, '_test', 1) as concat from Table where length(trim(identifier2)) > 10" + "select identifier, length(identifier2) as l, lower(identifier2) as low, upper(identifier2) as upp, substring(identifier2, 1, 3) as sub, trim(identifier2) as tr, concat(identifier2, '_test', 1) as con from Table where length(trim(identifier2)) > 10" } /** Created by smanciot on 15/02/17. From b5c075d5f1a2d867a947aa50a8647558efca3a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 17:45:11 +0200 Subject: [PATCH 13/48] update object class names, add support for RLIKE --- .../elastic/client/jest/JestClientApi.scala | 10 +-- .../client/rest/RestHighLevelClientApi.scala | 10 +-- .../sql/bridge/ElasticAggregation.scala | 10 +-- .../elastic/sql/bridge/package.scala | 53 +++++++----- .../client/rest/RestHighLevelClientApi.scala | 10 +-- .../client/java/ElasticsearchClientApi.scala | 10 +-- .../client/java/ElasticsearchClientApi.scala | 10 +-- .../sql/bridge/ElasticAggregation.scala | 10 +-- .../elastic/sql/bridge/package.scala | 53 +++++++----- .../sql/function/aggregate/package.scala | 12 +-- .../elastic/sql/function/cond/package.scala | 26 +++--- .../elastic/sql/function/string/package.scala | 2 +- .../elastic/sql/function/time/package.scala | 4 +- .../elastic/sql/operator/math/package.scala | 10 +-- .../elastic/sql/operator/package.scala | 69 +++++++-------- .../elastic/sql/operator/time/package.scala | 8 +- .../app/softnetwork/elastic/sql/package.scala | 41 ++++----- .../elastic/sql/parser/Parser.scala | 2 +- .../elastic/sql/parser/WhereParser.scala | 84 ++++++++++--------- .../parser/function/aggregate/package.scala | 12 +-- .../sql/parser/function/cond/package.scala | 24 +++--- .../sql/parser/function/string/package.scala | 4 +- .../sql/parser/operator/math/package.scala | 20 ++--- .../elastic/sql/query/GroupBy.scala | 2 +- .../softnetwork/elastic/sql/query/Where.scala | 66 +++++++-------- .../elastic/sql/time/package.scala | 52 ++++++++---- .../elastic/sql/SQLParserSpec.scala | 11 ++- 27 files changed, 332 insertions(+), 293 deletions(-) diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index e042bb68..578cb127 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -386,7 +386,7 @@ trait JestSingleValueAggregateApi extends SingleValueAggregateApi with JestCount field, aggType, aggType match { - case sql.function.aggregate.Count => + case sql.function.aggregate.COUNT => if (aggregation.distinct) NumericValue( root.getCardinalityAggregation(agg).getCardinality.doubleValue() @@ -396,13 +396,13 @@ trait JestSingleValueAggregateApi extends SingleValueAggregateApi with JestCount root.getValueCountAggregation(agg).getValueCount.doubleValue() ) } - case sql.function.aggregate.Sum => + case sql.function.aggregate.SUM => NumericValue(root.getSumAggregation(agg).getSum) - case sql.function.aggregate.Avg => + case sql.function.aggregate.AVG => NumericValue(root.getAvgAggregation(agg).getAvg) - case sql.function.aggregate.Min => + case sql.function.aggregate.MIN => NumericValue(root.getMinAggregation(agg).getMin) - case sql.function.aggregate.Max => + case sql.function.aggregate.MAX => NumericValue(root.getMaxAggregation(agg).getMax) case _ => EmptyValue }, diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index ca4d2a1d..f966d6dc 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -428,19 +428,19 @@ trait RestHighLevelClientSingleValueAggregateApi field, aggType, aggType match { - case sql.function.aggregate.Count => + case sql.function.aggregate.COUNT => if (aggregation.distinct) { NumericValue(root.get(agg).asInstanceOf[Cardinality].value()) } else { NumericValue(root.get(agg).asInstanceOf[ValueCount].value()) } - case sql.function.aggregate.Sum => + case sql.function.aggregate.SUM => NumericValue(root.get(agg).asInstanceOf[Sum].value()) - case sql.function.aggregate.Avg => + case sql.function.aggregate.AVG => NumericValue(root.get(agg).asInstanceOf[Avg].value()) - case sql.function.aggregate.Min => + case sql.function.aggregate.MIN => NumericValue(root.get(agg).asInstanceOf[Min].value()) - case sql.function.aggregate.Max => + case sql.function.aggregate.MAX => NumericValue(root.get(agg).asInstanceOf[Max].value()) case _ => EmptyValue }, diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 57627ab6..b2f8fdf2 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -103,16 +103,16 @@ object ElasticAggregation { val _agg = aggType match { - case Count => + case COUNT => if (distinct) cardinalityAgg(aggName, sourceField) else { valueCountAgg(aggName, sourceField) } - case Min => aggWithFieldOrScript(minAgg, (name, s) => minAgg(name, sourceField).script(s)) - case Max => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) - case Avg => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) - case Sum => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) + case MIN => aggWithFieldOrScript(minAgg, (name, s) => minAgg(name, sourceField).script(s)) + case MAX => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) + case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) + case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) } val filteredAggName = "filtered_agg" diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 321b48f8..0fcbc175 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} -import app.softnetwork.elastic.sql.function.aggregate.Count +import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ import com.sksamuel.elastic4s.ElasticApi @@ -155,7 +155,7 @@ package object bridge { value match { case n: NumericValue[_] => operator match { - case Ge => + case GE => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -168,7 +168,7 @@ package object bridge { d => rangeQuery(identifier.name) gte d ) } - case Gt => + case GT => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -181,7 +181,7 @@ package object bridge { d => rangeQuery(identifier.name) gt d ) } - case Le => + case LE => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -194,7 +194,7 @@ package object bridge { d => rangeQuery(identifier.name) lte d ) } - case Lt => + case LT => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -207,7 +207,7 @@ package object bridge { d => rangeQuery(identifier.name) lt d ) } - case Eq => + case EQ => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -220,7 +220,7 @@ package object bridge { d => termQuery(identifier.name, d) ) } - case Ne | Diff => + case NE | DIFF => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -237,49 +237,56 @@ package object bridge { } case l: StringValue => operator match { - case Like => + case LIKE => maybeNot match { case Some(_) => not(regexQuery(identifier.name, toRegex(l.value))) case _ => regexQuery(identifier.name, toRegex(l.value)) } - case Ge => + case RLIKE => + maybeNot match { + case Some(_) => + not(regexQuery(identifier.name, l.value)) + case _ => + regexQuery(identifier.name, l.value) + } + case GE => maybeNot match { case Some(_) => rangeQuery(identifier.name) lt l.value case _ => rangeQuery(identifier.name) gte l.value } - case Gt => + case GT => maybeNot match { case Some(_) => rangeQuery(identifier.name) lte l.value case _ => rangeQuery(identifier.name) gt l.value } - case Le => + case LE => maybeNot match { case Some(_) => rangeQuery(identifier.name) gt l.value case _ => rangeQuery(identifier.name) lte l.value } - case Lt => + case LT => maybeNot match { case Some(_) => rangeQuery(identifier.name) gte l.value case _ => rangeQuery(identifier.name) lt l.value } - case Eq => + case EQ => maybeNot match { case Some(_) => not(termQuery(identifier.name, l.value)) case _ => termQuery(identifier.name, l.value) } - case Ne | Diff => + case NE | DIFF => maybeNot match { case Some(_) => termQuery(identifier.name, l.value) @@ -290,14 +297,14 @@ package object bridge { } case b: BooleanValue => operator match { - case Eq => + case EQ => maybeNot match { case Some(_) => not(termQuery(identifier.name, b.value)) case _ => termQuery(identifier.name, b.value) } - case Ne | Diff => + case NE | DIFF => maybeNot match { case Some(_) => termQuery(identifier.name, b.value) @@ -313,12 +320,12 @@ package object bridge { case Some(script) => val o = if (maybeNot.isDefined) op.not else op o match { - case Gt => rangeQuery(identifier.name) gt script - case Ge => rangeQuery(identifier.name) gte script - case Lt => rangeQuery(identifier.name) lt script - case Le => rangeQuery(identifier.name) lte script - case Eq => rangeQuery(identifier.name) gte script lte script - case Ne | Diff => not(rangeQuery(identifier.name) gte script lte script) + case GT => rangeQuery(identifier.name) gt script + case GE => rangeQuery(identifier.name) gte script + case LT => rangeQuery(identifier.name) lt script + case LE => rangeQuery(identifier.name) lte script + case EQ => rangeQuery(identifier.name) gte script lte script + case NE | DIFF => not(rangeQuery(identifier.name) gte script lte script) } case _ => scriptQuery(Script(script = painless).lang("painless").scriptType("source")) @@ -447,7 +454,7 @@ package object bridge { sources = l.sources, query = Some( (aggregation.aggType match { - case Count if aggregation.sourceField.equalsIgnoreCase("_id") => + case COUNT if aggregation.sourceField.equalsIgnoreCase("_id") => SearchBodyBuilderFn( ElasticApi.search("") query { queryFiltered diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 9ed055dc..862d3d64 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -423,19 +423,19 @@ trait RestHighLevelClientSingleValueAggregateApi field, aggType, aggType match { - case sql.function.aggregate.Count => + case sql.function.aggregate.COUNT => if (aggregation.distinct) { NumericValue(root.get(agg).asInstanceOf[Cardinality].value()) } else { NumericValue(root.get(agg).asInstanceOf[ValueCount].value()) } - case sql.function.aggregate.Sum => + case sql.function.aggregate.SUM => NumericValue(root.get(agg).asInstanceOf[Sum].value()) - case sql.function.aggregate.Avg => + case sql.function.aggregate.AVG => NumericValue(root.get(agg).asInstanceOf[Avg].value()) - case sql.function.aggregate.Min => + case sql.function.aggregate.MIN => NumericValue(root.get(agg).asInstanceOf[Min].value()) - case sql.function.aggregate.Max => + case sql.function.aggregate.MAX => NumericValue(root.get(agg).asInstanceOf[Max].value()) case _ => EmptyValue }, diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 6dd5bf92..afa1c07b 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -401,7 +401,7 @@ trait ElasticsearchClientSingleValueAggregateApi field, aggType, aggType match { - case sql.function.aggregate.Count => + case sql.function.aggregate.COUNT => NumericValue( if (aggregation.distinct) { root.get(agg).cardinality().value().toDouble @@ -409,15 +409,15 @@ trait ElasticsearchClientSingleValueAggregateApi root.get(agg).valueCount().value() } ) - case sql.function.aggregate.Sum => + case sql.function.aggregate.SUM => NumericValue(root.get(agg).sum().value()) - case sql.function.aggregate.Avg => + case sql.function.aggregate.AVG => val avgAgg = root.get(agg).avg() aggregateValue(avgAgg.value(), avgAgg.valueAsString()) - case sql.function.aggregate.Min => + case sql.function.aggregate.MIN => val minAgg = root.get(agg).min() aggregateValue(minAgg.value(), minAgg.valueAsString()) - case sql.function.aggregate.Max => + case sql.function.aggregate.MAX => val maxAgg = root.get(agg).max() aggregateValue(maxAgg.value(), maxAgg.valueAsString()) case _ => EmptyValue diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 04bdd5df..9ca843bf 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -396,7 +396,7 @@ trait ElasticsearchClientSingleValueAggregateApi field, aggType, aggType match { - case sql.function.aggregate.Count => + case sql.function.aggregate.COUNT => NumericValue( if (aggregation.distinct) { root.get(agg).cardinality().value().toDouble @@ -404,15 +404,15 @@ trait ElasticsearchClientSingleValueAggregateApi root.get(agg).valueCount().value() } ) - case sql.function.aggregate.Sum => + case sql.function.aggregate.SUM => NumericValue(root.get(agg).sum().value()) - case sql.function.aggregate.Avg => + case sql.function.aggregate.AVG => val avgAgg = root.get(agg).avg() aggregateValue(avgAgg.value(), avgAgg.valueAsString()) - case sql.function.aggregate.Min => + case sql.function.aggregate.MIN => val minAgg = root.get(agg).min() aggregateValue(minAgg.value(), minAgg.valueAsString()) - case sql.function.aggregate.Max => + case sql.function.aggregate.MAX => val maxAgg = root.get(agg).max() aggregateValue(maxAgg.value(), maxAgg.valueAsString()) case _ => EmptyValue diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 96a805e5..230fc276 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -102,16 +102,16 @@ object ElasticAggregation { val _agg = aggType match { - case Count => + case COUNT => if (distinct) cardinalityAgg(aggName, sourceField) else { valueCountAgg(aggName, sourceField) } - case Min => aggWithFieldOrScript(minAgg, (name, s) => minAgg(name, sourceField).script(s)) - case Max => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) - case Avg => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) - case Sum => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) + case MIN => aggWithFieldOrScript(minAgg, (name, s) => minAgg(name, sourceField).script(s)) + case MAX => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) + case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) + case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) } val filteredAggName = "filtered_agg" diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 439c47aa..93812ea1 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} -import app.softnetwork.elastic.sql.function.aggregate.Count +import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ @@ -157,7 +157,7 @@ package object bridge { value match { case n: NumericValue[_] => operator match { - case Ge => + case GE => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -170,7 +170,7 @@ package object bridge { d => rangeQuery(identifier.name) gte d ) } - case Gt => + case GT => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -183,7 +183,7 @@ package object bridge { d => rangeQuery(identifier.name) gt d ) } - case Le => + case LE => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -196,7 +196,7 @@ package object bridge { d => rangeQuery(identifier.name) lte d ) } - case Lt => + case LT => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -209,7 +209,7 @@ package object bridge { d => rangeQuery(identifier.name) lt d ) } - case Eq => + case EQ => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -222,7 +222,7 @@ package object bridge { d => termQuery(identifier.name, d) ) } - case Ne | Diff => + case NE | DIFF => maybeNot match { case Some(_) => applyNumericOp(n)( @@ -239,49 +239,56 @@ package object bridge { } case l: StringValue => operator match { - case Like => + case LIKE => maybeNot match { case Some(_) => not(regexQuery(identifier.name, toRegex(l.value))) case _ => regexQuery(identifier.name, toRegex(l.value)) } - case Ge => + case RLIKE => + maybeNot match { + case Some(_) => + not(regexQuery(identifier.name, l.value)) + case _ => + regexQuery(identifier.name, l.value) + } + case GE => maybeNot match { case Some(_) => rangeQuery(identifier.name) lt l.value case _ => rangeQuery(identifier.name) gte l.value } - case Gt => + case GT => maybeNot match { case Some(_) => rangeQuery(identifier.name) lte l.value case _ => rangeQuery(identifier.name) gt l.value } - case Le => + case LE => maybeNot match { case Some(_) => rangeQuery(identifier.name) gt l.value case _ => rangeQuery(identifier.name) lte l.value } - case Lt => + case LT => maybeNot match { case Some(_) => rangeQuery(identifier.name) gte l.value case _ => rangeQuery(identifier.name) lt l.value } - case Eq => + case EQ => maybeNot match { case Some(_) => not(termQuery(identifier.name, l.value)) case _ => termQuery(identifier.name, l.value) } - case Ne | Diff => + case NE | DIFF => maybeNot match { case Some(_) => termQuery(identifier.name, l.value) @@ -292,14 +299,14 @@ package object bridge { } case b: BooleanValue => operator match { - case Eq => + case EQ => maybeNot match { case Some(_) => not(termQuery(identifier.name, b.value)) case _ => termQuery(identifier.name, b.value) } - case Ne | Diff => + case NE | DIFF => maybeNot match { case Some(_) => termQuery(identifier.name, b.value) @@ -315,12 +322,12 @@ package object bridge { case Some(script) => val o = if (maybeNot.isDefined) op.not else op o match { - case Gt => rangeQuery(identifier.name) gt script - case Ge => rangeQuery(identifier.name) gte script - case Lt => rangeQuery(identifier.name) lt script - case Le => rangeQuery(identifier.name) lte script - case Eq => rangeQuery(identifier.name) gte script lte script - case Ne | Diff => not(rangeQuery(identifier.name) gte script lte script) + case GT => rangeQuery(identifier.name) gt script + case GE => rangeQuery(identifier.name) gte script + case LT => rangeQuery(identifier.name) lt script + case LE => rangeQuery(identifier.name) lte script + case EQ => rangeQuery(identifier.name) gte script lte script + case NE | DIFF => not(rangeQuery(identifier.name) gte script lte script) } case _ => scriptQuery(Script(script = painless).lang("painless").scriptType("source")) @@ -445,7 +452,7 @@ package object bridge { sources = l.sources, query = Some( (aggregation.aggType match { - case Count if aggregation.sourceField.equalsIgnoreCase("_id") => + case COUNT if aggregation.sourceField.equalsIgnoreCase("_id") => SearchBodyBuilderFn( ElasticApi.search("") query { queryFiltered diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index a002f9f3..f6e3eb93 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -1,18 +1,20 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.Expr +import app.softnetwork.elastic.sql.{Expr, TokenRegex} package object aggregate { sealed trait AggregateFunction extends Function - case object Count extends Expr("COUNT") with AggregateFunction + case object COUNT extends Expr("COUNT") with AggregateFunction - case object Min extends Expr("MIN") with AggregateFunction + case object MIN extends Expr("MIN") with AggregateFunction - case object Max extends Expr("MAX") with AggregateFunction + case object MAX extends Expr("MAX") with AggregateFunction - case object Avg extends Expr("AVG") with AggregateFunction + case object AVG extends Expr("AVG") with AggregateFunction + + case object SUM extends Expr("SUM") with AggregateFunction case object Sum extends Expr("SUM") with AggregateFunction diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 346d0dc6..10b95cad 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -11,17 +11,17 @@ package object cond { } case object Coalesce extends Expr("COALESCE") with ConditionalOp - case object IsNullFunction extends Expr("ISNULL") with ConditionalOp - case object IsNotNullFunction extends Expr("ISNOTNULL") with ConditionalOp + case object IsNull extends Expr("ISNULL") with ConditionalOp + case object IsNotNull extends Expr("ISNOTNULL") with ConditionalOp case object NullIf extends Expr("NULLIF") with ConditionalOp case object Exists extends Expr("EXISTS") with ConditionalOp case object Case extends Expr("CASE") with ConditionalOp - case object When extends Expr("WHEN") with TokenRegex - case object Then extends Expr("THEN") with TokenRegex - case object Else extends Expr("ELSE") with TokenRegex - case object End extends Expr("END") with TokenRegex + case object WHEN extends Expr("WHEN") with TokenRegex + case object THEN extends Expr("THEN") with TokenRegex + case object ELSE extends Expr("ELSE") with TokenRegex + case object END extends Expr("END") with TokenRegex sealed trait ConditionalFunction[In <: SQLType] extends TransformFunction[In, SQLBool] @@ -35,8 +35,8 @@ package object cond { override def toPainless(base: String, idx: Int): String = s"($base$painless)" } - case class IsNullFunction(identifier: Identifier) extends ConditionalFunction[SQLAny] { - override def conditionalOp: ConditionalOp = IsNullFunction + case class IsNull(identifier: Identifier) extends ConditionalFunction[SQLAny] { + override def conditionalOp: ConditionalOp = IsNull override def args: List[PainlessScript] = List(identifier) @@ -53,8 +53,8 @@ package object cond { } } - case class IsNotNullFunction(identifier: Identifier) extends ConditionalFunction[SQLAny] { - override def conditionalOp: ConditionalOp = IsNotNullFunction + case class IsNotNull(identifier: Identifier) extends ConditionalFunction[SQLAny] { + override def conditionalOp: ConditionalOp = IsNotNull override def args: List[PainlessScript] = List(identifier) @@ -154,10 +154,10 @@ package object cond { override def sql: String = { val exprPart = expression.map(e => s"$Case ${e.sql}").getOrElse(Case.sql) val whenThen = conditions - .map { case (cond, res) => s"$When ${cond.sql} $Then ${res.sql}" } + .map { case (cond, res) => s"$WHEN ${cond.sql} $THEN ${res.sql}" } .mkString(" ") - val elsePart = default.map(d => s" $Else ${d.sql}").getOrElse("") - s"$exprPart $whenThen$elsePart $End" + val elsePart = default.map(d => s" $ELSE ${d.sql}").getOrElse("") + s"$exprPart $whenThen$elsePart $END" } override def out: SQLType = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 6bd36bef..3a1180e7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -21,7 +21,7 @@ package object string { override def painless: String = ".substring" override lazy val words: List[String] = List(sql, "SUBSTR") } - case object To extends Expr("TO") with TokenRegex + case object TO extends Expr("TO") with TokenRegex case object Length extends Expr("LENGTH") with StringOp sealed trait StringFunction[Out <: SQLType] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 8806f616..1f96e094 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -54,11 +54,11 @@ package object time { } sealed trait AddInterval[IO <: SQLTemporal] extends IntervalFunction[IO] { - override def operator: IntervalOperator = Plus + override def operator: IntervalOperator = PLUS } sealed trait SubtractInterval[IO <: SQLTemporal] extends IntervalFunction[IO] { - override def operator: IntervalOperator = Minus + override def operator: IntervalOperator = MINUS } case class SQLAddInterval(interval: TimeInterval) extends AddInterval[SQLTemporal] { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala index 42c882fc..4d91f947 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/package.scala @@ -8,10 +8,10 @@ package object math { override def toString: String = s" $sql " } - case object Add extends Expr("+") with ArithmeticOperator - case object Subtract extends Expr("-") with ArithmeticOperator - case object Multiply extends Expr("*") with ArithmeticOperator - case object Divide extends Expr("/") with ArithmeticOperator - case object Modulo extends Expr("%") with ArithmeticOperator + case object ADD extends Expr("+") with ArithmeticOperator + case object SUBTRACT extends Expr("-") with ArithmeticOperator + case object MULTIPLY extends Expr("*") with ArithmeticOperator + case object DIVIDE extends Expr("/") with ArithmeticOperator + case object MODULO extends Expr("%") with ArithmeticOperator } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala index 409aa985..650b8e38 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala @@ -4,16 +4,16 @@ package object operator { trait Operator extends Token with PainlessScript with TokenRegex { override def painless: String = this match { - case And => "&&" - case Or => "||" - case Not => "!" - case In => ".contains" - case Like | Match => ".matches" - case Eq => "==" - case Ne => "!=" - case IsNull => " == null" - case IsNotNull => " != null" - case _ => sql + case AND => "&&" + case OR => "||" + case NOT => "!" + case IN => ".contains" + case LIKE | RLIKE | MATCH => ".matches" + case EQ => "==" + case NE => "!=" + case IS_NULL => " == null" + case IS_NOT_NULL => " != null" + case _ => sql } } @@ -23,41 +23,42 @@ package object operator { sealed trait ComparisonOperator extends ExpressionOperator with PainlessScript { def not: ComparisonOperator = this match { - case Eq => Ne - case Ne | Diff => Eq - case Ge => Lt - case Gt => Le - case Le => Gt - case Lt => Ge + case EQ => NE + case NE | DIFF => EQ + case GE => LT + case GT => LE + case LE => GT + case LT => GE } } - case object Eq extends Expr("=") with ComparisonOperator - case object Ne extends Expr("<>") with ComparisonOperator - case object Diff extends Expr("!=") with ComparisonOperator - case object Ge extends Expr(">=") with ComparisonOperator - case object Gt extends Expr(">") with ComparisonOperator - case object Le extends Expr("<=") with ComparisonOperator - case object Lt extends Expr("<") with ComparisonOperator - case object In extends Expr("IN") with ComparisonOperator - case object Like extends Expr("LIKE") with ComparisonOperator - case object Between extends Expr("BETWEEN") with ComparisonOperator - case object IsNull extends Expr("IS NULL") with ComparisonOperator - case object IsNotNull extends Expr("IS NOT NULL") with ComparisonOperator + case object EQ extends Expr("=") with ComparisonOperator + case object NE extends Expr("<>") with ComparisonOperator + case object DIFF extends Expr("!=") with ComparisonOperator + case object GE extends Expr(">=") with ComparisonOperator + case object GT extends Expr(">") with ComparisonOperator + case object LE extends Expr("<=") with ComparisonOperator + case object LT extends Expr("<") with ComparisonOperator + case object IN extends Expr("IN") with ComparisonOperator + case object LIKE extends Expr("LIKE") with ComparisonOperator + case object RLIKE extends Expr("RLIKE") with ComparisonOperator + case object BETWEEN extends Expr("BETWEEN") with ComparisonOperator + case object IS_NULL extends Expr("IS NULL") with ComparisonOperator + case object IS_NOT_NULL extends Expr("IS NOT NULL") with ComparisonOperator - case object Match extends Expr("MATCH") with ComparisonOperator - case object Against extends Expr("AGAINST") with TokenRegex + case object MATCH extends Expr("MATCH") with ComparisonOperator + case object AGAINST extends Expr("AGAINST") with TokenRegex sealed trait LogicalOperator extends ExpressionOperator - case object Not extends Expr("NOT") with LogicalOperator + case object NOT extends Expr("NOT") with LogicalOperator sealed trait PredicateOperator extends LogicalOperator - case object And extends Expr("AND") with PredicateOperator - case object Or extends Expr("OR") with PredicateOperator + case object AND extends Expr("AND") with PredicateOperator + case object OR extends Expr("OR") with PredicateOperator - case object Union extends Expr("UNION") with Operator with TokenRegex + case object UNION extends Expr("UNION") with Operator with TokenRegex sealed trait ElasticOperator extends Operator with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala index b7086499..c51b5b93 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala @@ -8,17 +8,17 @@ package object time { override def script: String = sql override def toString: String = s" $sql " override def painless: String = this match { - case Plus => ".plus" - case Minus => ".minus" + case PLUS => ".plus" + case MINUS => ".minus" case _ => sql } } - case object Plus extends Expr("+") with IntervalOperator { + case object PLUS extends Expr("+") with IntervalOperator { override def painless: String = ".plus" } - case object Minus extends Expr("-") with IntervalOperator { + case object MINUS extends Expr("-") with IntervalOperator { override def painless: String = ".minus" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index b80f94c5..6b352528 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic -import app.softnetwork.elastic.sql.function.aggregate.{Max, Min} +import app.softnetwork.elastic.sql.function.aggregate.{MAX, MIN} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.parser.Validation import app.softnetwork.elastic.sql.query._ @@ -65,12 +65,12 @@ package object sql { None else operator match { - case Some(Eq) => values.find(_ == value) - case Some(Ne | Diff) => values.find(_ != value) - case Some(Ge) => values.filter(_ >= value).sorted.reverse.headOption - case Some(Gt) => values.filter(_ > value).sorted.reverse.headOption - case Some(Le) => values.filter(_ <= value).sorted.headOption - case Some(Lt) => values.filter(_ < value).sorted.headOption + case Some(EQ) => values.find(_ == value) + case Some(NE | DIFF) => values.find(_ != value) + case Some(GE) => values.filter(_ >= value).sorted.reverse.headOption + case Some(GT) => values.filter(_ > value).sorted.reverse.headOption + case Some(LE) => values.filter(_ <= value).sorted.headOption + case Some(LT) => values.filter(_ < value).sorted.headOption case _ => values.headOption } } @@ -120,11 +120,11 @@ package object sql { separator: String = "|" )(implicit ev: R => Ordered[R]): Option[R] = { operator match { - case Some(Eq) => values.find(v => v.toString contentEquals value) - case Some(Ne | Diff) => values.find(v => !(v.toString contentEquals value)) - case Some(Like) => values.find(v => pattern.matcher(v.toString).matches()) - case None => Some(values.mkString(separator)) - case _ => super.choose(values, operator, separator) + case Some(EQ) => values.find(v => v.toString contentEquals value) + case Some(NE | DIFF) => values.find(v => !(v.toString contentEquals value)) + case Some(LIKE | RLIKE) => values.find(v => pattern.matcher(v.toString).matches()) + case None => Some(values.mkString(separator)) + case _ => super.choose(values, operator, separator) } } override def out: SQLType = SQLTypes.Varchar @@ -303,8 +303,8 @@ package object sql { value.choose[T](values, Some(operator)) case _ => function match { - case Some(Min) => Some(values.min) - case Some(Max) => Some(values.max) + case Some(MIN) => Some(values.min) + case Some(MAX) => Some(values.max) // FIXME case Some(SQLSum) => Some(values.sum) // FIXME case Some(SQLAvg) => Some(values.sum / values.length ) case _ => values.headOption @@ -313,18 +313,7 @@ package object sql { } def toRegex(value: String): String = { - val startWith = value.startsWith("%") - val endWith = value.endsWith("%") - val v = - if (startWith && endWith) - value.substring(1, value.length - 1) - else if (startWith) - value.substring(1) - else if (endWith) - value.substring(0, value.length - 1) - else - value - s"""${if (startWith) ".*"}$v${if (endWith) ".*"}""" + value.replaceAll("%", ".*").replaceAll("_", ".") } case object Alias extends Expr("AS") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index b85c1cb7..7e1c709a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -45,7 +45,7 @@ object Parser } } - def union: PackratParser[Union.type] = Union.regex ^^ (_ => Union) + def union: PackratParser[UNION.type] = UNION.regex ^^ (_ => UNION) def requests: PackratParser[List[SQLSearchRequest]] = rep1sep(request, union) ^^ (s => s) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index cc76aabf..8d80a804 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -11,29 +11,30 @@ import app.softnetwork.elastic.sql.{ Token } import app.softnetwork.elastic.sql.operator.{ - Against, - And, - Between, + AGAINST, + AND, + BETWEEN, Child, ComparisonOperator, - Diff, - Eq, + DIFF, + EQ, ExpressionOperator, - Ge, - Gt, - In, - IsNotNull, - IsNull, - Le, - Like, - Lt, - Match, - Ne, + GE, + GT, + IN, + IS_NOT_NULL, + IS_NULL, + LE, + LIKE, + LT, + MATCH, + NE, + NOT, Nested, - Not, - Or, + OR, Parent, - PredicateOperator + PredicateOperator, + RLIKE } import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -56,19 +57,19 @@ import app.softnetwork.elastic.sql.query.{ trait WhereParser { self: Parser with GroupByParser with OrderByParser => - def isNull: PackratParser[Criteria] = identifier ~ IsNull.regex ^^ { case i ~ _ => + def isNull: PackratParser[Criteria] = identifier ~ IS_NULL.regex ^^ { case i ~ _ => IsNullExpr(i) } - def isNotNull: PackratParser[Criteria] = identifier ~ IsNotNull.regex ^^ { case i ~ _ => + def isNotNull: PackratParser[Criteria] = identifier ~ IS_NOT_NULL.regex ^^ { case i ~ _ => IsNotNullExpr(i) } - private def eq: PackratParser[ComparisonOperator] = Eq.sql ^^ (_ => Eq) + private def eq: PackratParser[ComparisonOperator] = EQ.sql ^^ (_ => EQ) - private def ne: PackratParser[ComparisonOperator] = Ne.sql ^^ (_ => Ne) + private def ne: PackratParser[ComparisonOperator] = NE.sql ^^ (_ => NE) - private def diff: PackratParser[ComparisonOperator] = Diff.sql ^^ (_ => Diff) + private def diff: PackratParser[ComparisonOperator] = DIFF.sql ^^ (_ => DIFF) private def any_identifier: PackratParser[Identifier] = identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | date_diff_identifier | extract_identifier | identifier @@ -79,24 +80,29 @@ trait WhereParser { } def like: PackratParser[GenericExpression] = - any_identifier ~ not.? ~ Like.regex ~ literal ^^ { case i ~ n ~ _ ~ v => - GenericExpression(i, Like, v, n) + any_identifier ~ not.? ~ LIKE.regex ~ literal ^^ { case i ~ n ~ _ ~ v => + GenericExpression(i, LIKE, v, n) } - private def ge: PackratParser[ComparisonOperator] = Ge.sql ^^ (_ => Ge) + def rlike: PackratParser[GenericExpression] = + any_identifier ~ not.? ~ RLIKE.regex ~ literal ^^ { case i ~ n ~ _ ~ v => + GenericExpression(i, RLIKE, v, n) + } + + private def ge: PackratParser[ComparisonOperator] = GE.sql ^^ (_ => GE) - def gt: PackratParser[ComparisonOperator] = Gt.sql ^^ (_ => Gt) + def gt: PackratParser[ComparisonOperator] = GT.sql ^^ (_ => GT) - private def le: PackratParser[ComparisonOperator] = Le.sql ^^ (_ => Le) + private def le: PackratParser[ComparisonOperator] = LE.sql ^^ (_ => LE) - def lt: PackratParser[ComparisonOperator] = Lt.sql ^^ (_ => Lt) + def lt: PackratParser[ComparisonOperator] = LT.sql ^^ (_ => LT) private def comparison: PackratParser[GenericExpression] = not.? ~ any_identifier ~ (ge | gt | le | lt) ~ (double | pi | long | literal | any_identifier) ^^ { case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) } - def in: PackratParser[ExpressionOperator] = In.regex ^^ (_ => In) + def in: PackratParser[ExpressionOperator] = IN.regex ^^ (_ => IN) private def inLiteral: PackratParser[Criteria] = any_identifier ~ not.? ~ in ~ start ~ rep1sep(literal, separator) ~ end ^^ { @@ -133,17 +139,17 @@ trait WhereParser { } def between: PackratParser[Criteria] = - any_identifier ~ not.? ~ Between.regex ~ literal ~ and ~ literal ^^ { + any_identifier ~ not.? ~ BETWEEN.regex ~ literal ~ and ~ literal ^^ { case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LiteralFromTo(from, to), n) } def betweenLongs: PackratParser[Criteria] = - any_identifier ~ not.? ~ Between.regex ~ long ~ and ~ long ^^ { + any_identifier ~ not.? ~ BETWEEN.regex ~ long ~ and ~ long ^^ { case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, LongFromTo(from, to), n) } def betweenDoubles: PackratParser[Criteria] = - any_identifier ~ not.? ~ Between.regex ~ double ~ and ~ double ^^ { + any_identifier ~ not.? ~ BETWEEN.regex ~ double ~ and ~ double ^^ { case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) } @@ -153,18 +159,18 @@ trait WhereParser { } def matchCriteria: PackratParser[MatchCriteria] = - Match.regex ~ start ~ rep1sep( + MATCH.regex ~ start ~ rep1sep( any_identifier, separator - ) ~ end ~ Against.regex ~ start ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ _ ~ _ ~ l ~ _ => + ) ~ end ~ AGAINST.regex ~ start ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ _ ~ _ ~ l ~ _ => MatchCriteria(i, l) } - def and: PackratParser[PredicateOperator] = And.regex ^^ (_ => And) + def and: PackratParser[PredicateOperator] = AND.regex ^^ (_ => AND) - def or: PackratParser[PredicateOperator] = Or.regex ^^ (_ => Or) + def or: PackratParser[PredicateOperator] = OR.regex ^^ (_ => OR) - def not: PackratParser[Not.type] = Not.regex ^^ (_ => Not) + def not: PackratParser[NOT.type] = NOT.regex ^^ (_ => NOT) def logical_criteria: PackratParser[Criteria] = (is_null | is_notnull) ^^ { case ConditionalFunctionAsCriteria(c) => @@ -172,7 +178,7 @@ trait WhereParser { } def criteria: PackratParser[Criteria] = - (equality | like | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ sql_distance | matchCriteria | logical_criteria) ^^ ( + (equality | like | rlike | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ sql_distance | matchCriteria | logical_criteria) ^^ ( c => c ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index a20bac13..24e07ca0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -1,22 +1,22 @@ package app.softnetwork.elastic.sql.parser.function import app.softnetwork.elastic.sql.Identifier -import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, Avg, Count, Max, Min, Sum} +import app.softnetwork.elastic.sql.function.aggregate.{AVG, AggregateFunction, COUNT, MAX, MIN, SUM} import app.softnetwork.elastic.sql.parser.Parser package object aggregate { trait AggregateParser { self: Parser => - def count: PackratParser[AggregateFunction] = Count.regex ^^ (_ => Count) + def count: PackratParser[AggregateFunction] = COUNT.regex ^^ (_ => COUNT) - def min: PackratParser[AggregateFunction] = Min.regex ^^ (_ => Min) + def min: PackratParser[AggregateFunction] = MIN.regex ^^ (_ => MIN) - def max: PackratParser[AggregateFunction] = Max.regex ^^ (_ => Max) + def max: PackratParser[AggregateFunction] = MAX.regex ^^ (_ => MAX) - def avg: PackratParser[AggregateFunction] = Avg.regex ^^ (_ => Avg) + def avg: PackratParser[AggregateFunction] = AVG.regex ^^ (_ => AVG) - def sum: PackratParser[AggregateFunction] = Sum.regex ^^ (_ => Sum) + def sum: PackratParser[AggregateFunction] = SUM.regex ^^ (_ => SUM) def aggregates: PackratParser[AggregateFunction] = count | min | max | avg | sum diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala index 4afe71ea..f7956052 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala @@ -5,13 +5,13 @@ import app.softnetwork.elastic.sql.function.cond.{ Case, Coalesce, ConditionalFunction, - Else, - End, - IsNotNullFunction, - IsNullFunction, + ELSE, + END, + IsNotNull, + IsNull, NullIf, - Then, - When + THEN, + WHEN } import app.softnetwork.elastic.sql.{Identifier, Null, PainlessScript, Token} import app.softnetwork.elastic.sql.parser.{ @@ -29,12 +29,12 @@ package object cond { def is_null: PackratParser[ConditionalFunction[_]] = "(?i)isnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => IsNullFunction(i) + case _ ~ _ ~ i ~ _ => IsNull(i) } def is_notnull: PackratParser[ConditionalFunction[_]] = "(?i)isnotnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => IsNotNullFunction(i) + case _ ~ _ ~ i ~ _ => IsNotNull(i) } def coalesce: PackratParser[Coalesce] = @@ -52,13 +52,13 @@ package object cond { def start_case: PackratParser[StartCase.type] = Case.regex ^^ (_ => StartCase) - def when_case: PackratParser[WhenCase.type] = When.regex ^^ (_ => WhenCase) + def when_case: PackratParser[WhenCase.type] = WHEN.regex ^^ (_ => WhenCase) - def then_case: PackratParser[ThenCase.type] = Then.regex ^^ (_ => ThenCase) + def then_case: PackratParser[ThenCase.type] = THEN.regex ^^ (_ => ThenCase) - def else_case: PackratParser[Else.type] = Else.regex ^^ (_ => Else) + def else_case: PackratParser[ELSE.type] = ELSE.regex ^^ (_ => ELSE) - def end_case: PackratParser[EndCase.type] = End.regex ^^ (_ => EndCase) + def end_case: PackratParser[EndCase.type] = END.regex ^^ (_ => EndCase) def case_condition: Parser[(PainlessScript, PainlessScript)] = when_case ~ (whereCriteria | valueExpr) ~ then_case.? ~ valueExpr ^^ { case _ ~ c ~ _ ~ r => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index 536e50f7..a39155b9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -10,7 +10,7 @@ import app.softnetwork.elastic.sql.function.string.{ StringFunction, StringFunctionWithOp, Substring, - To, + TO, Trim, Upper } @@ -27,7 +27,7 @@ package object string { } def substringFunction: PackratParser[StringFunction[SQLVarchar]] = - Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((To.regex | separator) ~ long).? ~ end ^^ { + Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((TO.regex | separator) ~ long).? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ eOpt ~ _ => Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala index 6d8ae538..77d3e3c8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala @@ -3,28 +3,28 @@ package app.softnetwork.elastic.sql.parser.operator import app.softnetwork.elastic.sql.function.{Function, FunctionWithIdentifier} import app.softnetwork.elastic.sql.{Identifier, PainlessScript} import app.softnetwork.elastic.sql.operator.math.{ - Add, + ADD, ArithmeticExpression, ArithmeticOperator, - Divide, - Modulo, - Multiply, - Subtract + DIVIDE, + MODULO, + MULTIPLY, + SUBTRACT } import app.softnetwork.elastic.sql.parser.Parser package object math { trait ArithmeticParser { self: Parser => - def add: PackratParser[ArithmeticOperator] = Add.sql ^^ (_ => Add) + def add: PackratParser[ArithmeticOperator] = ADD.sql ^^ (_ => ADD) - def subtract: PackratParser[ArithmeticOperator] = Subtract.sql ^^ (_ => Subtract) + def subtract: PackratParser[ArithmeticOperator] = SUBTRACT.sql ^^ (_ => SUBTRACT) - def multiply: PackratParser[ArithmeticOperator] = Multiply.sql ^^ (_ => Multiply) + def multiply: PackratParser[ArithmeticOperator] = MULTIPLY.sql ^^ (_ => MULTIPLY) - def divide: PackratParser[ArithmeticOperator] = Divide.sql ^^ (_ => Divide) + def divide: PackratParser[ArithmeticOperator] = DIVIDE.sql ^^ (_ => DIVIDE) - def modulo: PackratParser[ArithmeticOperator] = Modulo.sql ^^ (_ => Modulo) + def modulo: PackratParser[ArithmeticOperator] = MODULO.sql ^^ (_ => MODULO) def factor: PackratParser[PainlessScript] = "(" ~> arithmeticExpressionLevel2 <~ ")" ^^ { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 7504beaf..f404adef 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -65,7 +65,7 @@ object BucketSelectorScript { val leftStr = toPainless(left) val rightStr = toPainless(right) val opStr = op match { - case And | Or => op.painless + case AND | OR => op.painless case _ => throw new IllegalArgumentException(s"Unsupported logical operator: $op") } val not = maybeNot.nonEmpty diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 67ac671c..66be42ee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -2,11 +2,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function._ -import app.softnetwork.elastic.sql.function.cond.{ - ConditionalFunction, - IsNotNullFunction, - IsNullFunction -} +import app.softnetwork.elastic.sql.function.cond.{ConditionalFunction, IsNotNull, IsNull} import app.softnetwork.elastic.sql.function.geo.Distance import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ @@ -51,7 +47,7 @@ sealed trait Criteria extends Updateable with PainlessScript { val leftStr = left.painless val rightStr = right.painless val opStr = op match { - case And | Or => op.painless + case AND | OR => op.painless case _ => throw new IllegalArgumentException(s"Unsupported logical operator: $op") } val not = maybeNot.nonEmpty @@ -70,7 +66,7 @@ case class Predicate( leftCriteria: Criteria, operator: PredicateOperator, rightCriteria: Criteria, - not: Option[Not.type] = None, + not: Option[NOT.type] = None, group: Boolean = false ) extends Criteria { override def sql = s"${if (group) s"($leftCriteria" @@ -104,12 +100,12 @@ case class Predicate( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = { val query = asBoolQuery(currentQuery) operator match { - case And => + case AND => (not match { case Some(_) => query.not(rightCriteria.asFilter(Option(query))) case _ => query.filter(rightCriteria.asFilter(Option(query))) }).filter(leftCriteria.asFilter(Option(query))) - case Or => + case OR => (not match { case Some(_) => query.not(rightCriteria.asFilter(Option(query))) case _ => query.should(rightCriteria.asFilter(Option(query))) @@ -191,7 +187,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { override lazy val limit: Option[Limit] = identifier.limit override val functions: List[Function] = identifier.functions def maybeValue: Option[Token] - def maybeNot: Option[Not.type] + def maybeNot: Option[NOT.type] def notAsString: String = maybeNot.map(v => s"$v ").getOrElse("") def valueAsString: String = maybeValue.map(v => s" $v").getOrElse("") override def sql = s"$identifier $notAsString$operator$valueAsString" @@ -277,7 +273,7 @@ case class GenericExpression( identifier: Identifier, operator: ExpressionOperator, value: Token, - maybeNot: Option[Not.type] = None + maybeNot: Option[NOT.type] = None ) extends Expression { override def maybeValue: Option[Token] = Option(value) @@ -298,11 +294,11 @@ case class GenericExpression( } case class IsNullExpr(identifier: Identifier) extends Expression { - override val operator: Operator = IsNull + override val operator: Operator = IS_NULL override def maybeValue: Option[Token] = None - override def maybeNot: Option[Not.type] = None + override def maybeNot: Option[NOT.type] = None override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) @@ -316,11 +312,11 @@ case class IsNullExpr(identifier: Identifier) extends Expression { } case class IsNotNullExpr(identifier: Identifier) extends Expression { - override val operator: Operator = IsNotNull + override val operator: Operator = IS_NOT_NULL override def maybeValue: Option[Token] = None - override def maybeNot: Option[Not.type] = None + override def maybeNot: Option[NOT.type] = None override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) @@ -336,7 +332,7 @@ case class IsNotNullExpr(identifier: Identifier) extends Expression { sealed trait CriteriaWithConditionalFunction[In <: SQLType] extends Expression { def conditionalFunction: ConditionalFunction[In] override def maybeValue: Option[Token] = None - override def maybeNot: Option[Not.type] = None + override def maybeNot: Option[NOT.type] = None override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this override val functions: List[Function] = List(conditionalFunction) override def sql: String = conditionalFunction.sql @@ -344,15 +340,15 @@ sealed trait CriteriaWithConditionalFunction[In <: SQLType] extends Expression { object ConditionalFunctionAsCriteria { def unapply(f: ConditionalFunction[_]): Option[Criteria] = f match { - case IsNullFunction(id) => Some(IsNullCriteria(id)) - case IsNotNullFunction(id) => Some(IsNotNullCriteria(id)) - case _ => None + case IsNull(id) => Some(IsNullCriteria(id)) + case IsNotNull(id) => Some(IsNotNullCriteria(id)) + case _ => None } } case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { - override val conditionalFunction: ConditionalFunction[SQLAny] = IsNullFunction(identifier) - override val operator: Operator = IsNull + override val conditionalFunction: ConditionalFunction[SQLAny] = IsNull(identifier) + override val operator: Operator = IS_NULL override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -371,10 +367,10 @@ case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditiona case class IsNotNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { - override val conditionalFunction: ConditionalFunction[SQLAny] = IsNotNullFunction( + override val conditionalFunction: ConditionalFunction[SQLAny] = IsNotNull( identifier ) - override val operator: Operator = IsNotNull + override val operator: Operator = IS_NOT_NULL override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -395,7 +391,7 @@ case class IsNotNullCriteria(identifier: Identifier) case class InExpr[R, +T <: Value[R]]( identifier: Identifier, values: Values[R, T], - maybeNot: Option[Not.type] = None + maybeNot: Option[NOT.type] = None ) extends Expression { this: InExpr[R, T] => private[this] lazy val id = functions.headOption match { case Some(f) => s"$f($identifier)" @@ -403,7 +399,7 @@ case class InExpr[R, +T <: Value[R]]( } override def sql = s"$id $notAsString$operator $values" - override def operator: Operator = In + override def operator: Operator = IN override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -422,7 +418,7 @@ case class InExpr[R, +T <: Value[R]]( case class BetweenExpr[+T]( identifier: Identifier, fromTo: FromTo[T], - maybeNot: Option[Not.type] + maybeNot: Option[NOT.type] ) extends Expression { private[this] lazy val id = functions.headOption match { case Some(f) => s"$f($identifier)" @@ -430,7 +426,7 @@ case class BetweenExpr[+T]( } override def sql = s"$id $notAsString$operator $fromTo" - override def operator: Operator = Between + override def operator: Operator = BETWEEN override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { @@ -452,13 +448,13 @@ case class ElasticGeoDistance( ) extends Expression { override def sql = s"$Distance($identifier,($lat,$lon)) $operator $distance" override val functions: List[Function] = List(Distance) - override def operator: Operator = Le + override def operator: Operator = LE override def update(request: SQLSearchRequest): ElasticGeoDistance = this.copy(identifier = identifier.update(request)) override def maybeValue: Option[Token] = Some(distance) - override def maybeNot: Option[Not.type] = None + override def maybeNot: Option[NOT.type] = None override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } @@ -468,8 +464,8 @@ case class MatchCriteria( value: StringValue ) extends Criteria { override def sql: String = - s"$operator (${identifiers.mkString(",")}) $Against ($value)" - override def operator: Operator = Match + s"$operator (${identifiers.mkString(",")}) $AGAINST ($value)" + override def operator: Operator = MATCH override def update(request: SQLSearchRequest): Criteria = this.copy(identifiers = identifiers.map(_.update(request))) @@ -479,8 +475,8 @@ case class MatchCriteria( private[this] def toCriteria(matches: List[ElasticMatch], curr: Criteria): Criteria = matches match { case Nil => curr - case single :: Nil => Predicate(curr, Or, single) - case first :: rest => toCriteria(rest, Predicate(curr, Or, first)) + case single :: Nil => Predicate(curr, OR, single) + case first :: rest => toCriteria(rest, Predicate(curr, OR, first)) } lazy val criteria: Criteria = @@ -510,13 +506,13 @@ case class ElasticMatch( ) extends Expression { override def sql: String = s"$operator($identifier,$value${options.map(o => s""","$o"""").getOrElse("")})" - override def operator: Operator = Match + override def operator: Operator = MATCH override def update(request: SQLSearchRequest): Criteria = this.copy(identifier = identifier.update(request)) override def maybeValue: Option[Token] = Some(value) - override def maybeNot: Option[Not.type] = None + override def maybeNot: Option[NOT.type] = None override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index 6a85414c..5b73d02c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -6,9 +6,7 @@ import scala.util.matching.Regex package object time { - sealed trait TimeField extends PainlessScript { - lazy val regex: Regex = s"\\b(?i)$sql\\b".r - + sealed trait TimeField extends PainlessScript with TokenRegex { override def painless: String = s"ChronoField.$sql" override def nullable: Boolean = false @@ -16,18 +14,42 @@ package object time { object TimeField { case object YEAR extends Expr("YEAR") with TimeField - case object MONTH_OF_YEAR extends Expr("MONTH_OF_YEAR") with TimeField - case object DAY_OF_MONTH extends Expr("DAY_OF_MONTH") with TimeField - case object DAY_OF_WEEK extends Expr("DAY_OF_WEEK") with TimeField - case object DAY_OF_YEAR extends Expr("DAY_OF_YEAR") with TimeField - case object HOUR_OF_DAY extends Expr("HOUR_OF_DAY") with TimeField - case object MINUTE_OF_HOUR extends Expr("MINUTE_OF_HOUR") with TimeField - case object SECOND_OF_MINUTE extends Expr("SECOND_OF_MINUTE") with TimeField - case object NANO_OF_SECOND extends Expr("NANO_OF_SECOND") with TimeField - case object MICRO_OF_SECOND extends Expr("MICRO_OF_SECOND") with TimeField - case object MILLI_OF_SECOND extends Expr("MILLI_OF_SECOND") with TimeField - case object EPOCH_DAY extends Expr("EPOCH_DAY") with TimeField - case object OFFSET_SECONDS extends Expr("OFFSET_SECONDS") with TimeField + case object MONTH_OF_YEAR extends Expr("MONTH_OF_YEAR") with TimeField { + override val words: List[String] = List(sql, "MONTHOFYEAR", "MONTH") + } + case object DAY_OF_MONTH extends Expr("DAY_OF_MONTH") with TimeField { + override val words: List[String] = List(sql, "DAYOFMONTH", "DAY") + } + case object DAY_OF_WEEK extends Expr("DAY_OF_WEEK") with TimeField { + override val words: List[String] = List(sql, "DAYOFWEEK", "WEEKDAY") + } + case object DAY_OF_YEAR extends Expr("DAY_OF_YEAR") with TimeField { + override val words: List[String] = List(sql, "DAYOFYEAR") + } + case object HOUR_OF_DAY extends Expr("HOUR_OF_DAY") with TimeField { + override val words: List[String] = List(sql, "HOUROFDAY", "HOUR") + } + case object MINUTE_OF_HOUR extends Expr("MINUTE_OF_HOUR") with TimeField { + override val words: List[String] = List(sql, "MINUTEOFHOUR", "MINUTE") + } + case object SECOND_OF_MINUTE extends Expr("SECOND_OF_MINUTE") with TimeField { + override val words: List[String] = List(sql, "SECONDOFMINUTE", "SECOND") + } + case object NANO_OF_SECOND extends Expr("NANO_OF_SECOND") with TimeField { + override val words: List[String] = List(sql, "NANOFSECOND", "NANOSECOND") + } + case object MICRO_OF_SECOND extends Expr("MICRO_OF_SECOND") with TimeField { + override val words: List[String] = List(sql, "MICROOFSECOND", "MICROSECOND") + } + case object MILLI_OF_SECOND extends Expr("MILLI_OF_SECOND") with TimeField { + override val words: List[String] = List(sql, "MILLIOFSECOND", "MILLISECOND") + } + case object EPOCH_DAY extends Expr("EPOCH_DAY") with TimeField { + override val words: List[String] = List(sql, "EPOCHDAY") + } + case object OFFSET_SECONDS extends Expr("OFFSET_SECONDS") with TimeField { + override val words: List[String] = List(sql, "OFFSETSECONDS") + } } sealed trait TimeUnit extends PainlessScript with MathScript { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index b747fa8e..51606963 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -20,7 +20,8 @@ object Queries { val literalNe = """select * from Table where identifier <> 'un'""" val boolEq = """select * from Table where identifier = true""" val boolNe = """select * from Table where identifier <> false""" - val literalLike = """select * from Table where identifier like '%un%'""" + val literalLike = """select * from Table where identifier like '%u_n%'""" + val literalRlike = """select * from Table where identifier rlike '.*u.n.*'""" val literalNotLike = """select * from Table where identifier not like '%un%'""" val betweenExpression = """select * from Table where identifier between '1' and '2'""" val andPredicate = "select * from Table where identifier1 = 1 and identifier2 > 2" @@ -235,6 +236,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalLike) shouldBe true } + it should "parse literal rlike" in { + val result = Parser(literalRlike) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") + .equalsIgnoreCase(literalRlike) shouldBe true + } + it should "parse literal not like" in { val result = Parser(literalNotLike) result.toOption From d6ca17a11c8d51389cf4fcf217c61167744e87a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 18:00:11 +0200 Subject: [PATCH 14/48] update SQL queries to test --- .../elastic/sql/SQLParserSpec.scala | 230 +++++++++--------- 1 file changed, 115 insertions(+), 115 deletions(-) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 51606963..d8ae889c 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -6,164 +6,164 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers object Queries { - val numericalEq = "select t.col1, t.col2 from Table as t where t.identifier = 1.0" - val numericalLt = "select * from Table where identifier < 1" - val numericalLe = "select * from Table where identifier <= 1" - val numericalGt = "select * from Table where identifier > 1" - val numericalGe = "select * from Table where identifier >= 1" - val numericalNe = "select * from Table where identifier <> 1" - val literalEq = """select * from Table where identifier = 'un'""" - val literalLt = "select * from Table where createdAt < 'now-35M/M'" - val literalLe = "select * from Table where createdAt <= 'now-35M/M'" - val literalGt = "select * from Table where createdAt > 'now-35M/M'" - val literalGe = "select * from Table where createdAt >= 'now-35M/M'" - val literalNe = """select * from Table where identifier <> 'un'""" - val boolEq = """select * from Table where identifier = true""" - val boolNe = """select * from Table where identifier <> false""" - val literalLike = """select * from Table where identifier like '%u_n%'""" - val literalRlike = """select * from Table where identifier rlike '.*u.n.*'""" - val literalNotLike = """select * from Table where identifier not like '%un%'""" - val betweenExpression = """select * from Table where identifier between '1' and '2'""" - val andPredicate = "select * from Table where identifier1 = 1 and identifier2 > 2" - val orPredicate = "select * from Table where identifier1 = 1 or identifier2 > 2" + val numericalEq = "SELECT t.col1, t.col2 FROM Table AS t WHERE t.identifier = 1.0" + val numericalLt = "SELECT * FROM Table WHERE identifier < 1" + val numericalLe = "SELECT * FROM Table WHERE identifier <= 1" + val numericalGt = "SELECT * FROM Table WHERE identifier > 1" + val numericalGe = "SELECT * FROM Table WHERE identifier >= 1" + val numericalNe = "SELECT * FROM Table WHERE identifier <> 1" + val literalEq = """SELECT * FROM Table WHERE identifier = 'un'""" + val literalLt = "SELECT * FROM Table WHERE createdAt < 'now-35M/M'" + val literalLe = "SELECT * FROM Table WHERE createdAt <= 'now-35M/M'" + val literalGt = "SELECT * FROM Table WHERE createdAt > 'now-35M/M'" + val literalGe = "SELECT * FROM Table WHERE createdAt >= 'now-35M/M'" + val literalNe = """SELECT * FROM Table WHERE identifier <> 'un'""" + val boolEq = """SELECT * FROM Table WHERE identifier = true""" + val boolNe = """SELECT * FROM Table WHERE identifier <> false""" + val literalLike = """SELECT * FROM Table WHERE identifier LIKE '%u_n%'""" + val literalRlike = """SELECT * FROM Table WHERE identifier RLIKE '.*u.n.*'""" + val literalNotLike = """SELECT * FROM Table WHERE identifier NOT LIKE '%un%'""" + val betweenExpression = """SELECT * FROM Table WHERE identifier BETWEEN '1' AND '2'""" + val andPredicate = "SELECT * FROM Table WHERE identifier1 = 1 AND identifier2 > 2" + val orPredicate = "SELECT * FROM Table WHERE identifier1 = 1 OR identifier2 > 2" val leftPredicate = - "select * from Table where (identifier1 = 1 and identifier2 > 2) or identifier3 = 3" + "SELECT * FROM Table WHERE (identifier1 = 1 AND identifier2 > 2) OR identifier3 = 3" val rightPredicate = - "select * from Table where identifier1 = 1 and (identifier2 > 2 or identifier3 = 3)" + "SELECT * FROM Table WHERE identifier1 = 1 AND (identifier2 > 2 OR identifier3 = 3)" val predicates = - "select * from Table where (identifier1 = 1 and identifier2 > 2) or (identifier3 = 3 and identifier4 = 4)" + "SELECT * FROM Table WHERE (identifier1 = 1 AND identifier2 > 2) OR (identifier3 = 3 AND identifier4 = 4)" val nestedPredicate = - "select * from Table where identifier1 = 1 and nested(nested.identifier2 > 2 or nested.identifier3 = 3)" + "SELECT * FROM Table WHERE identifier1 = 1 AND nested(nested.identifier2 > 2 OR nested.identifier3 = 3)" val nestedCriteria = - "select * from Table where identifier1 = 1 and nested(nested.identifier3 = 3)" + "SELECT * FROM Table WHERE identifier1 = 1 AND nested(nested.identifier3 = 3)" val childPredicate = - "select * from Table where identifier1 = 1 and child(child.identifier2 > 2 or child.identifier3 = 3)" - val childCriteria = "select * from Table where identifier1 = 1 and child(child.identifier3 = 3)" + "SELECT * FROM Table WHERE identifier1 = 1 AND child(child.identifier2 > 2 OR child.identifier3 = 3)" + val childCriteria = "SELECT * FROM Table WHERE identifier1 = 1 AND child(child.identifier3 = 3)" val parentPredicate = - "select * from Table where identifier1 = 1 and parent(parent.identifier2 > 2 or parent.identifier3 = 3)" + "SELECT * FROM Table WHERE identifier1 = 1 AND parent(parent.identifier2 > 2 OR parent.identifier3 = 3)" val parentCriteria = - "select * from Table where identifier1 = 1 and parent(parent.identifier3 = 3)" - val inLiteralExpression = "select * from Table where identifier in ('val1','val2','val3')" - val inNumericalExpressionWithIntValues = "select * from Table where identifier in (1,2,3)" + "SELECT * FROM Table WHERE identifier1 = 1 AND parent(parent.identifier3 = 3)" + val inLiteralExpression = "SELECT * FROM Table WHERE identifier IN ('val1','val2','val3')" + val inNumericalExpressionWithIntValues = "SELECT * FROM Table WHERE identifier IN (1,2,3)" val inNumericalExpressionWithDoubleValues = - "select * from Table where identifier in (1.0,2.1,3.4)" + "SELECT * FROM Table WHERE identifier IN (1.0,2.1,3.4)" val notInLiteralExpression = - "select * from Table where identifier not in ('val1','val2','val3')" - val notInNumericalExpressionWithIntValues = "select * from Table where identifier not in (1,2,3)" + "SELECT * FROM Table WHERE identifier NOT IN ('val1','val2','val3')" + val notInNumericalExpressionWithIntValues = "SELECT * FROM Table WHERE identifier NOT IN (1,2,3)" val notInNumericalExpressionWithDoubleValues = - "select * from Table where identifier not in (1.0,2.1,3.4)" + "SELECT * FROM Table WHERE identifier NOT IN (1.0,2.1,3.4)" val nestedWithBetween = - "select * from Table where nested(ciblage.Archivage_CreationDate between 'now-3M/M' and 'now' and ciblage.statutComportement = 1)" - val count = "select count(t.id) as c1 from Table as t where t.nom = 'Nom'" - val countDistinct = "select count(distinct t.id) as c2 from Table as t where t.nom = 'Nom'" + "SELECT * FROM Table WHERE nested(ciblage.Archivage_CreationDate BETWEEN 'now-3M/M' AND 'now' AND ciblage.statutComportement = 1)" + val COUNT = "SELECT COUNT(t.id) AS c1 FROM Table AS t WHERE t.nom = 'Nom'" + val countDistinct = "SELECT COUNT(distinct t.id) AS c2 FROM Table AS t WHERE t.nom = 'Nom'" val countNested = - "select count(email.value) as email from crmgp where profile.postalCode in ('75001','75002')" - val isNull = "select * from Table where identifier is null" - val isNotNull = "select * from Table where identifier is not null" + "SELECT COUNT(email.value) AS email FROM crmgp WHERE profile.postalCode IN ('75001','75002')" + val isNull = "SELECT * FROM Table WHERE identifier is null" + val isNotNull = "SELECT * FROM Table WHERE identifier is NOT null" val geoDistanceCriteria = - "select * from Table where distance(profile.location,(-70.0,40.0)) <= '5km'" - val except = "select * except(col1,col2) from Table" + "SELECT * FROM Table WHERE distance(profile.location,(-70.0,40.0)) <= '5km'" + val except = "SELECT * except(col1,col2) FROM Table" val matchCriteria = - "select * from Table where match (identifier1,identifier2,identifier3) against ('value')" + "SELECT * FROM Table WHERE match (identifier1,identifier2,identifier3) against ('value')" val groupBy = - "select identifier, count(identifier2) from Table where identifier2 is not null group by identifier" - val orderBy = "select * from Table order by identifier desc" - val limit = "select * from Table limit 10" + "SELECT identifier, COUNT(identifier2) FROM Table WHERE identifier2 is NOT null group by identifier" + val orderBy = "SELECT * FROM Table order by identifier desc" + val limit = "SELECT * FROM Table limit 10" val groupByWithOrderByAndLimit: String = - """select identifier, count(identifier2) - |from Table - |where identifier is not null + """SELECT identifier, COUNT(identifier2) + |FROM Table + |WHERE identifier is NOT null |group by identifier |order by identifier2 desc |limit 10""".stripMargin.replaceAll("\n", " ") val groupByWithHaving: String = - """select count(CustomerID) as cnt, City, Country - |from Customers + """SELECT COUNT(CustomerID) AS cnt, City, Country + |FROM Customers |group by Country, City - |having Country <> 'USA' and City <> 'Berlin' and count(CustomerID) > 1 - |order by count(CustomerID) desc, Country asc""".stripMargin.replaceAll("\n", " ") + |having Country <> 'USA' AND City <> 'Berlin' AND COUNT(CustomerID) > 1 + |order by COUNT(CustomerID) desc, Country asc""".stripMargin.replaceAll("\n", " ") val dateTimeWithIntervalFields: String = - "select current_timestamp() - interval 3 day as ct, current_date as cd, current_time as t, now as n from dual" + "SELECT current_timestamp() - INTERVAL 3 day AS ct, CURRENT_DATE AS cd, current_time AS t, now AS n FROM dual" val fieldsWithInterval: String = - "select createdAt - interval 35 minute as ct, identifier from Table" + "SELECT createdAt - INTERVAL 35 MINUTE AS ct, identifier FROM Table" val filterWithDateTimeAndInterval: String = - "select * from Table where createdAt < current_timestamp() and createdAt >= current_timestamp() - interval 10 day" + "SELECT * FROM Table WHERE createdAt < current_timestamp() AND createdAt >= current_timestamp() - INTERVAL 10 day" val filterWithDateAndInterval: String = - "select * from Table where createdAt < current_date and createdAt >= current_date() - interval 10 day" + "SELECT * FROM Table WHERE createdAt < CURRENT_DATE AND createdAt >= CURRENT_DATE() - INTERVAL 10 day" val filterWithTimeAndInterval: String = - "select * from Table where createdAt < current_time and createdAt >= current_time() - interval 10 minute" + "SELECT * FROM Table WHERE createdAt < current_time AND createdAt >= current_time() - INTERVAL 10 MINUTE" val groupByWithHavingAndDateTimeFunctions: String = - """select count(CustomerID) as cnt, City, Country, max(createdAt) as lastSeen - |from Table + """SELECT COUNT(CustomerID) AS cnt, City, Country, MAX(createdAt) AS lastSeen + |FROM Table |group by Country, City - |having Country <> 'USA' and City != 'Berlin' and count(CustomerID) > 1 and lastSeen > now - interval 7 day + |having Country <> 'USA' AND City != 'Berlin' AND COUNT(CustomerID) > 1 AND lastSeen > now - INTERVAL 7 day |order by Country asc""".stripMargin .replaceAll("\n", " ") val dateParse = - "select identifier, count(identifier2) as ct, max(date_parse(createdAt, 'yyyy-MM-dd')) as lastSeen from Table where identifier2 is not null group by identifier order by count(identifier2) desc" + "SELECT identifier, COUNT(identifier2) AS ct, MAX(date_parse(createdAt, 'yyyy-MM-dd')) AS lastSeen FROM Table WHERE identifier2 is NOT null group by identifier order by COUNT(identifier2) desc" val dateTimeParse: String = - """select identifier, count(identifier2) as ct, - |max( + """SELECT identifier, COUNT(identifier2) AS ct, + |MAX( |year( |date_trunc( |datetime_parse( |createdAt, |'yyyy-MM-ddTHH:mm:ssZ' - |), minute))) as lastSeen - |from Table - |where identifier2 is not null + |), MINUTE))) AS lastSeen + |FROM Table + |WHERE identifier2 is NOT null |group by identifier - |order by count(identifier2) desc""".stripMargin + |order by COUNT(identifier2) desc""".stripMargin .replaceAll("\n", " ") .replaceAll("\\( ", "(") .replaceAll(" \\)", ")") - val dateDiff = "select date_diff(createdAt, updatedAt, day) as diff, identifier from Table" + val dateDiff = "SELECT date_diff(createdAt, updatedAt, day) AS diff, identifier FROM Table" val aggregationWithDateDiff = - "select max(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, day)) as max_diff from Table group by identifier" + "SELECT MAX(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, day)) AS max_diff FROM Table group by identifier" val dateFormat = - "select identifier, date_format(date_trunc(lastUpdated, month), 'yyyy-MM-dd') as lastSeen from Table where identifier2 is not null" + "SELECT identifier, date_format(date_trunc(lastUpdated, month), 'yyyy-MM-dd') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeFormat = - "select identifier, datetime_format(date_trunc(lastUpdated, month), 'yyyy-MM-ddThh:mm:ssZ') as lastSeen from Table where identifier2 is not null" + "SELECT identifier, datetime_format(date_trunc(lastUpdated, month), 'yyyy-MM-ddThh:mm:ssZ') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateAdd = - "select identifier, date_add(lastUpdated, interval 10 day) as lastSeen from Table where identifier2 is not null" + "SELECT identifier, date_add(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateSub = - "select identifier, date_sub(lastUpdated, interval 10 day) as lastSeen from Table where identifier2 is not null" + "SELECT identifier, date_sub(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeAdd = - "select identifier, datetime_add(lastUpdated, interval 10 day) as lastSeen from Table where identifier2 is not null" + "SELECT identifier, datetime_add(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeSub = - "select identifier, datetime_sub(lastUpdated, interval 10 day) as lastSeen from Table where identifier2 is not null" + "SELECT identifier, datetime_sub(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" - val isnull = "select isnull(identifier) as flag from Table" - val isnotnull = "select identifier, isnotnull(identifier2) as flag from Table" - val isNullCriteria = "select * from Table where isnull(identifier)" - val isNotNullCriteria = "select * from Table where isnotnull(identifier)" + val isnull = "SELECT ISNULL(identifier) AS flag FROM Table" + val isnotnull = "SELECT identifier, ISNOTNULL(identifier2) AS flag FROM Table" + val isNullCriteria = "SELECT * FROM Table WHERE ISNULL(identifier)" + val isNotNullCriteria = "SELECT * FROM Table WHERE ISNOTNULL(identifier)" val coalesce: String = - "select coalesce(createdAt - interval 35 minute, current_date) as c, identifier from Table" + "SELECT COALESCE(createdAt - INTERVAL 35 MINUTE, CURRENT_DATE) AS c, identifier FROM Table" val nullif: String = - "select coalesce(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd') - interval 2 day), current_date) as c, identifier from Table" + "SELECT COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 day), CURRENT_DATE) AS c, identifier FROM Table" val cast: String = - "select cast(coalesce(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd')), current_date - interval 2 hour) bigint) as c, identifier from Table" + "SELECT CAST(COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 hour) bigint) AS c, identifier FROM Table" val allCasts = - "select cast(identifier as int) as c1, cast(identifier as bigint) as c2, cast(identifier as double) as c3, cast(identifier as real) as c4, cast(identifier as boolean) as c5, cast(identifier as char) as c6, cast(identifier as varchar) as c7, cast(createdAt as date) as c8, cast(createdAt as time) as c9, cast(createdAt as datetime) as c10, cast(createdAt as timestamp) as c11, cast(identifier as smallint) as c12, cast(identifier as tinyint) as c13 from Table" + "SELECT CAST(identifier AS int) AS c1, CAST(identifier AS bigint) AS c2, CAST(identifier AS double) AS c3, CAST(identifier AS real) AS c4, CAST(identifier AS boolean) AS c5, CAST(identifier AS char) AS c6, CAST(identifier AS varchar) AS c7, CAST(createdAt AS date) AS c8, CAST(createdAt AS time) AS c9, CAST(createdAt AS datetime) AS c10, CAST(createdAt AS timestamp) AS c11, CAST(identifier AS smallint) AS c12, CAST(identifier AS tinyint) AS c13 FROM Table" val caseWhen: String = - "select case when lastUpdated > now - interval 7 day then lastUpdated when isnotnull(lastSeen) then lastSeen + interval 2 day else createdAt end as c, identifier from Table" + "SELECT CASE WHEN lastUpdated > now - INTERVAL 7 day THEN lastUpdated WHEN ISNOTNULL(lastSeen) THEN lastSeen + INTERVAL 2 day ELSE createdAt END AS c, identifier FROM Table" val caseWhenExpr: String = - "select case current_date - interval 7 day when cast(lastUpdated as date) - interval 3 day then lastUpdated when lastSeen then lastSeen + interval 2 day else createdAt end as c, identifier from Table" + "SELECT CASE CURRENT_DATE - INTERVAL 7 day WHEN CAST(lastUpdated AS date) - INTERVAL 3 day THEN lastUpdated WHEN lastSeen THEN lastSeen + INTERVAL 2 day ELSE createdAt END AS c, identifier FROM Table" val extract: String = - "select extract(day_of_month from createdAt) as dom, extract(day_of_week from createdAt) as dow, extract(day_of_year from createdAt) as doy, extract(month_of_year from createdAt) as m, extract(year from createdAt) as y, extract(hour_of_day from createdAt) as h, extract(minute_of_hour from createdAt) as minutes, extract(second_of_minute from createdAt) as s from Table" + "SELECT EXTRACT(day_of_month FROM createdAt) AS dom, EXTRACT(day_of_week FROM createdAt) AS dow, EXTRACT(day_of_year FROM createdAt) AS doy, EXTRACT(month_of_year FROM createdAt) AS m, EXTRACT(year FROM createdAt) AS y, EXTRACT(hour_of_day FROM createdAt) AS h, EXTRACT(minute_of_hour FROM createdAt) AS minutes, EXTRACT(second_of_minute FROM createdAt) AS s FROM Table" val arithmetic: String = - "select identifier, identifier + 1 as add, identifier - 1 as sub, identifier * 2 as mul, identifier / 2 as div, identifier % 2 as mod, (identifier * identifier2) - 10 as group1 from Table where identifier * (extract(year from current_date) - 10) > 10000" + "SELECT identifier, identifier + 1 AS add, identifier - 1 AS sub, identifier * 2 AS mul, identifier / 2 AS div, identifier % 2 AS mod, (identifier * identifier2) - 10 FROM Table WHERE identifier * (EXTRACT(year FROM CURRENT_DATE) - 10) > 10000" val mathematical: String = - "select identifier, (abs(identifier) + 1.0) * 2, ceil(identifier), floor(identifier), sqrt(identifier), exp(identifier), log(identifier), log10(identifier), pow(identifier, 3), round(identifier), round(identifier, 2), sign(identifier), cos(identifier), acos(identifier), sin(identifier), asin(identifier), tan(identifier), atan(identifier), atan2(identifier, 3.0) from Table where sqrt(identifier) > 100.0" + "SELECT identifier, (ABS(identifier) + 1.0) * 2, CEIL(identifier), FLOOR(identifier), SQRT(identifier), EXP(identifier), LOG(identifier), LOG10(identifier), POW(identifier, 3), ROUND(identifier), ROUND(identifier, 2), SIGN(identifier), COS(identifier), ACOS(identifier), SIN(identifier), ASIN(identifier), TAN(identifier), ATAN(identifier), ATAN2(identifier, 3.0) FROM Table WHERE SQRT(identifier) > 100.0" val string: String = - "select identifier, length(identifier2) as l, lower(identifier2) as low, upper(identifier2) as upp, substring(identifier2, 1, 3) as sub, trim(identifier2) as tr, concat(identifier2, '_test', 1) as con from Table where length(trim(identifier2)) > 10" + "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" } /** Created by smanciot on 15/02/17. @@ -228,7 +228,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalEq) shouldBe true } - it should "parse literal like" in { + it should "parse literal LIKE" in { val result = Parser(literalLike) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -236,7 +236,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalLike) shouldBe true } - it should "parse literal rlike" in { + it should "parse literal RLIKE" in { val result = Parser(literalRlike) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -244,7 +244,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalRlike) shouldBe true } - it should "parse literal not like" in { + it should "parse literal NOT LIKE" in { val result = Parser(literalNotLike) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -305,7 +305,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(boolNe) shouldBe true } - it should "parse between" in { + it should "parse BETWEEN" in { val result = Parser(betweenExpression) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -313,7 +313,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(betweenExpression) shouldBe true } - it should "parse and predicate" in { + it should "parse AND predicate" in { val result = Parser(andPredicate) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -321,7 +321,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(andPredicate) shouldBe true } - it should "parse or predicate" in { + it should "parse OR predicate" in { val result = Parser(orPredicate) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -425,7 +425,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(inNumericalExpressionWithDoubleValues) shouldBe true } - it should "parse not in literal expression" in { + it should "parse NOT in literal expression" in { val result = Parser(notInLiteralExpression) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -433,7 +433,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(notInLiteralExpression) shouldBe true } - it should "parse not in numerical expression with Int values" in { + it should "parse NOT in numerical expression with Int values" in { val result = Parser(notInNumericalExpressionWithIntValues) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -441,7 +441,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(notInNumericalExpressionWithIntValues) shouldBe true } - it should "parse not in numerical expression with Double values" in { + it should "parse NOT in numerical expression with Double values" in { val result = Parser(notInNumericalExpressionWithDoubleValues) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -449,7 +449,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(notInNumericalExpressionWithDoubleValues) shouldBe true } - it should "parse nested with between" in { + it should "parse nested with BETWEEN" in { val result = Parser(nestedWithBetween) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -457,15 +457,15 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(nestedWithBetween) shouldBe true } - it should "parse count" in { - val result = Parser(count) + it should "parse COUNT" in { + val result = Parser(COUNT) result.toOption .flatMap(_.left.toOption.map(_.sql)) .getOrElse("") - .equalsIgnoreCase(count) shouldBe true + .equalsIgnoreCase(COUNT) shouldBe true } - it should "parse distinct count" in { + it should "parse distinct COUNT" in { val result = Parser(countDistinct) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -473,7 +473,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(countDistinct) shouldBe true } - it should "parse count with nested criteria" in { + it should "parse COUNT with nested criteria" in { val result = Parser(countNested) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -489,7 +489,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(isNull) shouldBe true } - it should "parse is not null" in { + it should "parse is NOT null" in { val result = Parser(isNotNull) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -545,7 +545,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(limit) shouldBe true } - it should "parse group by with order by and limit" in { + it should "parse group by with order by AND limit" in { val result = Parser(groupByWithOrderByAndLimit) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -569,7 +569,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(dateTimeWithIntervalFields) shouldBe true } - it should "parse fields with interval" in { + it should "parse fields with INTERVAL" in { val result = Parser(fieldsWithInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -577,7 +577,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(fieldsWithInterval) shouldBe true } - it should "parse filter with date time and interval" in { + it should "parse filter with date time AND INTERVAL" in { val result = Parser(filterWithDateTimeAndInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -585,7 +585,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithDateTimeAndInterval) shouldBe true } - it should "parse filter with date and interval" in { + it should "parse filter with date AND INTERVAL" in { val result = Parser(filterWithDateAndInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -593,7 +593,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithDateAndInterval) shouldBe true } - it should "parse filter with time and interval" in { + it should "parse filter with time AND INTERVAL" in { val result = Parser(filterWithTimeAndInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -601,7 +601,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithTimeAndInterval) shouldBe true } - it should "parse group by with having and date time functions" in { + it should "parse group by with having AND date time functions" in { val result = Parser(groupByWithHavingAndDateTimeFunctions) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -753,7 +753,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(allCasts) shouldBe true } - it should "parse case when expression" in { + it should "parse CASE WHEN expression" in { val result = Parser(caseWhen) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -761,7 +761,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(caseWhen) shouldBe true } - it should "parse case when with expression" in { + it should "parse CASE WHEN with expression" in { val result = Parser(caseWhenExpr) result.toOption .flatMap(_.left.toOption.map(_.sql)) From f29663cd263c4d131e2fb29313d02fe1dc2eb904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 23 Sep 2025 19:01:06 +0200 Subject: [PATCH 15/48] init top hits aggregation --- .../sql/function/aggregate/package.scala | 46 ++++++++++++++++++- .../app/softnetwork/elastic/sql/package.scala | 2 +- .../elastic/sql/parser/Parser.scala | 2 +- .../elastic/sql/parser/SelectParser.scala | 2 +- .../parser/function/aggregate/package.scala | 30 ++++++++++-- .../elastic/sql/SQLParserSpec.scala | 10 ++++ 6 files changed, 84 insertions(+), 8 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index f6e3eb93..37c9f8e1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -1,6 +1,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, TokenRegex} +import app.softnetwork.elastic.sql.query.OrderBy +import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex} package object aggregate { @@ -16,6 +17,47 @@ package object aggregate { case object SUM extends Expr("SUM") with AggregateFunction - case object Sum extends Expr("SUM") with AggregateFunction + sealed trait TopHits extends TokenRegex + + case object FIRST_VALUE extends Expr("FIRST_VALUE") with TopHits + + case object LAST_VALUE extends Expr("LAST_VALUE") with TopHits + + case object OVER extends Expr("OVER") with TokenRegex + + case object PARTITION_BY extends Expr("PARTITION BY") with TokenRegex + + sealed abstract class TopHitsAggregation( + identifier: Identifier, + partitionBy: Seq[Identifier] = Seq.empty, + orderBy: OrderBy + ) extends AggregateFunction + with FunctionWithIdentifier { + def topHits: TopHits + override def sql: String = { + val partitionByStr = + if (partitionBy.nonEmpty) s"$PARTITION_BY ${partitionBy.mkString(", ")}" + else "" + s"$topHits($identifier) $OVER ($partitionByStr$orderBy)" + } + + override def toSQL(base: String): String = sql + } + + case class FirstValue( + identifier: Identifier, + partitionBy: Seq[Identifier] = Seq.empty, + orderBy: OrderBy + ) extends TopHitsAggregation(identifier, partitionBy, orderBy) { + override def topHits: TopHits = FIRST_VALUE + } + + case class LastValue( + identifier: Identifier, + partitionBy: Seq[Identifier] = Seq.empty, + orderBy: OrderBy + ) extends TopHitsAggregation(identifier, partitionBy, orderBy) { + override def topHits: TopHits = LAST_VALUE + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 6b352528..9feb2696 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -50,7 +50,7 @@ package object sql { abstract class Expr(override val sql: String) extends Token - case object Distinct extends Expr("distinct") with TokenRegex + case object Distinct extends Expr("DISTINCT") with TokenRegex abstract class Value[+T](val value: T)(implicit ev$1: T => Ordered[T]) extends Token diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 7e1c709a..99173aae 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -82,7 +82,7 @@ trait Parser with MathParser with StringParser with TemporalParser - with TypeParser { _: WhereParser => + with TypeParser { _: WhereParser with OrderByParser => def start: PackratParser[Delimiter] = "(" ^^ (_ => StartPredicate) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala index 765d8510..decbd4a4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -6,7 +6,7 @@ trait SelectParser { self: Parser with WhereParser => def field: PackratParser[Field] = - (identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | case_when_identifier | identifier) ~ alias.? ^^ { + (identifierWithTopHits | identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | case_when_identifier | identifier) ~ alias.? ^^ { case i ~ a => Field(i, a) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index 24e07ca0..017b2454 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -1,12 +1,13 @@ package app.softnetwork.elastic.sql.parser.function import app.softnetwork.elastic.sql.Identifier -import app.softnetwork.elastic.sql.function.aggregate.{AVG, AggregateFunction, COUNT, MAX, MIN, SUM} -import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.function.aggregate._ +import app.softnetwork.elastic.sql.parser.{OrderByParser, Parser} +import app.softnetwork.elastic.sql.query.OrderBy package object aggregate { - trait AggregateParser { self: Parser => + trait AggregateParser { self: Parser with OrderByParser => def count: PackratParser[AggregateFunction] = COUNT.regex ^^ (_ => COUNT) @@ -26,6 +27,29 @@ package object aggregate { i.withFunctions(a +: i.functions) } + def partition_by: PackratParser[Seq[Identifier]] = + PARTITION_BY.regex ~> rep1sep(identifier, separator) + + private[this] def top_hits: PackratParser[(Identifier, Seq[Identifier], OrderBy)] = + start ~ identifier ~ end ~ OVER.regex ~ start ~ partition_by.? ~ orderBy ~ end ^^ { + case _ ~ id ~ _ ~ _ ~ _ ~ pb ~ ob ~ _ => + (id, pb.getOrElse(Seq.empty), ob) + } + + def first_value: PackratParser[TopHitsAggregation] = + FIRST_VALUE.regex ~ top_hits ^^ { case _ ~ top => + FirstValue(top._1, top._2, top._3) + } + + def last_value: PackratParser[TopHitsAggregation] = + LAST_VALUE.regex ~ top_hits ^^ { case _ ~ top => + LastValue(top._1, top._2, top._3) + } + + def identifierWithTopHits: PackratParser[Identifier] = (first_value | last_value) ^^ { th => + th.identifier.withFunctions(List(th)) + } + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index d8ae889c..f0c8bf45 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -164,6 +164,9 @@ object Queries { val string: String = "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" + + val topHits: String = + "SELECT department, COUNT(DISTINCT salary), FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC), LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date DESC) FROM emp GROUP BY department" } /** Created by smanciot on 15/02/17. @@ -801,4 +804,11 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(string) shouldBe true } + it should "parse top hits functions" in { + val result = Parser(topHits) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") shouldBe topHits + } + } From 5984945065ef2f40c22f44c0182c87e3f346a148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 24 Sep 2025 14:15:44 +0200 Subject: [PATCH 16/48] generate top hits aggregation --- .../sql/bridge/ElasticAggregation.scala | 48 ++++++++++++- .../elastic/sql/bridge/package.scala | 2 +- .../elastic/sql/SQLQuerySpec.scala | 7 ++ .../sql/bridge/ElasticAggregation.scala | 46 ++++++++++++- .../elastic/sql/bridge/package.scala | 2 +- .../elastic/sql/SQLQuerySpec.scala | 7 ++ .../sql/function/aggregate/package.scala | 67 +++++++++++++++---- .../parser/function/aggregate/package.scala | 2 +- .../elastic/sql/query/SQLSearchRequest.scala | 24 ++++++- .../elastic/sql/query/Select.scala | 19 +++++- .../elastic/sql/type/SQLTypes.scala | 38 +++++------ .../elastic/sql/SQLParserSpec.scala | 2 +- 12 files changed, 218 insertions(+), 46 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index b2f8fdf2..efe255c3 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -5,6 +5,7 @@ import app.softnetwork.elastic.sql.query.{ Bucket, BucketSelectorScript, Criteria, + Desc, ElasticBoolQuery, Field, SortOrder @@ -21,6 +22,7 @@ import com.sksamuel.elastic4s.ElasticApi.{ nestedAggregation, sumAgg, termsAgg, + topHitsAgg, valueCountAgg } import com.sksamuel.elastic4s.script.Script @@ -31,6 +33,7 @@ import com.sksamuel.elastic4s.searches.aggs.{ TermsAggregation, TermsOrder } +import com.sksamuel.elastic4s.searches.sort.FieldSort import scala.language.implicitConversions @@ -78,8 +81,15 @@ object ElasticAggregation { field else if (distinct) s"${aggType}_distinct_${sourceField.replace(".", "_")}" - else - s"${aggType}_${sourceField.replace(".", "_")}" + else { + aggType match { + case th: TopHitsAggregation => + s"${th.topHits.sql.toLowerCase}_${sourceField.replace(".", "_")}" + case _ => + s"${aggType}_${sourceField.replace(".", "_")}" + + } + } } var aggPath = Seq[String]() @@ -113,6 +123,40 @@ object ElasticAggregation { case MAX => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) + case th: TopHitsAggregation => + val topHits = + topHitsAgg(aggName) + .fetchSource( + th.identifier.name +: th.fields + .filterNot(_.isScriptField) + .map(_.sourceField) + .toArray, + Array.empty + ) + .copy( + scripts = th.fields + .filter(_.isScriptField) + .map(f => f.sourceField -> Script(f.painless).lang("painless")) + .toMap + ) + .size(1) sortBy th.orderBy.sorts.map(sort => + sort.order match { + case Some(Desc) => + th.topHits match { + case FIRST_VALUE => FieldSort(sort.field).desc() + case LAST_VALUE => FieldSort(sort.field).asc() + } + case _ => + th.topHits match { + case FIRST_VALUE => FieldSort(sort.field).asc() + case LAST_VALUE => FieldSort(sort.field).desc() + } + } + ) + /*th.fields.filter(_.isScriptField).foldLeft(topHits) { (agg, f) => + agg.script(f.sourceField, Script(f.painless, lang = Some("painless"))) + }*/ + topHits } val filteredAggName = "filtered_agg" diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 0fcbc175..827aef7a 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -100,7 +100,7 @@ package object bridge { } } - _search = scriptFields match { + _search = scriptFields.filterNot(_.aggregation) match { case Nil => _search case _ => _search scriptfields scriptFields.map { field => diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index e726b389..e2d0f187 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2431,4 +2431,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("false:", "false : ") } + it should "handle top hits aggregation" in { + val select: ElasticSearchRequest = + SQLQuery(topHits) + val query = select.query + println(query) + } + } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 230fc276..c86b7d9f 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -7,6 +7,7 @@ import app.softnetwork.elastic.sql.query.{ Field, Bucket, Criteria, + Desc, SortOrder } import app.softnetwork.elastic.sql.function._ @@ -21,6 +22,7 @@ import com.sksamuel.elastic4s.ElasticApi.{ nestedAggregation, sumAgg, termsAgg, + topHitsAgg, valueCountAgg } import com.sksamuel.elastic4s.requests.script.Script @@ -31,6 +33,7 @@ import com.sksamuel.elastic4s.requests.searches.aggs.{ TermsAggregation, TermsOrder, } +import com.sksamuel.elastic4s.requests.searches.sort.FieldSort import scala.language.implicitConversions @@ -77,8 +80,15 @@ object ElasticAggregation { field else if (distinct) s"${aggType}_distinct_${sourceField.replace(".", "_")}" - else - s"${aggType}_${sourceField.replace(".", "_")}" + else { + aggType match { + case th: TopHitsAggregation => + s"${th.topHits.sql.toLowerCase}_${sourceField.replace(".", "_")}" + case _ => + s"${aggType}_${sourceField.replace(".", "_")}" + + } + } } var aggPath = Seq[String]() @@ -112,6 +122,38 @@ object ElasticAggregation { case MAX => aggWithFieldOrScript(maxAgg, (name, s) => maxAgg(name, sourceField).script(s)) case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) + case th: TopHitsAggregation => + val topHits = + topHitsAgg(aggName) + .fetchSource( + th.identifier.name +: th.fields + .filterNot(_.isScriptField) + .map(_.sourceField) + .toArray, + Array.empty + ).copy( + scripts = th.fields.filter(_.isScriptField).map(f => + f.sourceField -> Script(f.painless).lang("painless") + ).toMap + ) + .size(1) sortBy th.orderBy.sorts.map(sort => + sort.order match { + case Some(Desc) => + th.topHits match { + case FIRST_VALUE => FieldSort(sort.field).desc() + case LAST_VALUE => FieldSort(sort.field).asc() + } + case _ => + th.topHits match { + case FIRST_VALUE => FieldSort(sort.field).asc() + case LAST_VALUE => FieldSort(sort.field).desc() + } + } + ) + /*th.fields.filter(_.isScriptField).foldLeft(topHits) { (agg, f) => + agg.script(f.sourceField, Script(f.painless, lang = Some("painless"))) + }*/ + topHits } val filteredAggName = "filtered_agg" diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 93812ea1..76eae43c 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -102,7 +102,7 @@ package object bridge { } } - _search = scriptFields match { + _search = scriptFields.filterNot(_.aggregation) match { case Nil => _search case _ => _search scriptfields scriptFields.map { field => diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 4f526caf..689e6c6e 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2420,4 +2420,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("false:", "false : ") } + it should "handle top hits aggregation" in { + val select: ElasticSearchRequest = + SQLQuery(topHits) + val query = select.query + println(query) + } + } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 37c9f8e1..2b0b5680 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.query.OrderBy -import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex} +import app.softnetwork.elastic.sql.query.{Bucket, Field, OrderBy, SQLSearchRequest} +import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} package object aggregate { @@ -19,21 +19,32 @@ package object aggregate { sealed trait TopHits extends TokenRegex - case object FIRST_VALUE extends Expr("FIRST_VALUE") with TopHits + case object FIRST_VALUE extends Expr("FIRST_VALUE") with TopHits { + override val words: List[String] = List(sql, "FIRST") + } - case object LAST_VALUE extends Expr("LAST_VALUE") with TopHits + case object LAST_VALUE extends Expr("LAST_VALUE") with TopHits { + override val words: List[String] = List(sql, "LAST") + } case object OVER extends Expr("OVER") with TokenRegex case object PARTITION_BY extends Expr("PARTITION BY") with TokenRegex - sealed abstract class TopHitsAggregation( - identifier: Identifier, - partitionBy: Seq[Identifier] = Seq.empty, - orderBy: OrderBy - ) extends AggregateFunction - with FunctionWithIdentifier { + sealed trait TopHitsAggregation + extends AggregateFunction + with FunctionWithIdentifier + with Updateable { + def partitionBy: Seq[Identifier] + def orderBy: OrderBy def topHits: TopHits + + lazy val buckets: Seq[Bucket] = partitionBy.map(Bucket) + + lazy val bucketNames: Map[String, Bucket] = buckets.map { b => + b.identifier.identifierName -> b + }.toMap + override def sql: String = { val partitionByStr = if (partitionBy.nonEmpty) s"$PARTITION_BY ${partitionBy.mkString(", ")}" @@ -42,22 +53,50 @@ package object aggregate { } override def toSQL(base: String): String = sql + + def fields: Seq[Field] + + def update(request: SQLSearchRequest): TopHitsAggregation } case class FirstValue( identifier: Identifier, partitionBy: Seq[Identifier] = Seq.empty, - orderBy: OrderBy - ) extends TopHitsAggregation(identifier, partitionBy, orderBy) { + orderBy: OrderBy, + fields: Seq[Field] = Seq.empty + ) extends TopHitsAggregation { override def topHits: TopHits = FIRST_VALUE + override def update(request: SQLSearchRequest): FirstValue = { + val updated = this.copy(partitionBy = partitionBy.map(_.update(request))) + updated.copy( + fields = request.select.fields + .filterNot(field => + field.aggregation || request.bucketNames.keys.toSeq + .contains(field.identifier.identifierName) + ) + .filterNot(f => request.excludes.contains(f.sourceField)) + ) + } } case class LastValue( identifier: Identifier, partitionBy: Seq[Identifier] = Seq.empty, - orderBy: OrderBy - ) extends TopHitsAggregation(identifier, partitionBy, orderBy) { + orderBy: OrderBy, + fields: Seq[Field] = Seq.empty + ) extends TopHitsAggregation { override def topHits: TopHits = LAST_VALUE + override def update(request: SQLSearchRequest): LastValue = { + val updated = this.copy(partitionBy = partitionBy.map(_.update(request))) + updated.copy( + fields = request.select.fields + .filterNot(field => + field.aggregation || request.bucketNames.keys.toSeq + .contains(field.identifier.identifierName) + ) + .filterNot(f => request.excludes.contains(f.sourceField)) + ) + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index 017b2454..8b9a9b6e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -47,7 +47,7 @@ package object aggregate { } def identifierWithTopHits: PackratParser[Identifier] = (first_value | last_value) ^^ { th => - th.identifier.withFunctions(List(th)) + th.identifier.withFunctions(th +: th.identifier.functions) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala index c3595606..2587292e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.function.aggregate.TopHitsAggregation import app.softnetwork.elastic.sql.{asString, Identifier, Token} case class SQLSearchRequest( @@ -18,7 +19,10 @@ case class SQLSearchRequest( lazy val fieldAliases: Map[String, String] = select.fieldAliases lazy val tableAliases: Map[String, String] = from.tableAliases lazy val unnests: Seq[(String, String, Option[Limit])] = from.unnests - lazy val bucketNames: Map[String, Bucket] = groupBy.map(_.bucketNames).getOrElse(Map.empty) + lazy val bucketNames: Map[String, Bucket] = buckets.map { b => + b.identifier.identifierName -> b + }.toMap + lazy val sorts: Map[String, SortOrder] = orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap @@ -44,7 +48,12 @@ case class SQLSearchRequest( Seq.empty } - lazy val aggregates: Seq[Field] = select.fields.filter(_.aggregation) + lazy val topHitsFields: Seq[Field] = select.fields.filter(_.topHits.nonEmpty) + + lazy val topHitsAggs: Seq[TopHitsAggregation] = topHitsFields.flatMap(_.topHits) + + lazy val aggregates: Seq[Field] = + select.fields.filter(_.aggregation).filterNot(_.topHits.isDefined) ++ topHitsFields lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) @@ -52,7 +61,16 @@ case class SQLSearchRequest( source.sql } - lazy val buckets: Seq[Bucket] = groupBy.map(_.buckets).getOrElse(Seq.empty) + lazy val topHitsBuckets: Seq[Bucket] = topHitsAggs + .flatMap(_.bucketNames) + .filterNot(bucket => + groupBy.map(_.bucketNames).getOrElse(Map.empty).keys.toSeq.contains(bucket._1) + ) + .toMap + .values + .toSeq + + lazy val buckets: Seq[Bucket] = groupBy.map(_.buckets).getOrElse(Seq.empty) ++ topHitsBuckets override def validate(): Either[String, Unit] = { for { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 80d7a6b9..d0be9421 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.function.aggregate.TopHitsAggregation import app.softnetwork.elastic.sql.function.{Function, FunctionChain} import app.softnetwork.elastic.sql.{ asString, @@ -44,8 +45,22 @@ case class Field( override def functions: List[Function] = identifier.functions - def update(request: SQLSearchRequest): Field = - this.copy(identifier = identifier.update(request)) + lazy val topHits: Option[TopHitsAggregation] = + functions.collectFirst { case th: TopHitsAggregation => th } + + def update(request: SQLSearchRequest): Field = { + val updated = + topHits match { + case Some(th) => + val topHitsAggregation = th.update(request) + identifier.functions match { + case _ :: tail => identifier.withFunctions(functions = topHitsAggregation +: tail) + case _ => identifier.withFunctions(functions = List(topHitsAggregation)) + } + case None => identifier + } + this.copy(identifier = updated.update(request)) + } def painless: String = identifier.painless diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index afb0c4b1..057448dc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -1,36 +1,36 @@ package app.softnetwork.elastic.sql.`type` object SQLTypes { - case object Any extends SQLAny { val typeId = "any" } + case object Any extends SQLAny { val typeId = "ANY" } - case object Null extends SQLAny { val typeId = "null" } + case object Null extends SQLAny { val typeId = "NULL" } - case object Temporal extends SQLTemporal { val typeId = "temporal" } + case object Temporal extends SQLTemporal { val typeId = "TEMPORAL" } - case object Date extends SQLTemporal with SQLDate { val typeId = "date" } - case object Time extends SQLTemporal with SQLTime { val typeId = "time" } - case object DateTime extends SQLTemporal with SQLDateTime { val typeId = "datetime" } - case object Timestamp extends SQLTimestamp { val typeId = "timestamp" } + case object Date extends SQLTemporal with SQLDate { val typeId = "DATE" } + case object Time extends SQLTemporal with SQLTime { val typeId = "TIME" } + case object DateTime extends SQLTemporal with SQLDateTime { val typeId = "DATETIME" } + case object Timestamp extends SQLTimestamp { val typeId = "TIMESTAMP" } - case object Numeric extends SQLNumeric { val typeId = "numeric" } + case object Numeric extends SQLNumeric { val typeId = "NUMERIC" } - case object TinyInt extends SQLTinyInt { val typeId = "tinyint" } - case object SmallInt extends SQLSmallInt { val typeId = "smallint" } - case object Int extends SQLInt { val typeId = "int" } - case object BigInt extends SQLBigInt { val typeId = "bigint" } - case object Double extends SQLDouble { val typeId = "double" } - case object Real extends SQLReal { val typeId = "real" } + case object TinyInt extends SQLTinyInt { val typeId = "TINYINT" } + case object SmallInt extends SQLSmallInt { val typeId = "SMALLINT" } + case object Int extends SQLInt { val typeId = "INT" } + case object BigInt extends SQLBigInt { val typeId = "BIGINT" } + case object Double extends SQLDouble { val typeId = "DOUBLE" } + case object Real extends SQLReal { val typeId = "REAL" } - case object Literal extends SQLLiteral { val typeId = "literal" } + case object Literal extends SQLLiteral { val typeId = "LITERAL" } - case object Char extends SQLChar { val typeId = "char" } - case object Varchar extends SQLVarchar { val typeId = "varchar" } + case object Char extends SQLChar { val typeId = "CHAR" } + case object Varchar extends SQLVarchar { val typeId = "VARCHAR" } - case object Boolean extends SQLBool { val typeId = "boolean" } + case object Boolean extends SQLBool { val typeId = "BOOLEAN" } case class Array(elementType: SQLType) extends SQLArray { val typeId = s"array<${elementType.typeId}>" } - case object Struct extends SQLStruct { val typeId = "struct" } + case object Struct extends SQLStruct { val typeId = "STRUCT" } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index f0c8bf45..264ef627 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -166,7 +166,7 @@ object Queries { "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" val topHits: String = - "SELECT department, COUNT(DISTINCT salary), FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC), LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date DESC) FROM emp GROUP BY department" + "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary FROM emp" } /** Created by smanciot on 15/02/17. From c87aef61b0024cb1a49aed5247a29c731db9c5d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 24 Sep 2025 14:45:48 +0200 Subject: [PATCH 17/48] add support for group by with field index --- .../elastic/sql/SQLQuerySpec.scala | 2 +- .../app/softnetwork/elastic/sql/package.scala | 4 +- .../elastic/sql/parser/GroupByParser.scala | 2 +- .../elastic/sql/parser/type/package.scala | 6 -- .../elastic/sql/query/GroupBy.scala | 21 +++++- .../elastic/sql/SQLParserSpec.scala | 73 +++++++++---------- 6 files changed, 58 insertions(+), 50 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index e2d0f187..7825d87e 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1084,7 +1084,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle group by with having and date time functions" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHavingAndDateTimeFunctions) + SQLQuery(groupByWithHavingAndDateTimeFunctions.replace("GROUP BY 3, 2", "GROUP BY 3, 2")) val query = select.query println(query) query shouldBe diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 9feb2696..5611c609 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -453,11 +453,11 @@ package object sql { object Identifier { def apply(): Identifier = GenericIdentifier("") - def apply(function: Function): Identifier = GenericIdentifier("", functions = function :: Nil) + def apply(function: Function): Identifier = apply(List(function)) def apply(functions: List[Function]): Identifier = apply().withFunctions(functions) def apply(name: String): Identifier = GenericIdentifier(name) def apply(name: String, function: Function): Identifier = - GenericIdentifier(name, functions = function :: Nil) + apply(name).withFunctions(List(function)) } case class GenericIdentifier( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala index 4e4f683b..c6d74c01 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/GroupByParser.scala @@ -5,7 +5,7 @@ import app.softnetwork.elastic.sql.query.{Bucket, GroupBy} trait GroupByParser { self: Parser with WhereParser => - def bucket: PackratParser[Bucket] = identifier ^^ { i => + def bucket: PackratParser[Bucket] = (long | identifier) ^^ { i => Bucket(i) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index 44e92052..c9dc2063 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -3,7 +3,6 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.{ BooleanValue, DoubleValue, - Identifier, LongValue, PiValue, StringValue, @@ -32,11 +31,6 @@ package object `type` { def boolean: PackratParser[BooleanValue] = """(?i)(true|false)\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) - def value_identifier: PackratParser[Identifier] = - (literal | long | double | pi | boolean) ^^ { v => - Identifier(v) - } - def char_type: PackratParser[SQLTypes.Char.type] = "(?i)char".r ^^ (_ => SQLTypes.Char) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index f404adef..8c073ed2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.operator._ -import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} +import app.softnetwork.elastic.sql.{Expr, Identifier, LongValue, TokenRegex, Updateable} case object GroupBy extends Expr("GROUP BY") with TokenRegex @@ -27,8 +27,23 @@ case class Bucket( identifier: Identifier ) extends Updateable { override def sql: String = s"$identifier" - def update(request: SQLSearchRequest): Bucket = - this.copy(identifier = identifier.update(request)) + def update(request: SQLSearchRequest): Bucket = { + identifier.functions.headOption match { + case Some(func: LongValue) => + if (func.value <= 0) { + throw new IllegalArgumentException(s"Bucket index must be greater than 0: ${func.value}") + } else if (request.select.fields.size < func.value) { + throw new IllegalArgumentException( + s"Bucket index ${func.value} is out of bounds [1, ${request.fields.size}]" + ) + } else { + val field = request.select.fields(func.value.toInt - 1) + this.copy(identifier = field.identifier) + } + case _ => this.copy(identifier = identifier.update(request)) + } + } + lazy val sourceBucket: String = if (identifier.nested) { identifier.tableAlias diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 264ef627..82d8b17e 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -13,10 +13,10 @@ object Queries { val numericalGe = "SELECT * FROM Table WHERE identifier >= 1" val numericalNe = "SELECT * FROM Table WHERE identifier <> 1" val literalEq = """SELECT * FROM Table WHERE identifier = 'un'""" - val literalLt = "SELECT * FROM Table WHERE createdAt < 'now-35M/M'" - val literalLe = "SELECT * FROM Table WHERE createdAt <= 'now-35M/M'" - val literalGt = "SELECT * FROM Table WHERE createdAt > 'now-35M/M'" - val literalGe = "SELECT * FROM Table WHERE createdAt >= 'now-35M/M'" + val literalLt = "SELECT * FROM Table WHERE createdAt < 'NOW-35M/M'" + val literalLe = "SELECT * FROM Table WHERE createdAt <= 'NOW-35M/M'" + val literalGt = "SELECT * FROM Table WHERE createdAt > 'NOW-35M/M'" + val literalGe = "SELECT * FROM Table WHERE createdAt >= 'NOW-35M/M'" val literalNe = """SELECT * FROM Table WHERE identifier <> 'un'""" val boolEq = """SELECT * FROM Table WHERE identifier = true""" val boolNe = """SELECT * FROM Table WHERE identifier <> false""" @@ -53,7 +53,7 @@ object Queries { val notInNumericalExpressionWithDoubleValues = "SELECT * FROM Table WHERE identifier NOT IN (1.0,2.1,3.4)" val nestedWithBetween = - "SELECT * FROM Table WHERE nested(ciblage.Archivage_CreationDate BETWEEN 'now-3M/M' AND 'now' AND ciblage.statutComportement = 1)" + "SELECT * FROM Table WHERE nested(ciblage.Archivage_CreationDate BETWEEN 'NOW-3M/M' AND 'NOW' AND ciblage.statutComportement = 1)" val COUNT = "SELECT COUNT(t.id) AS c1 FROM Table AS t WHERE t.nom = 'Nom'" val countDistinct = "SELECT COUNT(distinct t.id) AS c2 FROM Table AS t WHERE t.nom = 'Nom'" val countNested = @@ -66,41 +66,41 @@ object Queries { val matchCriteria = "SELECT * FROM Table WHERE match (identifier1,identifier2,identifier3) against ('value')" val groupBy = - "SELECT identifier, COUNT(identifier2) FROM Table WHERE identifier2 is NOT null group by identifier" - val orderBy = "SELECT * FROM Table order by identifier desc" + "SELECT identifier, COUNT(identifier2) FROM Table WHERE identifier2 is NOT null GROUP BY identifier" + val orderBy = "SELECT * FROM Table ORDER BY identifier DESC" val limit = "SELECT * FROM Table limit 10" val groupByWithOrderByAndLimit: String = """SELECT identifier, COUNT(identifier2) |FROM Table |WHERE identifier is NOT null - |group by identifier - |order by identifier2 desc + |GROUP BY identifier + |ORDER BY identifier2 DESC |limit 10""".stripMargin.replaceAll("\n", " ") val groupByWithHaving: String = """SELECT COUNT(CustomerID) AS cnt, City, Country |FROM Customers - |group by Country, City - |having Country <> 'USA' AND City <> 'Berlin' AND COUNT(CustomerID) > 1 - |order by COUNT(CustomerID) desc, Country asc""".stripMargin.replaceAll("\n", " ") + |GROUP BY Country, City + |HAVING Country <> 'USA' AND City <> 'Berlin' AND COUNT(CustomerID) > 1 + |ORDER BY COUNT(CustomerID) DESC, Country ASC""".stripMargin.replaceAll("\n", " ") val dateTimeWithIntervalFields: String = - "SELECT current_timestamp() - INTERVAL 3 day AS ct, CURRENT_DATE AS cd, current_time AS t, now AS n FROM dual" + "SELECT current_timestamp() - INTERVAL 3 DAY AS ct, CURRENT_DATE AS cd, current_time AS t, NOW AS n FROM dual" val fieldsWithInterval: String = "SELECT createdAt - INTERVAL 35 MINUTE AS ct, identifier FROM Table" val filterWithDateTimeAndInterval: String = - "SELECT * FROM Table WHERE createdAt < current_timestamp() AND createdAt >= current_timestamp() - INTERVAL 10 day" + "SELECT * FROM Table WHERE createdAt < current_timestamp() AND createdAt >= current_timestamp() - INTERVAL 10 DAY" val filterWithDateAndInterval: String = - "SELECT * FROM Table WHERE createdAt < CURRENT_DATE AND createdAt >= CURRENT_DATE() - INTERVAL 10 day" + "SELECT * FROM Table WHERE createdAt < CURRENT_DATE AND createdAt >= CURRENT_DATE() - INTERVAL 10 DAY" val filterWithTimeAndInterval: String = "SELECT * FROM Table WHERE createdAt < current_time AND createdAt >= current_time() - INTERVAL 10 MINUTE" val groupByWithHavingAndDateTimeFunctions: String = """SELECT COUNT(CustomerID) AS cnt, City, Country, MAX(createdAt) AS lastSeen |FROM Table - |group by Country, City - |having Country <> 'USA' AND City != 'Berlin' AND COUNT(CustomerID) > 1 AND lastSeen > now - INTERVAL 7 day - |order by Country asc""".stripMargin + |GROUP BY Country, City + |HAVING Country <> 'USA' AND City != 'Berlin' AND COUNT(CustomerID) > 1 AND lastSeen > NOW - INTERVAL 7 DAY + |ORDER BY Country ASC""".stripMargin .replaceAll("\n", " ") val dateParse = - "SELECT identifier, COUNT(identifier2) AS ct, MAX(date_parse(createdAt, 'yyyy-MM-dd')) AS lastSeen FROM Table WHERE identifier2 is NOT null group by identifier order by COUNT(identifier2) desc" + "SELECT identifier, COUNT(identifier2) AS ct, MAX(date_parse(createdAt, 'yyyy-MM-dd')) AS lastSeen FROM Table WHERE identifier2 is NOT null GROUP BY identifier ORDER BY COUNT(identifier2) DESC" val dateTimeParse: String = """SELECT identifier, COUNT(identifier2) AS ct, |MAX( @@ -112,29 +112,29 @@ object Queries { |), MINUTE))) AS lastSeen |FROM Table |WHERE identifier2 is NOT null - |group by identifier - |order by COUNT(identifier2) desc""".stripMargin + |GROUP BY identifier + |ORDER BY COUNT(identifier2) DESC""".stripMargin .replaceAll("\n", " ") .replaceAll("\\( ", "(") .replaceAll(" \\)", ")") - val dateDiff = "SELECT date_diff(createdAt, updatedAt, day) AS diff, identifier FROM Table" + val dateDiff = "SELECT date_diff(createdAt, updatedAt, DAY) AS diff, identifier FROM Table" val aggregationWithDateDiff = - "SELECT MAX(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, day)) AS max_diff FROM Table group by identifier" + "SELECT MAX(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, DAY)) AS max_diff FROM Table GROUP BY identifier" val dateFormat = "SELECT identifier, date_format(date_trunc(lastUpdated, month), 'yyyy-MM-dd') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeFormat = "SELECT identifier, datetime_format(date_trunc(lastUpdated, month), 'yyyy-MM-ddThh:mm:ssZ') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateAdd = - "SELECT identifier, date_add(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, date_add(lastUpdated, INTERVAL 10 DAY) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateSub = - "SELECT identifier, date_sub(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, date_sub(lastUpdated, INTERVAL 10 DAY) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeAdd = - "SELECT identifier, datetime_add(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, datetime_add(lastUpdated, INTERVAL 10 DAY) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeSub = - "SELECT identifier, datetime_sub(lastUpdated, INTERVAL 10 day) AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, datetime_sub(lastUpdated, INTERVAL 10 DAY) AS lastSeen FROM Table WHERE identifier2 is NOT null" val isnull = "SELECT ISNULL(identifier) AS flag FROM Table" val isnotnull = "SELECT identifier, ISNOTNULL(identifier2) AS flag FROM Table" @@ -143,15 +143,15 @@ object Queries { val coalesce: String = "SELECT COALESCE(createdAt - INTERVAL 35 MINUTE, CURRENT_DATE) AS c, identifier FROM Table" val nullif: String = - "SELECT COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 day), CURRENT_DATE) AS c, identifier FROM Table" + "SELECT COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 DAY), CURRENT_DATE) AS c, identifier FROM Table" val cast: String = "SELECT CAST(COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 hour) bigint) AS c, identifier FROM Table" val allCasts = "SELECT CAST(identifier AS int) AS c1, CAST(identifier AS bigint) AS c2, CAST(identifier AS double) AS c3, CAST(identifier AS real) AS c4, CAST(identifier AS boolean) AS c5, CAST(identifier AS char) AS c6, CAST(identifier AS varchar) AS c7, CAST(createdAt AS date) AS c8, CAST(createdAt AS time) AS c9, CAST(createdAt AS datetime) AS c10, CAST(createdAt AS timestamp) AS c11, CAST(identifier AS smallint) AS c12, CAST(identifier AS tinyint) AS c13 FROM Table" val caseWhen: String = - "SELECT CASE WHEN lastUpdated > now - INTERVAL 7 day THEN lastUpdated WHEN ISNOTNULL(lastSeen) THEN lastSeen + INTERVAL 2 day ELSE createdAt END AS c, identifier FROM Table" + "SELECT CASE WHEN lastUpdated > NOW - INTERVAL 7 DAY THEN lastUpdated WHEN ISNOTNULL(lastSeen) THEN lastSeen + INTERVAL 2 DAY ELSE createdAt END AS c, identifier FROM Table" val caseWhenExpr: String = - "SELECT CASE CURRENT_DATE - INTERVAL 7 day WHEN CAST(lastUpdated AS date) - INTERVAL 3 day THEN lastUpdated WHEN lastSeen THEN lastSeen + INTERVAL 2 day ELSE createdAt END AS c, identifier FROM Table" + "SELECT CASE CURRENT_DATE - INTERVAL 7 DAY WHEN CAST(lastUpdated AS date) - INTERVAL 3 DAY THEN lastUpdated WHEN lastSeen THEN lastSeen + INTERVAL 2 DAY ELSE createdAt END AS c, identifier FROM Table" val extract: String = "SELECT EXTRACT(day_of_month FROM createdAt) AS dom, EXTRACT(day_of_week FROM createdAt) AS dow, EXTRACT(day_of_year FROM createdAt) AS doy, EXTRACT(month_of_year FROM createdAt) AS m, EXTRACT(year FROM createdAt) AS y, EXTRACT(hour_of_day FROM createdAt) AS h, EXTRACT(minute_of_hour FROM createdAt) AS minutes, EXTRACT(second_of_minute FROM createdAt) AS s FROM Table" @@ -524,7 +524,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(matchCriteria) shouldBe true } - it should "parse group by" in { + it should "parse GROUP BY" in { val result = Parser(groupBy) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -532,7 +532,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(groupBy) shouldBe true } - it should "parse order by" in { + it should "parse ORDER BY" in { val result = Parser(orderBy) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -548,7 +548,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(limit) shouldBe true } - it should "parse group by with order by AND limit" in { + it should "parse GROUP BY with ORDER BY AND limit" in { val result = Parser(groupByWithOrderByAndLimit) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -556,7 +556,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(groupByWithOrderByAndLimit) shouldBe true } - it should "parse group by with having" in { + it should "parse GROUP BY with HAVING" in { val result = Parser(groupByWithHaving) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -604,12 +604,11 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithTimeAndInterval) shouldBe true } - it should "parse group by with having AND date time functions" in { + it should "parse GROUP BY with HAVING AND date time functions" in { val result = Parser(groupByWithHavingAndDateTimeFunctions) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") - .equalsIgnoreCase(groupByWithHavingAndDateTimeFunctions) shouldBe true + .getOrElse("") shouldBe groupByWithHavingAndDateTimeFunctions } it should "parse date_parse function" in { From 5418167137a6e3765d4b4d9536309d863479f044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 24 Sep 2025 14:52:56 +0200 Subject: [PATCH 18/48] update specifications for queries --- .../elastic/sql/SQLQuerySpec.scala | 95 +++++++++++++++++++ .../elastic/sql/SQLQuerySpec.scala | 95 +++++++++++++++++++ .../elastic/sql/SQLParserSpec.scala | 6 +- 3 files changed, 193 insertions(+), 3 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 7825d87e..84ee2566 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2436,6 +2436,101 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { SQLQuery(topHits) val query = select.query println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "size": 0, + | "script_fields": { + | "hire_date": { + | "script": { + | "lang": "painless", + | "source": "(!doc.containsKey('hire_date') || doc['hire_date'].empty ? null : doc['hire_date'].value)" + | } + | } + | }, + | "_source": true, + | "aggs": { + | "dept": { + | "terms": { + | "field": "department.keyword" + | }, + | "aggs": { + | "cnt": { + | "cardinality": { + | "field": "salary" + | } + | }, + | "first_salary": { + | "top_hits": { + | "size": 1, + | "sort": [ + | { + | "hire_date": { + | "order": "asc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "salary", + | "firstName" + | ] + | } + | } + | }, + | "last_salary": { + | "top_hits": { + | "size": 1, + | "sort": [ + | { + | "hire_date": { + | "order": "desc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "salary", + | "firstName" + | ] + | } + | } + | } + | } + | } + | } + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("-", " - ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") } } diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 689e6c6e..79df6c6b 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2425,6 +2425,101 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { SQLQuery(topHits) val query = select.query println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "size": 0, + | "script_fields": { + | "hire_date": { + | "script": { + | "lang": "painless", + | "source": "(!doc.containsKey('hire_date') || doc['hire_date'].empty ? null : doc['hire_date'].value)" + | } + | } + | }, + | "_source": true, + | "aggs": { + | "dept": { + | "terms": { + | "field": "department.keyword" + | }, + | "aggs": { + | "cnt": { + | "cardinality": { + | "field": "salary" + | } + | }, + | "first_salary": { + | "top_hits": { + | "size": 1, + | "sort": [ + | { + | "hire_date": { + | "order": "asc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "salary", + | "firstName" + | ] + | } + | } + | }, + | "last_salary": { + | "top_hits": { + | "size": 1, + | "sort": [ + | { + | "hire_date": { + | "order": "desc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "salary", + | "firstName" + | ] + | } + | } + | } + | } + | } + | } + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("-", " - ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 82d8b17e..15ec429b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -83,15 +83,15 @@ object Queries { |HAVING Country <> 'USA' AND City <> 'Berlin' AND COUNT(CustomerID) > 1 |ORDER BY COUNT(CustomerID) DESC, Country ASC""".stripMargin.replaceAll("\n", " ") val dateTimeWithIntervalFields: String = - "SELECT current_timestamp() - INTERVAL 3 DAY AS ct, CURRENT_DATE AS cd, current_time AS t, NOW AS n FROM dual" + "SELECT CURRENT_TIMESTAMP() - INTERVAL 3 DAY AS ct, CURRENT_DATE AS cd, CURRENT_TIME AS t, NOW AS n FROM dual" val fieldsWithInterval: String = "SELECT createdAt - INTERVAL 35 MINUTE AS ct, identifier FROM Table" val filterWithDateTimeAndInterval: String = - "SELECT * FROM Table WHERE createdAt < current_timestamp() AND createdAt >= current_timestamp() - INTERVAL 10 DAY" + "SELECT * FROM Table WHERE createdAt < CURRENT_TIMESTAMP() AND createdAt >= CURRENT_TIMESTAMP() - INTERVAL 10 DAY" val filterWithDateAndInterval: String = "SELECT * FROM Table WHERE createdAt < CURRENT_DATE AND createdAt >= CURRENT_DATE() - INTERVAL 10 DAY" val filterWithTimeAndInterval: String = - "SELECT * FROM Table WHERE createdAt < current_time AND createdAt >= current_time() - INTERVAL 10 MINUTE" + "SELECT * FROM Table WHERE createdAt < CURRENT_TIME AND createdAt >= CURRENT_TIME() - INTERVAL 10 MINUTE" val groupByWithHavingAndDateTimeFunctions: String = """SELECT COUNT(CustomerID) AS cnt, City, Country, MAX(createdAt) AS lastSeen |FROM Table From dca9e30c0ad5a7f5151ab514d2e6619ea6cc14ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Sep 2025 12:57:08 +0200 Subject: [PATCH 19/48] add support for last day function, update extract function --- .../elastic/sql/SQLQuerySpec.scala | 73 +++++++++++++++- .../elastic/sql/SQLQuerySpec.scala | 73 +++++++++++++++- .../elastic/sql/function/package.scala | 11 ++- .../elastic/sql/function/time/package.scala | 84 ++++++++++++++----- .../elastic/sql/parser/Parser.scala | 8 +- .../elastic/sql/parser/SelectParser.scala | 16 +++- .../elastic/sql/parser/WhereParser.scala | 11 ++- .../sql/parser/function/convert/package.scala | 9 +- .../sql/parser/function/time/package.scala | 31 ++++++- .../elastic/sql/type/SQLTypeUtils.scala | 6 +- .../elastic/sql/SQLParserSpec.scala | 9 ++ 11 files changed, 290 insertions(+), 41 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 84ee2566..fc566c48 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1766,7 +1766,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def v0 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.minus(35, ChronoUnit.MINUTES) : null);if (v0 != null) return v0; return (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()).atStartOfDay(ZoneId.of('Z')); }" + | "source": "{ def v0 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.minus(35, ChronoUnit.MINUTES) : null);if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')); }" | } | } | }, @@ -1859,7 +1859,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()).atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS); }.toInstant().toEpochMilli()" + | "source": "{ def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS); }.toInstant().toEpochMilli()" | } | } | }, @@ -1955,7 +1955,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def expr = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def e0 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def val0 = e0 != null ? (e0.minus(3, ChronoUnit.DAYS)).atStartOfDay(ZoneId.of('Z')) : null; if (expr == val0) return e0; def val1 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value); if (expr == val1) return val1.plus(2, ChronoUnit.DAYS); def dval = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); return dval; }" + | "source": "{ def expr = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def e0 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def val0 = e0 != null ? e0.minus(3, ChronoUnit.DAYS).atStartOfDay(ZoneId.of('Z')) : null; if (expr == val0) return e0; def val1 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value); if (expr == val1) return val1.plus(2, ChronoUnit.DAYS); def dval = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); return dval; }" | } | } | }, @@ -2533,4 +2533,71 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") } + it should "handle last day function" in { + val select: ElasticSearchRequest = + SQLQuery(lastDay) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "bool": { + | "filter": [ + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def e1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); e1.withDayOfMonth(e1.lengthOfMonth())).get(ChronoField.DAY_OF_MONTH) > 28" + | } + | } + | } + | ] + | } + | }, + | "script_fields": { + | "ld": { + | "script": { + | "lang": "painless", + | "source": "(def e1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e1 != null ? e1.withDayOfMonth(e1.lengthOfMonth()) : null)" + | } + | } + | }, + | "_source": { + | "includes": [ + | "identifier" + | ] + | } + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("-", " - ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + } + } diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 79df6c6b..6dde58de 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1755,7 +1755,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def v0 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.minus(35, ChronoUnit.MINUTES) : null);if (v0 != null) return v0; return (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()).atStartOfDay(ZoneId.of('Z')); }" + | "source": "{ def v0 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.minus(35, ChronoUnit.MINUTES) : null);if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')); }" | } | } | }, @@ -1848,7 +1848,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return (ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()).atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS); }.toInstant().toEpochMilli()" + | "source": "{ def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS); }.toInstant().toEpochMilli()" | } | } | }, @@ -1944,7 +1944,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def expr = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def e0 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def val0 = e0 != null ? (e0.minus(3, ChronoUnit.DAYS)).atStartOfDay(ZoneId.of('Z')) : null; if (expr == val0) return e0; def val1 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value); if (expr == val1) return val1.plus(2, ChronoUnit.DAYS); def dval = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); return dval; }" + | "source": "{ def expr = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def e0 = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); def val0 = e0 != null ? e0.minus(3, ChronoUnit.DAYS).atStartOfDay(ZoneId.of('Z')) : null; if (expr == val0) return e0; def val1 = (!doc.containsKey('lastSeen') || doc['lastSeen'].empty ? null : doc['lastSeen'].value); if (expr == val1) return val1.plus(2, ChronoUnit.DAYS); def dval = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); return dval; }" | } | } | }, @@ -2522,4 +2522,71 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") } + it should "handle last day function" in { + val select: ElasticSearchRequest = + SQLQuery(lastDay) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "bool": { + | "filter": [ + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def e1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); e1.withDayOfMonth(e1.lengthOfMonth())).get(ChronoField.DAY_OF_MONTH) > 28" + | } + | } + | } + | ] + | } + | }, + | "script_fields": { + | "ld": { + | "script": { + | "lang": "painless", + | "source": "(def e1 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e1 != null ? e1.withDayOfMonth(e1.lengthOfMonth()) : null)" + | } + | } + | }, + | "_source": { + | "includes": [ + | "identifier" + | ] + | } + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("-", " - ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + } + } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index df688762..a1d74b48 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator @@ -112,6 +112,9 @@ package object function { def fun: Option[PainlessScript] = None def args: List[PainlessScript] + + def argTypes: List[SQLType] = args.map(_.out) + def argsSeparator: String = ", " def inputType: In @@ -139,7 +142,9 @@ package object function { args .filter(_.nullable) .zipWithIndex - .map { case (a, i) => s"def arg$i = ${a.painless};" } + .map { case (a, i) => + s"def arg$i = ${SQLTypeUtils.coerce(a.painless, a.out, argTypes(i), nullable = false)};" + } .mkString(" ") val callArgs = args.zipWithIndex @@ -147,7 +152,7 @@ package object function { if (a.nullable) s"arg$i" else - a.painless + SQLTypeUtils.coerce(a.painless, a.out, argTypes(i), nullable = false) } if (args.exists(_.nullable)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 1f96e094..5fad4b6f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -155,9 +155,12 @@ package object time { override def painless: String = ".get" } - case class Extract(field: TimeField, override val sql: String = Extract.sql) + case class Extract(field: TimeField) extends DateTimeFunction with TransformFunction[SQLTemporal, SQLNumeric] { + + override val sql: String = Extract.sql + override def fun: Option[PainlessScript] = Some(Extract) override def args: List[PainlessScript] = List(field) @@ -171,36 +174,75 @@ package object time { import TimeField._ - object Year extends Extract(YEAR, YEAR.sql) { + sealed trait TimeFieldExtract extends Extract { + override val sql: String = field.sql override def toSQL(base: String): String = s"$sql($base)" } - object MonthOfYear extends Extract(MONTH_OF_YEAR, MONTH_OF_YEAR.sql) { - override def toSQL(base: String): String = s"$sql($base)" - } + object Year extends Extract(YEAR) with TimeFieldExtract - object DayOfMonth extends Extract(DAY_OF_MONTH, DAY_OF_MONTH.sql) { - override def toSQL(base: String): String = s"$sql($base)" - } + object MonthOfYear extends Extract(MONTH_OF_YEAR) with TimeFieldExtract - object DayOfWeek extends Extract(DAY_OF_WEEK, DAY_OF_WEEK.sql) { - override def toSQL(base: String): String = s"$sql($base)" - } + object DayOfMonth extends Extract(DAY_OF_MONTH) with TimeFieldExtract - object DayOfYear extends Extract(DAY_OF_YEAR, DAY_OF_YEAR.sql) { - override def toSQL(base: String): String = s"$sql($base)" - } + object DayOfWeek extends Extract(DAY_OF_WEEK) with TimeFieldExtract - object HourOfDay extends Extract(HOUR_OF_DAY, HOUR_OF_DAY.sql) { - override def toSQL(base: String): String = s"$sql($base)" - } + object DayOfYear extends Extract(DAY_OF_YEAR) with TimeFieldExtract - object MinuteOfHour extends Extract(MINUTE_OF_HOUR, MINUTE_OF_HOUR.sql) { - override def toSQL(base: String): String = s"$sql($base)" + object HourOfDay extends Extract(HOUR_OF_DAY) with TimeFieldExtract + + object MinuteOfHour extends Extract(MINUTE_OF_HOUR) with TimeFieldExtract + + object SecondOfMinute extends Extract(SECOND_OF_MINUTE) with TimeFieldExtract + + object NanoOfSecond extends Extract(NANO_OF_SECOND) with TimeFieldExtract + + object MicroOfSecond extends Extract(MICRO_OF_SECOND) with TimeFieldExtract + + object MilliOfSecond extends Extract(MILLI_OF_SECOND) with TimeFieldExtract + + object EpochDay extends Extract(EPOCH_DAY) with TimeFieldExtract + + object OffsetSeconds extends Extract(OFFSET_SECONDS) with TimeFieldExtract + + case object LastDayOfMonth extends Expr("LAST_DAY") with TokenRegex with PainlessScript { + override def painless: String = ".withDayOfMonth" + override lazy val words: List[String] = List(sql, "LASTDAY") } - object SecondOfMinute extends Extract(SECOND_OF_MINUTE, SECOND_OF_MINUTE.sql) { - override def toSQL(base: String): String = s"$sql($base)" + case class LastDayOfMonth(date: PainlessScript) + extends DateFunction + with TransformFunction[SQLDate, SQLDate] { + override def fun: Option[PainlessScript] = Some(LastDayOfMonth) + + override def args: List[PainlessScript] = List(date) + + override def inputType: SQLDate = SQLTypes.Date + override def outputType: SQLDate = SQLTypes.Date + + override def nullable: Boolean = date.nullable + + override def sql: String = LastDayOfMonth.sql + + override def toSQL(base: String): String = { + s"$sql($base)" + } + + override def toPainless(base: String, idx: Int): String = { + val arg = SQLTypeUtils.coerce(base, date.out, SQLTypes.Date, nullable = false) + if (nullable && base.nonEmpty) + s"(def e$idx = $arg; e$idx != null ? ${toPainlessCall(List(s"e$idx"))} : null)" + else + s"(def e$idx = $arg; ${toPainlessCall(List(s"e$idx"))})" + } + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case arg :: Nil => s"$arg${LastDayOfMonth.painless}($arg.lengthOfMonth())" + case _ => throw new IllegalArgumentException("LastDayOfMonth requires exactly one argument") + } + } + } case object DateDiff extends Expr("DATE_DIFF") with TokenRegex with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 99173aae..1bed7113 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -94,6 +94,7 @@ trait Parser // les plus spécifiques en premier identifierWithTransformation | // transformations appliquées à un identifier date_diff_identifier | // date_diff(...) retournant un identifier-like + last_day_identifier | extract_identifier | identifierWithSystemFunction | // CURRENT_DATE, NOW, etc. (+/- interval) identifierWithIntervalFunction | @@ -248,7 +249,12 @@ trait Parser } def identifierWithTransformation: PackratParser[Identifier] = - mathematicalFunctionWithIdentifier | castFunctionWithIdentifier | conditionalFunctionWithIdentifier | dateFunctionWithIdentifier | dateTimeFunctionWithIdentifier | stringFunctionWithIdentifier + mathematicalFunctionWithIdentifier | + castFunctionWithIdentifier | + conditionalFunctionWithIdentifier | + dateFunctionWithIdentifier | + dateTimeFunctionWithIdentifier | + stringFunctionWithIdentifier def identifierWithFunction: PackratParser[Identifier] = rep1sep( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala index decbd4a4..ea1125bb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -6,9 +6,19 @@ trait SelectParser { self: Parser with WhereParser => def field: PackratParser[Field] = - (identifierWithTopHits | identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | case_when_identifier | identifier) ~ alias.? ^^ { - case i ~ a => - Field(i, a) + (identifierWithTopHits | + identifierWithArithmeticExpression | + identifierWithTransformation | + identifierWithAggregation | + identifierWithSystemFunction | + identifierWithIntervalFunction | + identifierWithFunction | + date_diff_identifier | + last_day_identifier | + extract_identifier | + case_when_identifier | + identifier) ~ alias.? ^^ { case i ~ a => + Field(i, a) } def except: PackratParser[Except] = Except.regex ~ start ~ rep1sep(field, separator) ~ end ^^ { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index 8d80a804..d3452600 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -72,7 +72,16 @@ trait WhereParser { private def diff: PackratParser[ComparisonOperator] = DIFF.sql ^^ (_ => DIFF) private def any_identifier: PackratParser[Identifier] = - identifierWithTransformation | identifierWithAggregation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | date_diff_identifier | extract_identifier | identifier + identifierWithTransformation | + identifierWithAggregation | + identifierWithSystemFunction | + identifierWithIntervalFunction | + identifierWithArithmeticExpression | + identifierWithFunction | + date_diff_identifier | + last_day_identifier | + extract_identifier | + identifier private def equality: PackratParser[GenericExpression] = not.? ~ any_identifier ~ (eq | ne | diff) ~ (boolean | literal | double | pi | long | any_identifier) ^^ { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala index 0724be4f..8e16a6cc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala @@ -9,7 +9,14 @@ package object convert { trait ConvertParser { self: Parser => def castFunctionWithIdentifier: PackratParser[Identifier] = - "(?i)cast".r ~ start ~ (identifierWithTransformation | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | date_diff_identifier | extract_identifier | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { + "(?i)cast".r ~ start ~ (identifierWithTransformation | + identifierWithSystemFunction | + identifierWithIntervalFunction | + identifierWithFunction | + date_diff_identifier | + last_day_identifier | + extract_identifier | + identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index e3097549..710442c8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -174,7 +174,7 @@ package object time { } def extract_identifier: PackratParser[Identifier] = - Extract.regex ~ start ~ time_field ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { + Extract.regex ~ start ~ time_field ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | last_day_identifier | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => i.withFunctions(Extract(u) +: i.functions) } @@ -197,9 +197,36 @@ package object time { MINUTE_OF_HOUR.regex ^^ (_ => MinuteOfHour) def second_of_minute_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = SECOND_OF_MINUTE.regex ^^ (_ => SecondOfMinute) + def nano_of_second_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + NANO_OF_SECOND.regex ^^ (_ => NanoOfSecond) + def micro_of_second_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + MICRO_OF_SECOND.regex ^^ (_ => MicroOfSecond) + def milli_of_second_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + MILLI_OF_SECOND.regex ^^ (_ => MilliOfSecond) + def epoch_day_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + EPOCH_DAY.regex ^^ (_ => EpochDay) + def offset_seconds_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + OFFSET_SECONDS.regex ^^ (_ => OffsetSeconds) def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - year_tr | month_of_year_tr | day_of_month_tr | day_of_week_tr | day_of_year_tr | hour_of_day_tr | minute_of_hour_tr | second_of_minute_tr + year_tr | + month_of_year_tr | + day_of_month_tr | + day_of_week_tr | + day_of_year_tr | + hour_of_day_tr | + minute_of_hour_tr | + second_of_minute_tr | + milli_of_second_tr | + micro_of_second_tr | + nano_of_second_tr | + epoch_day_tr | + offset_seconds_tr + + def last_day_identifier: Parser[Identifier] = + LastDayOfMonth.regex ~ start ~ (castFunctionWithIdentifier | identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { + case _ ~ _ ~ i ~ _ => i.withFunctions(LastDayOfMonth(i) +: i.functions) + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 6571bcba..50c244ec 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -99,11 +99,11 @@ object SQLTypeUtils { (from, to) match { // ---- DATE & TIME ---- case (SQLTypes.Date, SQLTypes.DateTime | SQLTypes.Timestamp) => - s"($expr).atStartOfDay(ZoneId.of('Z'))" + s"$expr.atStartOfDay(ZoneId.of('Z'))" case (SQLTypes.DateTime | SQLTypes.Timestamp, SQLTypes.Date) => - s"($expr).toLocalDate()" + s"$expr.toLocalDate()" case (SQLTypes.DateTime | SQLTypes.Timestamp, SQLTypes.Time) => - s"($expr).toLocalTime()" + s"$expr.toLocalTime()" // ---- NUMERIQUES ---- case (SQLTypes.Int, SQLTypes.BigInt) => diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 15ec429b..98eb7b59 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -167,6 +167,9 @@ object Queries { val topHits: String = "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary FROM emp" + + val lastDay: String = + "SELECT LAST_DAY(CAST(createdAt AS DATE)) AS ld, identifier FROM Table WHERE EXTRACT(DAY_OF_MONTH FROM LAST_DAY(CURRENT_TIMESTAMP)) > 28" } /** Created by smanciot on 15/02/17. @@ -810,4 +813,10 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .getOrElse("") shouldBe topHits } + it should "parse last_day function" in { + val result = Parser(lastDay) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") shouldBe lastDay + } } From af2cbb15f89695a2ef25bab2f3c87a7a399fb1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Sep 2025 18:02:51 +0200 Subject: [PATCH 20/48] add support for today, quarter and week functions, simplify parsing --- .../elastic/sql/SQLQuerySpec.scala | 42 +++++++ .../elastic/sql/SQLQuerySpec.scala | 42 +++++++ .../elastic/sql/function/time/package.scala | 23 +++- .../elastic/sql/parser/Parser.scala | 19 ++- .../elastic/sql/parser/SelectParser.scala | 5 - .../elastic/sql/parser/WhereParser.scala | 4 - .../sql/parser/function/cond/package.scala | 4 +- .../sql/parser/function/convert/package.scala | 4 - .../sql/parser/function/time/package.scala | 96 ++++++++------- .../elastic/sql/parser/time/package.scala | 30 ++++- .../elastic/sql/time/package.scala | 81 ++++++++----- .../elastic/sql/SQLParserSpec.scala | 111 ++++++++++-------- 12 files changed, 300 insertions(+), 161 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index fc566c48..edf5d89f 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2051,6 +2051,48 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" | } + | }, + | "nano": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.NANO_OF_SECOND) : null)" + | } + | }, + | "micro": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MICRO_OF_SECOND) : null)" + | } + | }, + | "milli": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MILLI_OF_SECOND) : null)" + | } + | }, + | "epoch": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.EPOCH_DAY) : null)" + | } + | }, + | "off": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.OFFSET_SECONDS) : null)" + | } + | }, + | "w": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR) : null)" + | } + | }, + | "q": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR) : null)" + | } | } | }, | "_source": true diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 6dde58de..87d5eb36 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2040,6 +2040,48 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" | } + | }, + | "nano": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.NANO_OF_SECOND) : null)" + | } + | }, + | "micro": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MICRO_OF_SECOND) : null)" + | } + | }, + | "milli": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MILLI_OF_SECOND) : null)" + | } + | }, + | "epoch": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.EPOCH_DAY) : null)" + | } + | }, + | "off": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.OFFSET_SECONDS) : null)" + | } + | }, + | "w": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR) : null)" + | } + | }, + | "q": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR) : null)" + | } | } | }, | "_source": true diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 5fad4b6f..f7ceb182 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -12,7 +12,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTypes, SQLVarchar } -import app.softnetwork.elastic.sql.time.{TimeField, TimeInterval, TimeUnit} +import app.softnetwork.elastic.sql.time.{IsoField, TimeField, TimeInterval, TimeUnit} package object time { @@ -129,6 +129,10 @@ package object time { case object NowWithParens extends Expr("NOW()") with CurrentDateTimeFunction + case object Today extends Expr("TODAY") with CurrentDateFunction + + case object TodayWithParens extends Expr("TODAY()") with CurrentDateFunction + case object DateTrunc extends Expr("DATE_TRUNC") with TokenRegex with PainlessScript { override def painless: String = ".truncatedTo" override lazy val words: List[String] = List(sql, "DATETRUNC") @@ -205,22 +209,29 @@ package object time { object OffsetSeconds extends Extract(OFFSET_SECONDS) with TimeFieldExtract + import IsoField._ + + object QuarterOfYear extends Extract(QUARTER_OF_YEAR) with TimeFieldExtract + + object WeekOfWeekBasedYear extends Extract(WEEK_OF_WEEK_BASED_YEAR) with TimeFieldExtract + case object LastDayOfMonth extends Expr("LAST_DAY") with TokenRegex with PainlessScript { override def painless: String = ".withDayOfMonth" override lazy val words: List[String] = List(sql, "LASTDAY") } - case class LastDayOfMonth(date: PainlessScript) + case class LastDayOfMonth(identifier: Identifier) extends DateFunction - with TransformFunction[SQLDate, SQLDate] { + with TransformFunction[SQLDate, SQLDate] + with FunctionWithIdentifier { override def fun: Option[PainlessScript] = Some(LastDayOfMonth) - override def args: List[PainlessScript] = List(date) + override def args: List[PainlessScript] = List(identifier) override def inputType: SQLDate = SQLTypes.Date override def outputType: SQLDate = SQLTypes.Date - override def nullable: Boolean = date.nullable + override def nullable: Boolean = identifier.nullable override def sql: String = LastDayOfMonth.sql @@ -229,7 +240,7 @@ package object time { } override def toPainless(base: String, idx: Int): String = { - val arg = SQLTypeUtils.coerce(base, date.out, SQLTypes.Date, nullable = false) + val arg = SQLTypeUtils.coerce(base, identifier.out, SQLTypes.Date, nullable = false) if (nullable && base.nonEmpty) s"(def e$idx = $arg; e$idx != null ? ${toPainlessCall(List(s"e$idx"))} : null)" else diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 1bed7113..ce00caae 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -91,15 +91,10 @@ trait Parser def separator: PackratParser[Delimiter] = "," ^^ (_ => Separator) def valueExpr: PackratParser[PainlessScript] = - // les plus spécifiques en premier - identifierWithTransformation | // transformations appliquées à un identifier - date_diff_identifier | // date_diff(...) retournant un identifier-like - last_day_identifier | - extract_identifier | - identifierWithSystemFunction | // CURRENT_DATE, NOW, etc. (+/- interval) + // the order is important here + identifierWithTransformation | // transformations applied to an identifier identifierWithIntervalFunction | - identifierWithTemporalFunction | // chaîne de fonctions appliquées à un identifier - identifierWithFunction | // fonctions appliquées à un identifier + identifierWithFunction | // fonctions applied to an identifier literal | // 'string' pi | double | @@ -252,15 +247,19 @@ trait Parser mathematicalFunctionWithIdentifier | castFunctionWithIdentifier | conditionalFunctionWithIdentifier | + systemFunctionWithIdentifier | dateFunctionWithIdentifier | dateTimeFunctionWithIdentifier | - stringFunctionWithIdentifier + stringFunctionWithIdentifier | + date_diff_identifier | + extract_identifier | + case_when_identifier def identifierWithFunction: PackratParser[Identifier] = rep1sep( sql_functions, start - ) ~ start.? ~ (identifierWithSystemFunction | identifierWithIntervalFunction | identifier).? ~ rep1( + ) ~ start.? ~ (identifierWithTransformation | identifierWithIntervalFunction | identifier).? ~ rep1( end ) ^^ { case f ~ _ ~ i ~ _ => i match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala index ea1125bb..6746f1ee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -10,13 +10,8 @@ trait SelectParser { identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | - identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | - date_diff_identifier | - last_day_identifier | - extract_identifier | - case_when_identifier | identifier) ~ alias.? ^^ { case i ~ a => Field(i, a) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index d3452600..e4c7d454 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -74,13 +74,9 @@ trait WhereParser { private def any_identifier: PackratParser[Identifier] = identifierWithTransformation | identifierWithAggregation | - identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithArithmeticExpression | identifierWithFunction | - date_diff_identifier | - last_day_identifier | - extract_identifier | identifier private def equality: PackratParser[GenericExpression] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala index f7956052..b2a7e2a8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala @@ -28,12 +28,12 @@ package object cond { trait CondParser { self: Parser with WhereParser => def is_null: PackratParser[ConditionalFunction[_]] = - "(?i)isnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { + "(?i)isnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { case _ ~ _ ~ i ~ _ => IsNull(i) } def is_notnull: PackratParser[ConditionalFunction[_]] = - "(?i)isnotnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithTemporalFunction | identifier) ~ end ^^ { + "(?i)isnotnull".r ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { case _ ~ _ ~ i ~ _ => IsNotNull(i) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala index 8e16a6cc..8164a8ae 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala @@ -10,12 +10,8 @@ package object convert { def castFunctionWithIdentifier: PackratParser[Identifier] = "(?i)cast".r ~ start ~ (identifierWithTransformation | - identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | - date_diff_identifier | - last_day_identifier | - extract_identifier | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index 710442c8..046ba0ac 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -10,7 +10,7 @@ import app.softnetwork.elastic.sql.function.{ import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.parser.time.TimeParser import app.softnetwork.elastic.sql.parser.{Delimiter, Parser} -import app.softnetwork.elastic.sql.time.TimeField +import app.softnetwork.elastic.sql.time.{IsoField, TimeField} package object time { @@ -38,13 +38,19 @@ package object time { if (p.isDefined) NowWithParens else Now } - def identifierWithSystemFunction: PackratParser[Identifier] = - (current_date | current_time | current_timestamp | now) ~ intervalFunction.? ^^ { - case f1 ~ f2 => - f2 match { - case Some(f) => Identifier(List(f, f1)) - case None => Identifier(f1) - } + def today: PackratParser[CurrentFunction] = Today.regex ~ parens.? ^^ { case _ ~ p => + if (p.isDefined) TodayWithParens else Today + } + + def systemFunctions: PackratParser[CurrentFunction] = + current_date | current_time | current_timestamp | now | today + + def systemFunctionWithIdentifier: PackratParser[Identifier] = + systemFunctions ~ intervalFunction.? ^^ { case f1 ~ f2 => + f2 match { + case Some(f) => Identifier(List(f, f1)) + case None => Identifier(f1) + } } } @@ -52,19 +58,19 @@ package object time { trait DateParser { self: Parser with TemporalParser => def date_add: PackratParser[DateFunction with FunctionWithIdentifier] = - DateAdd.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateAdd.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateAdd(i, t) } def date_sub: PackratParser[DateFunction with FunctionWithIdentifier] = - DateSub.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateSub.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateSub(i, t) } def date_parse: PackratParser[DateFunction with FunctionWithIdentifier] = - DateParse.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { + DateParse.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: StringValue => @@ -75,19 +81,26 @@ package object time { } def date_format: PackratParser[DateFunction with FunctionWithIdentifier] = - DateFormat.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { + DateFormat.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ f ~ _ => DateFormat(i, f.value) } - def date_functions: PackratParser[DateFunction] = date_add | date_sub | date_parse | date_format + def last_day: Parser[DateFunction with FunctionWithIdentifier] = + LastDayOfMonth.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { + case _ ~ _ ~ i ~ _ => LastDayOfMonth(i) + } + + def date_functions: PackratParser[DateFunction] = + date_add | date_sub | date_parse | date_format | last_day def dateFunctionWithIdentifier: PackratParser[Identifier] = - (date_parse | date_format | date_add | date_sub) ~ intervalFunction.? ^^ { case t ~ af => - af match { - case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) - case None => t.identifier.withFunctions(t +: t.identifier.functions) - } + (date_parse | date_format | date_add | date_sub | last_day) ~ intervalFunction.? ^^ { + case t ~ af => + af match { + case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) + case None => t.identifier.withFunctions(t +: t.identifier.functions) + } } } @@ -95,19 +108,19 @@ package object time { trait DateTimeParser { self: Parser with TemporalParser => def datetime_add: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - DateTimeAdd.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateTimeAdd.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateTimeAdd(i, t) } def datetime_sub: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - DateTimeSub.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ interval ~ end ^^ { + DateTimeSub.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ interval ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => DateTimeSub(i, t) } def datetime_parse: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - DateTimeParse.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { + DateTimeParse.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | literal | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ li ~ _ ~ f ~ _ => li match { case l: SQLLiteral => @@ -118,7 +131,7 @@ package object time { } def datetime_format: PackratParser[DateTimeFunction with FunctionWithIdentifier] = - DateTimeFormat.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ literal ~ end ^^ { + DateTimeFormat.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ literal ~ end ^^ { case _ ~ _ ~ i ~ _ ~ f ~ _ => DateTimeFormat(i, f.value) } @@ -140,26 +153,8 @@ package object time { trait TemporalParser extends SystemParser with TimeParser with DateParser with DateTimeParser { self: Parser => - def identifierWithTemporalFunction: PackratParser[Identifier] = - rep1sep( - date_trunc | extractors | date_functions | datetime_functions, - start - ) ~ start.? ~ (identifierWithSystemFunction | identifier).? ~ rep( - end - ) ^^ { case f ~ _ ~ i ~ _ => - i match { - case Some(id) => id.withFunctions(id.functions ++ f) - case None => - f.lastOption match { - case Some(fi: FunctionWithIdentifier) => - fi.identifier.withFunctions(f ++ fi.identifier.functions) - case _ => Identifier(f) - } - } - } - def date_diff: PackratParser[BinaryFunction[_, _, _]] = - DateDiff.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { + DateDiff.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ time_unit ~ end ^^ { case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) } @@ -168,13 +163,13 @@ package object time { } def date_trunc: PackratParser[FunctionWithIdentifier] = - DateTrunc.regex ~ start ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifier) ~ separator ~ time_unit ~ end ^^ { + DateTrunc.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ time_unit ~ end ^^ { case _ ~ _ ~ i ~ _ ~ u ~ _ => DateTrunc(i, u) } def extract_identifier: PackratParser[Identifier] = - Extract.regex ~ start ~ time_field ~ "(?i)from".r ~ (identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | last_day_identifier | identifier) ~ end ^^ { + Extract.regex ~ start ~ time_field ~ "(?i)from".r ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => i.withFunctions(Extract(u) +: i.functions) } @@ -208,6 +203,12 @@ package object time { def offset_seconds_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = OFFSET_SECONDS.regex ^^ (_ => OffsetSeconds) + def quarter_of_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + IsoField.QUARTER_OF_YEAR.regex ^^ (_ => QuarterOfYear) + + def week_of_week_based_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + IsoField.WEEK_OF_WEEK_BASED_YEAR.regex ^^ (_ => WeekOfWeekBasedYear) + def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = year_tr | month_of_year_tr | @@ -221,12 +222,9 @@ package object time { micro_of_second_tr | nano_of_second_tr | epoch_day_tr | - offset_seconds_tr - - def last_day_identifier: Parser[Identifier] = - LastDayOfMonth.regex ~ start ~ (castFunctionWithIdentifier | identifierWithTemporalFunction | identifierWithSystemFunction | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { - case _ ~ _ ~ i ~ _ => i.withFunctions(LastDayOfMonth(i) +: i.functions) - } + offset_seconds_tr | + quarter_of_year_tr | + week_of_week_based_year_tr } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala index 607e80a6..f04abfa3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala @@ -4,7 +4,7 @@ import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.function.TransformFunction import app.softnetwork.elastic.sql.function.time.{SQLAddInterval, SQLSubtractInterval} -import app.softnetwork.elastic.sql.time.{Interval, TimeField, TimeInterval, TimeUnit} +import app.softnetwork.elastic.sql.time.{Interval, IsoField, TimeField, TimeInterval, TimeUnit} package object time { @@ -35,8 +35,30 @@ package object time { def offset_seconds: PackratParser[TimeField] = OFFSET_SECONDS.regex ^^ (_ => OFFSET_SECONDS) + import IsoField._ + + def quarter_of_year: PackratParser[TimeField] = + QUARTER_OF_YEAR.regex ^^ (_ => QUARTER_OF_YEAR) + + def week_of_week_based_year: PackratParser[TimeField] = + WEEK_OF_WEEK_BASED_YEAR.regex ^^ (_ => WEEK_OF_WEEK_BASED_YEAR) + def time_field: PackratParser[TimeField] = - year | month_of_year | day_of_month | day_of_week | day_of_year | hour_of_day | minute_of_hour | second_of_minute | nano_of_second | micro_of_second | milli_of_second | epoch_day | offset_seconds + year | + month_of_year | + day_of_month | + day_of_week | + day_of_year | + hour_of_day | + minute_of_hour | + second_of_minute | + nano_of_second | + micro_of_second | + milli_of_second | + epoch_day | + offset_seconds | + quarter_of_year | + week_of_week_based_year import TimeUnit._ @@ -71,7 +93,9 @@ package object time { add_interval | substract_interval def identifierWithIntervalFunction: PackratParser[Identifier] = - (identifierWithFunction | identifier) ~ intervalFunction ^^ { case i ~ f => + (identifierWithTransformation | + identifierWithFunction | + identifier) ~ intervalFunction ^^ { case i ~ f => i.withFunctions(f +: i.functions) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index 5b73d02c..24c26015 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -7,55 +7,82 @@ import scala.util.matching.Regex package object time { sealed trait TimeField extends PainlessScript with TokenRegex { - override def painless: String = s"ChronoField.$sql" + override def painless: String = s"ChronoField.$timeField" override def nullable: Boolean = false + + def timeField: String + + override lazy val words: List[String] = + List(timeField, timeField.replaceAll("_", ""), sql).distinct } object TimeField { - case object YEAR extends Expr("YEAR") with TimeField - case object MONTH_OF_YEAR extends Expr("MONTH_OF_YEAR") with TimeField { - override val words: List[String] = List(sql, "MONTHOFYEAR", "MONTH") + case object YEAR extends Expr("YEAR") with TimeField { + override val timeField: String = "YEAR" } - case object DAY_OF_MONTH extends Expr("DAY_OF_MONTH") with TimeField { - override val words: List[String] = List(sql, "DAYOFMONTH", "DAY") + case object MONTH_OF_YEAR extends Expr("MONTH") with TimeField { + override val timeField: String = "MONTH_OF_YEAR" } - case object DAY_OF_WEEK extends Expr("DAY_OF_WEEK") with TimeField { - override val words: List[String] = List(sql, "DAYOFWEEK", "WEEKDAY") + case object DAY_OF_MONTH extends Expr("DAY") with TimeField { + override val timeField: String = "DAY_OF_MONTH" } - case object DAY_OF_YEAR extends Expr("DAY_OF_YEAR") with TimeField { - override val words: List[String] = List(sql, "DAYOFYEAR") + case object DAY_OF_WEEK extends Expr("WEEKDAY") with TimeField { + override val timeField: String = "DAY_OF_WEEK" } - case object HOUR_OF_DAY extends Expr("HOUR_OF_DAY") with TimeField { - override val words: List[String] = List(sql, "HOUROFDAY", "HOUR") + case object DAY_OF_YEAR extends Expr("YEARDAY") with TimeField { + override val timeField: String = "DAY_OF_YEAR" } - case object MINUTE_OF_HOUR extends Expr("MINUTE_OF_HOUR") with TimeField { - override val words: List[String] = List(sql, "MINUTEOFHOUR", "MINUTE") + case object HOUR_OF_DAY extends Expr("HOUR") with TimeField { + override val timeField: String = "HOUR_OF_DAY" } - case object SECOND_OF_MINUTE extends Expr("SECOND_OF_MINUTE") with TimeField { - override val words: List[String] = List(sql, "SECONDOFMINUTE", "SECOND") + case object MINUTE_OF_HOUR extends Expr("MINUTE") with TimeField { + override val timeField: String = "MINUTE_OF_HOUR" } - case object NANO_OF_SECOND extends Expr("NANO_OF_SECOND") with TimeField { - override val words: List[String] = List(sql, "NANOFSECOND", "NANOSECOND") + case object SECOND_OF_MINUTE extends Expr("SECOND") with TimeField { + override val timeField: String = "SECOND_OF_MINUTE" } - case object MICRO_OF_SECOND extends Expr("MICRO_OF_SECOND") with TimeField { - override val words: List[String] = List(sql, "MICROOFSECOND", "MICROSECOND") + case object NANO_OF_SECOND extends Expr("NANOSECOND") with TimeField { + override val timeField: String = "NANO_OF_SECOND" } - case object MILLI_OF_SECOND extends Expr("MILLI_OF_SECOND") with TimeField { - override val words: List[String] = List(sql, "MILLIOFSECOND", "MILLISECOND") + case object MICRO_OF_SECOND extends Expr("MICROSECOND") with TimeField { + override val timeField: String = "MICRO_OF_SECOND" } - case object EPOCH_DAY extends Expr("EPOCH_DAY") with TimeField { - override val words: List[String] = List(sql, "EPOCHDAY") + case object MILLI_OF_SECOND extends Expr("MILLISECOND") with TimeField { + override val timeField: String = "MILLI_OF_SECOND" } - case object OFFSET_SECONDS extends Expr("OFFSET_SECONDS") with TimeField { - override val words: List[String] = List(sql, "OFFSETSECONDS") + case object EPOCH_DAY extends Expr("EPOCHDAY") with TimeField { + override val timeField: String = "EPOCH_DAY" + } + case object OFFSET_SECONDS extends Expr("OFFSET") with TimeField { + override val timeField: String = "OFFSET_SECONDS" } } + sealed trait IsoField extends TimeField { + def isoField: String + def timeField: String = isoField + override def painless: String = s"java.time.temporal.IsoFields.$isoField" + } + + object IsoField { + + case object QUARTER_OF_YEAR extends Expr("QUARTER") with IsoField { + override val isoField: String = "QUARTER_OF_YEAR" + } + + case object WEEK_OF_WEEK_BASED_YEAR extends Expr("WEEK") with IsoField { + override val isoField: String = "WEEK_OF_WEEK_BASED_YEAR" + } + + } + sealed trait TimeUnit extends PainlessScript with MathScript { lazy val regex: Regex = s"\\b(?i)$sql(s)?\\b".r - override def painless: String = s"ChronoUnit.${sql.toUpperCase()}S" + def timeUnit: String = sql.toUpperCase() + "S" + + override def painless: String = s"ChronoUnit.$timeUnit" override def nullable: Boolean = false } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 98eb7b59..cf9abe77 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -83,7 +83,7 @@ object Queries { |HAVING Country <> 'USA' AND City <> 'Berlin' AND COUNT(CustomerID) > 1 |ORDER BY COUNT(CustomerID) DESC, Country ASC""".stripMargin.replaceAll("\n", " ") val dateTimeWithIntervalFields: String = - "SELECT CURRENT_TIMESTAMP() - INTERVAL 3 DAY AS ct, CURRENT_DATE AS cd, CURRENT_TIME AS t, NOW AS n FROM dual" + "SELECT CURRENT_TIMESTAMP() - INTERVAL 3 DAY AS ct, CURRENT_DATE AS cd, CURRENT_TIME AS t, NOW AS n, TODAY as td FROM dual" val fieldsWithInterval: String = "SELECT createdAt - INTERVAL 35 MINUTE AS ct, identifier FROM Table" val filterWithDateTimeAndInterval: String = @@ -100,7 +100,7 @@ object Queries { |ORDER BY Country ASC""".stripMargin .replaceAll("\n", " ") val dateParse = - "SELECT identifier, COUNT(identifier2) AS ct, MAX(date_parse(createdAt, 'yyyy-MM-dd')) AS lastSeen FROM Table WHERE identifier2 is NOT null GROUP BY identifier ORDER BY COUNT(identifier2) DESC" + "SELECT identifier, COUNT(identifier2) AS ct, MAX(DATE_PARSE(createdAt, 'yyyy-MM-dd')) AS lastSeen FROM Table WHERE identifier2 is NOT null GROUP BY identifier ORDER BY COUNT(identifier2) DESC" val dateTimeParse: String = """SELECT identifier, COUNT(identifier2) AS ct, |MAX( @@ -124,9 +124,9 @@ object Queries { "SELECT MAX(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, DAY)) AS max_diff FROM Table GROUP BY identifier" val dateFormat = - "SELECT identifier, date_format(date_trunc(lastUpdated, month), 'yyyy-MM-dd') AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, date_format(date_trunc(lastUpdated, MONTH), 'yyyy-MM-dd') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeFormat = - "SELECT identifier, datetime_format(date_trunc(lastUpdated, month), 'yyyy-MM-ddThh:mm:ssZ') AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, datetime_format(date_trunc(lastUpdated, MONTH), 'yyyy-MM-ddThh:mm:ssZ') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateAdd = "SELECT identifier, date_add(lastUpdated, INTERVAL 10 DAY) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateSub = @@ -143,9 +143,9 @@ object Queries { val coalesce: String = "SELECT COALESCE(createdAt - INTERVAL 35 MINUTE, CURRENT_DATE) AS c, identifier FROM Table" val nullif: String = - "SELECT COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 DAY), CURRENT_DATE) AS c, identifier FROM Table" + "SELECT COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 DAY), CURRENT_DATE) AS c, identifier FROM Table" val cast: String = - "SELECT CAST(COALESCE(nullif(createdAt, date_parse('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 hour) bigint) AS c, identifier FROM Table" + "SELECT CAST(COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 HOUR) BIGINT) AS c, identifier FROM Table" val allCasts = "SELECT CAST(identifier AS int) AS c1, CAST(identifier AS bigint) AS c2, CAST(identifier AS double) AS c3, CAST(identifier AS real) AS c4, CAST(identifier AS boolean) AS c5, CAST(identifier AS char) AS c6, CAST(identifier AS varchar) AS c7, CAST(createdAt AS date) AS c8, CAST(createdAt AS time) AS c9, CAST(createdAt AS datetime) AS c10, CAST(createdAt AS timestamp) AS c11, CAST(identifier AS smallint) AS c12, CAST(identifier AS tinyint) AS c13 FROM Table" val caseWhen: String = @@ -154,7 +154,7 @@ object Queries { "SELECT CASE CURRENT_DATE - INTERVAL 7 DAY WHEN CAST(lastUpdated AS date) - INTERVAL 3 DAY THEN lastUpdated WHEN lastSeen THEN lastSeen + INTERVAL 2 DAY ELSE createdAt END AS c, identifier FROM Table" val extract: String = - "SELECT EXTRACT(day_of_month FROM createdAt) AS dom, EXTRACT(day_of_week FROM createdAt) AS dow, EXTRACT(day_of_year FROM createdAt) AS doy, EXTRACT(month_of_year FROM createdAt) AS m, EXTRACT(year FROM createdAt) AS y, EXTRACT(hour_of_day FROM createdAt) AS h, EXTRACT(minute_of_hour FROM createdAt) AS minutes, EXTRACT(second_of_minute FROM createdAt) AS s FROM Table" + "SELECT EXTRACT(DAY FROM createdAt) AS dom, EXTRACT(WEEKDAY FROM createdAt) AS dow, EXTRACT(YEARDAY FROM createdAt) AS doy, EXTRACT(MONTH FROM createdAt) AS m, EXTRACT(YEAR FROM createdAt) AS y, EXTRACT(HOUR FROM createdAt) AS h, EXTRACT(MINUTE FROM createdAt) AS minutes, EXTRACT(SECOND FROM createdAt) AS s, EXTRACT(NANOSECOND FROM createdAt) AS nano, EXTRACT(MICROSECOND FROM createdAt) AS micro, EXTRACT(MILLISECOND FROM createdAt) AS milli, EXTRACT(EPOCHDAY FROM createdAt) AS epoch, EXTRACT(OFFSET FROM createdAt) AS off, EXTRACT(WEEK FROM createdAt) AS w, EXTRACT(QUARTER FROM createdAt) AS q FROM Table" val arithmetic: String = "SELECT identifier, identifier + 1 AS add, identifier - 1 AS sub, identifier * 2 AS mul, identifier / 2 AS div, identifier % 2 AS mod, (identifier * identifier2) - 10 FROM Table WHERE identifier * (EXTRACT(year FROM CURRENT_DATE) - 10) > 10000" @@ -169,7 +169,10 @@ object Queries { "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary FROM emp" val lastDay: String = - "SELECT LAST_DAY(CAST(createdAt AS DATE)) AS ld, identifier FROM Table WHERE EXTRACT(DAY_OF_MONTH FROM LAST_DAY(CURRENT_TIMESTAMP)) > 28" + "SELECT LAST_DAY(CAST(createdAt AS DATE)) AS ld, identifier FROM Table WHERE EXTRACT(DAY FROM LAST_DAY(CURRENT_TIMESTAMP)) > 28" + + val extractors: String = + "SELECT WEEKDAY(createdAt) AS dow, YEARDAY(createdAt) AS doy, DAY(createdAt) AS dom, WEEKDAY(createdAt) AS dow2, YEARDAY(createdAt) AS doy2, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" } /** Created by smanciot on 15/02/17. @@ -178,7 +181,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { import Queries._ - "SQLParser" should "parse numerical eq" in { + "SQLParser" should "parse numerical EQ" in { val result = Parser(numericalEq) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -186,7 +189,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalEq) shouldBe true } - it should "parse numerical ne" in { + it should "parse numerical NE" in { val result = Parser(numericalNe) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -194,7 +197,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalNe) shouldBe true } - it should "parse numerical lt" in { + it should "parse numerical LT" in { val result = Parser(numericalLt) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -202,7 +205,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalLt) shouldBe true } - it should "parse numerical le" in { + it should "parse numerical LE" in { val result = Parser(numericalLe) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -210,7 +213,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalLe) shouldBe true } - it should "parse numerical gt" in { + it should "parse numerical GT" in { val result = Parser(numericalGt) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -218,7 +221,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalGt) shouldBe true } - it should "parse numerical ge" in { + it should "parse numerical GE" in { val result = Parser(numericalGe) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -226,7 +229,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalGe) shouldBe true } - it should "parse literal eq" in { + it should "parse literal EQ" in { val result = Parser(literalEq) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -258,7 +261,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalNotLike) shouldBe true } - it should "parse literal ne" in { + it should "parse literal NE" in { val result = Parser(literalNe) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -266,7 +269,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalNe) shouldBe true } - it should "parse literal lt" in { + it should "parse literal LT" in { val result = Parser(literalLt) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -274,7 +277,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalLt) shouldBe true } - it should "parse literal le" in { + it should "parse literal LE" in { val result = Parser(literalLe) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -282,7 +285,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalLe) shouldBe true } - it should "parse literal gt" in { + it should "parse literal GT" in { val result = Parser(literalGt) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -290,12 +293,12 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(literalGt) shouldBe true } - it should "parse literal ge" in { + it should "parse literal GE" in { val result = Parser(literalGe) result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") equalsIgnoreCase literalGe } - it should "parse boolean eq" in { + it should "parse boolean EQ" in { val result = Parser(boolEq) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -303,7 +306,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(boolEq) shouldBe true } - it should "parse boolean ne" in { + it should "parse boolean NE" in { val result = Parser(boolNe) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -407,7 +410,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(parentCriteria) shouldBe true } - it should "parse in literal expression" in { + it should "parse IN literal expression" in { val result = Parser(inLiteralExpression) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -415,7 +418,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(inLiteralExpression) shouldBe true } - it should "parse in numerical expression with Int values" in { + it should "parse IN numerical expression with Int values" in { val result = Parser(inNumericalExpressionWithIntValues) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -423,7 +426,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(inNumericalExpressionWithIntValues) shouldBe true } - it should "parse in numerical expression with Double values" in { + it should "parse IN numerical expression with Double values" in { val result = Parser(inNumericalExpressionWithDoubleValues) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -431,7 +434,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(inNumericalExpressionWithDoubleValues) shouldBe true } - it should "parse NOT in literal expression" in { + it should "parse NOT IN literal expression" in { val result = Parser(notInLiteralExpression) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -439,7 +442,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(notInLiteralExpression) shouldBe true } - it should "parse NOT in numerical expression with Int values" in { + it should "parse NOT IN numerical expression with Int values" in { val result = Parser(notInNumericalExpressionWithIntValues) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -447,7 +450,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(notInNumericalExpressionWithIntValues) shouldBe true } - it should "parse NOT in numerical expression with Double values" in { + it should "parse NOT IN numerical expression with Double values" in { val result = Parser(notInNumericalExpressionWithDoubleValues) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -471,7 +474,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(COUNT) shouldBe true } - it should "parse distinct COUNT" in { + it should "parse DISTINCT COUNT" in { val result = Parser(countDistinct) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -487,7 +490,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(countNested) shouldBe true } - it should "parse is null" in { + it should "parse IS NULL" in { val result = Parser(isNull) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -495,7 +498,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(isNull) shouldBe true } - it should "parse is NOT null" in { + it should "parse IS NOT NULL" in { val result = Parser(isNotNull) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -511,7 +514,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(geoDistanceCriteria) shouldBe true } - it should "parse except fields" in { + it should "parse EXCEPT fields" in { val result = Parser(except) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -519,7 +522,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(except) shouldBe true } - it should "parse match criteria" in { + it should "parse MATCH criteria" in { val result = Parser(matchCriteria) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -543,7 +546,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(orderBy) shouldBe true } - it should "parse limit" in { + it should "parse LIMIT" in { val result = Parser(limit) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -551,7 +554,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(limit) shouldBe true } - it should "parse GROUP BY with ORDER BY AND limit" in { + it should "parse GROUP BY with ORDER BY and LIMIT" in { val result = Parser(groupByWithOrderByAndLimit) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -583,7 +586,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(fieldsWithInterval) shouldBe true } - it should "parse filter with date time AND INTERVAL" in { + it should "parse filter with date time and INTERVAL" in { val result = Parser(filterWithDateTimeAndInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -591,7 +594,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithDateTimeAndInterval) shouldBe true } - it should "parse filter with date AND INTERVAL" in { + it should "parse filter with date and interval" in { val result = Parser(filterWithDateAndInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -599,7 +602,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithDateAndInterval) shouldBe true } - it should "parse filter with time AND INTERVAL" in { + it should "parse filter with time and interval" in { val result = Parser(filterWithTimeAndInterval) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -607,7 +610,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(filterWithTimeAndInterval) shouldBe true } - it should "parse GROUP BY with HAVING AND date time functions" in { + it should "parse GROUP BY with HAVING and date time functions" in { val result = Parser(groupByWithHavingAndDateTimeFunctions) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -694,7 +697,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(dateTimeSub) shouldBe true } - it should "parse isnull function" in { + it should "parse ISNULL function" in { val result = Parser(isnull) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -702,7 +705,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(isnull) shouldBe true } - it should "parse isnotnull function" in { + it should "parse ISNOTNULL function" in { val result = Parser(isnotnull) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -710,7 +713,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(isnotnull) shouldBe true } - it should "parse isnull criteria" in { + it should "parse ISNULL criteria" in { val result = Parser(isNullCriteria) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -718,7 +721,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(isNullCriteria) shouldBe true } - it should "parse isnotnull criteria" in { + it should "parse ISNOTNULL criteria" in { val result = Parser(isNotNullCriteria) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -726,7 +729,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(isNotNullCriteria) shouldBe true } - it should "parse coalesce function" in { + it should "parse COALESCE function" in { val result = Parser(coalesce) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -734,7 +737,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(coalesce) shouldBe true } - it should "parse nullif function" in { + it should "parse NULLIF function" in { val result = Parser(nullif) result.toOption .flatMap(_.left.toOption.map(_.sql)) @@ -742,12 +745,11 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(nullif) shouldBe true } - it should "parse cast function" in { + it should "parse CAST function" in { val result = Parser(cast) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") - .equalsIgnoreCase(cast) shouldBe true + .getOrElse("") shouldBe cast } it should "parse all casts function" in { @@ -774,12 +776,11 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(caseWhenExpr) shouldBe true } - it should "parse extract function" in { + it should "parse EXTRACT function" in { val result = Parser(extract) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") - .equalsIgnoreCase(extract) shouldBe true + .getOrElse("") shouldBe extract } it should "parse arithmetic expressions" in { @@ -819,4 +820,12 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .flatMap(_.left.toOption.map(_.sql)) .getOrElse("") shouldBe lastDay } + + it should "parse all date extractors" in { + val result = Parser(extractors) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") shouldBe extractors + } + } From 071bcc7a27c0956f422ede11279ff3fc8375f771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Sep 2025 18:21:52 +0200 Subject: [PATCH 21/48] add specifications for today, quarter and week functions --- .../elastic/sql/SQLQuerySpec.scala | 135 ++++++++++++++++++ .../elastic/sql/SQLQuerySpec.scala | 135 ++++++++++++++++++ .../elastic/sql/SQLParserSpec.scala | 2 +- 3 files changed, 271 insertions(+), 1 deletion(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index edf5d89f..25274038 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2642,4 +2642,139 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") } + it should "handle all extractors" in { + val select: ElasticSearchRequest = + SQLQuery(extractors) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "script_fields": { + | "y": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.YEAR) : null)" + | } + | }, + | "m": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MONTH_OF_YEAR) : null)" + | } + | }, + | "wd": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_WEEK) : null)" + | } + | }, + | "yd": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_YEAR) : null)" + | } + | }, + | "d": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_MONTH) : null)" + | } + | }, + | "h": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.HOUR_OF_DAY) : null)" + | } + | }, + | "minutes": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MINUTE_OF_HOUR) : null)" + | } + | }, + | "s": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" + | } + | }, + | "nano": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.NANO_OF_SECOND) : null)" + | } + | }, + | "micro": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MICRO_OF_SECOND) : null)" + | } + | }, + | "milli": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MILLI_OF_SECOND) : null)" + | } + | }, + | "epoch": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.EPOCH_DAY) : null)" + | } + | }, + | "off": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.OFFSET_SECONDS) : null)" + | } + | }, + | "w": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR) : null)" + | } + | }, + | "q": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR) : null)" + | } + | } + | }, + | "_source": true + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("-", " - ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + } } diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 87d5eb36..45f65149 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2631,4 +2631,139 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") } + it should "handle all extractors" in { + val select: ElasticSearchRequest = + SQLQuery(extractors) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "script_fields": { + | "y": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.YEAR) : null)" + | } + | }, + | "m": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MONTH_OF_YEAR) : null)" + | } + | }, + | "wd": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_WEEK) : null)" + | } + | }, + | "yd": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_YEAR) : null)" + | } + | }, + | "d": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.DAY_OF_MONTH) : null)" + | } + | }, + | "h": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.HOUR_OF_DAY) : null)" + | } + | }, + | "minutes": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MINUTE_OF_HOUR) : null)" + | } + | }, + | "s": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.SECOND_OF_MINUTE) : null)" + | } + | }, + | "nano": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.NANO_OF_SECOND) : null)" + | } + | }, + | "micro": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MICRO_OF_SECOND) : null)" + | } + | }, + | "milli": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.MILLI_OF_SECOND) : null)" + | } + | }, + | "epoch": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.EPOCH_DAY) : null)" + | } + | }, + | "off": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(ChronoField.OFFSET_SECONDS) : null)" + | } + | }, + | "w": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR) : null)" + | } + | }, + | "q": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? e0.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR) : null)" + | } + | } + | }, + | "_source": true + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("-", " - ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index cf9abe77..47873027 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -172,7 +172,7 @@ object Queries { "SELECT LAST_DAY(CAST(createdAt AS DATE)) AS ld, identifier FROM Table WHERE EXTRACT(DAY FROM LAST_DAY(CURRENT_TIMESTAMP)) > 28" val extractors: String = - "SELECT WEEKDAY(createdAt) AS dow, YEARDAY(createdAt) AS doy, DAY(createdAt) AS dom, WEEKDAY(createdAt) AS dow2, YEARDAY(createdAt) AS doy2, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" + "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" } /** Created by smanciot on 15/02/17. From 3850ba199c1324a8bc75f534f21753b240c848e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Sep 2025 18:25:00 +0200 Subject: [PATCH 22/48] fix criteria specifications --- .../scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index bb3865f3..864a0011 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -148,7 +148,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"regexp" : { | "identifier" : { - | "value" : ".*un.*" + | "value" : ".*u.n.*" | } | } | } @@ -681,8 +681,8 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range" : { | "ciblage.Archivage_CreationDate" : { - | "gte" : "now-3M/M", - | "lte" : "now" + | "gte" : "NOW-3M/M", + | "lte" : "NOW" | } | } | }, From 78ec29638acf87fe7622d01950a9a1518fe925d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Sep 2025 18:34:29 +0200 Subject: [PATCH 23/48] fix criteria specifications for es 7+ --- .../scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index a25955d0..4f3aa58f 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -147,7 +147,7 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { |"query":{ | "bool":{"filter":[{"regexp" : { | "identifier" : { - | "value" : ".*un.*" + | "value" : ".*u.n.*" | } | } | } @@ -680,8 +680,8 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { | { | "range" : { | "ciblage.Archivage_CreationDate" : { - | "gte" : "now-3M/M", - | "lte" : "now" + | "gte" : "NOW-3M/M", + | "lte" : "NOW" | } | } | }, From fceb0859301ad5fe3c3e7acaec21a702a08dc251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Sep 2025 20:01:24 +0200 Subject: [PATCH 24/48] add support for ARRAY_AGG aggregation --- .../sql/bridge/ElasticAggregation.scala | 17 +++-- .../elastic/sql/SQLQuerySpec.scala | 22 ++++++ .../sql/bridge/ElasticAggregation.scala | 17 +++-- .../elastic/sql/SQLQuerySpec.scala | 22 ++++++ .../sql/function/aggregate/package.scala | 71 ++++++++++++------- .../elastic/sql/parser/Parser.scala | 2 +- .../parser/function/aggregate/package.scala | 31 ++++---- .../elastic/sql/SQLParserSpec.scala | 2 +- 8 files changed, 133 insertions(+), 51 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index efe255c3..f9a46d9c 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -124,6 +124,13 @@ object ElasticAggregation { case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) case th: TopHitsAggregation => + val limit = { + th match { + case _: LastValue => 1 +// case _: FirstValue => 1 + case _ => th.limit.map(_.limit).getOrElse(1) + } + } val topHits = topHitsAgg(aggName) .fetchSource( @@ -139,17 +146,17 @@ object ElasticAggregation { .map(f => f.sourceField -> Script(f.painless).lang("painless")) .toMap ) - .size(1) sortBy th.orderBy.sorts.map(sort => + .size(limit) sortBy th.orderBy.sorts.map(sort => sort.order match { case Some(Desc) => th.topHits match { - case FIRST_VALUE => FieldSort(sort.field).desc() - case LAST_VALUE => FieldSort(sort.field).asc() + case LAST_VALUE => FieldSort(sort.field).asc() + case _ => FieldSort(sort.field).desc() } case _ => th.topHits match { - case FIRST_VALUE => FieldSort(sort.field).asc() - case LAST_VALUE => FieldSort(sort.field).desc() + case LAST_VALUE => FieldSort(sort.field).desc() + case _ => FieldSort(sort.field).asc() } } ) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 25274038..d53bb1bb 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2539,6 +2539,28 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | ] | } | } + | }, + | "employees": { + | "top_hits": { + | "size": 1000, + | "sort": [ + | { + | "hire_date": { + | "order": "asc" + | } + | }, + | { + | "salary": { + | "order": "desc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "name" + | ] + | } + | } | } | } | } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index c86b7d9f..6b799e46 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -123,6 +123,13 @@ object ElasticAggregation { case AVG => aggWithFieldOrScript(avgAgg, (name, s) => avgAgg(name, sourceField).script(s)) case SUM => aggWithFieldOrScript(sumAgg, (name, s) => sumAgg(name, sourceField).script(s)) case th: TopHitsAggregation => + val limit = { + th match { + case _: LastValue => 1 + // case _: FirstValue => 1 + case _ => th.limit.map(_.limit).getOrElse(1) + } + } val topHits = topHitsAgg(aggName) .fetchSource( @@ -136,17 +143,17 @@ object ElasticAggregation { f.sourceField -> Script(f.painless).lang("painless") ).toMap ) - .size(1) sortBy th.orderBy.sorts.map(sort => + .size(limit) sortBy th.orderBy.sorts.map(sort => sort.order match { case Some(Desc) => th.topHits match { - case FIRST_VALUE => FieldSort(sort.field).desc() - case LAST_VALUE => FieldSort(sort.field).asc() + case LAST_VALUE => FieldSort(sort.field).asc() + case _ => FieldSort(sort.field).desc() } case _ => th.topHits match { - case FIRST_VALUE => FieldSort(sort.field).asc() - case LAST_VALUE => FieldSort(sort.field).desc() + case LAST_VALUE => FieldSort(sort.field).desc() + case _ => FieldSort(sort.field).asc() } } ) diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 45f65149..2384953f 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2528,6 +2528,28 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | ] | } | } + | }, + | "employees": { + | "top_hits": { + | "size": 1000, + | "sort": [ + | { + | "hire_date": { + | "order": "asc" + | } + | }, + | { + | "salary": { + | "order": "desc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "name" + | ] + | } + | } | } | } | } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 2b0b5680..3b7669ee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.query.{Bucket, Field, OrderBy, SQLSearchRequest} -import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} +import app.softnetwork.elastic.sql.query.{Bucket, Field, Limit, OrderBy, SQLSearchRequest} +import app.softnetwork.elastic.sql.{asString, Expr, Identifier, TokenRegex, Updateable} package object aggregate { @@ -27,6 +27,10 @@ package object aggregate { override val words: List[String] = List(sql, "LAST") } + case object ARRAY_AGG extends Expr("ARRAY_AGG") with TopHits { + override val words: List[String] = List(sql, "ARRAY") + } + case object OVER extends Expr("OVER") with TokenRegex case object PARTITION_BY extends Expr("PARTITION BY") with TokenRegex @@ -36,8 +40,10 @@ package object aggregate { with FunctionWithIdentifier with Updateable { def partitionBy: Seq[Identifier] + def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation def orderBy: OrderBy def topHits: TopHits + def limit: Option[Limit] lazy val buckets: Seq[Bucket] = partitionBy.map(Bucket) @@ -49,26 +55,18 @@ package object aggregate { val partitionByStr = if (partitionBy.nonEmpty) s"$PARTITION_BY ${partitionBy.mkString(", ")}" else "" - s"$topHits($identifier) $OVER ($partitionByStr$orderBy)" + s"$topHits($identifier) $OVER ($partitionByStr$orderBy${asString(limit)})" } override def toSQL(base: String): String = sql def fields: Seq[Field] - def update(request: SQLSearchRequest): TopHitsAggregation - } + def withFields(fields: Seq[Field]): TopHitsAggregation - case class FirstValue( - identifier: Identifier, - partitionBy: Seq[Identifier] = Seq.empty, - orderBy: OrderBy, - fields: Seq[Field] = Seq.empty - ) extends TopHitsAggregation { - override def topHits: TopHits = FIRST_VALUE - override def update(request: SQLSearchRequest): FirstValue = { - val updated = this.copy(partitionBy = partitionBy.map(_.update(request))) - updated.copy( + def update(request: SQLSearchRequest): TopHitsAggregation = { + val updated = this.withPartitionBy(partitionBy = partitionBy.map(_.update(request))) + updated.withFields( fields = request.select.fields .filterNot(field => field.aggregation || request.bucketNames.keys.toSeq @@ -79,24 +77,43 @@ package object aggregate { } } + case class FirstValue( + identifier: Identifier, + partitionBy: Seq[Identifier] = Seq.empty, + orderBy: OrderBy, + fields: Seq[Field] = Seq.empty, + limit: Option[Limit] = None + ) extends TopHitsAggregation { + override def topHits: TopHits = FIRST_VALUE + override def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation = + this.copy(partitionBy = partitionBy) + override def withFields(fields: Seq[Field]): TopHitsAggregation = this.copy(fields = fields) + } + case class LastValue( identifier: Identifier, partitionBy: Seq[Identifier] = Seq.empty, orderBy: OrderBy, - fields: Seq[Field] = Seq.empty + fields: Seq[Field] = Seq.empty, + limit: Option[Limit] = None ) extends TopHitsAggregation { override def topHits: TopHits = LAST_VALUE - override def update(request: SQLSearchRequest): LastValue = { - val updated = this.copy(partitionBy = partitionBy.map(_.update(request))) - updated.copy( - fields = request.select.fields - .filterNot(field => - field.aggregation || request.bucketNames.keys.toSeq - .contains(field.identifier.identifierName) - ) - .filterNot(f => request.excludes.contains(f.sourceField)) - ) - } + override def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation = + this.copy(partitionBy = partitionBy) + override def withFields(fields: Seq[Field]): TopHitsAggregation = this.copy(fields = fields) + } + + case class ArrayAgg( + identifier: Identifier, + partitionBy: Seq[Identifier] = Seq.empty, + orderBy: OrderBy, + fields: Seq[Field] = Seq.empty, + limit: Option[Limit] = None + ) extends TopHitsAggregation { + override def topHits: TopHits = ARRAY_AGG + override def withPartitionBy(partitionBy: Seq[Identifier]): TopHitsAggregation = + this.copy(partitionBy = partitionBy) + override def withFields(fields: Seq[Field]): TopHitsAggregation = this } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index ce00caae..9ba1c688 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -82,7 +82,7 @@ trait Parser with MathParser with StringParser with TemporalParser - with TypeParser { _: WhereParser with OrderByParser => + with TypeParser { _: WhereParser with OrderByParser with LimitParser => def start: PackratParser[Delimiter] = "(" ^^ (_ => StartPredicate) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index 8b9a9b6e..01cf69c9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -2,12 +2,12 @@ package app.softnetwork.elastic.sql.parser.function import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.function.aggregate._ -import app.softnetwork.elastic.sql.parser.{OrderByParser, Parser} -import app.softnetwork.elastic.sql.query.OrderBy +import app.softnetwork.elastic.sql.parser.{LimitParser, OrderByParser, Parser} +import app.softnetwork.elastic.sql.query.{Limit, OrderBy} package object aggregate { - trait AggregateParser { self: Parser with OrderByParser => + trait AggregateParser { self: Parser with OrderByParser with LimitParser => def count: PackratParser[AggregateFunction] = COUNT.regex ^^ (_ => COUNT) @@ -30,25 +30,32 @@ package object aggregate { def partition_by: PackratParser[Seq[Identifier]] = PARTITION_BY.regex ~> rep1sep(identifier, separator) - private[this] def top_hits: PackratParser[(Identifier, Seq[Identifier], OrderBy)] = - start ~ identifier ~ end ~ OVER.regex ~ start ~ partition_by.? ~ orderBy ~ end ^^ { - case _ ~ id ~ _ ~ _ ~ _ ~ pb ~ ob ~ _ => - (id, pb.getOrElse(Seq.empty), ob) + private[this] def top_hits + : PackratParser[(Identifier, Seq[Identifier], OrderBy, Option[Limit])] = + start ~ identifier ~ end ~ OVER.regex ~ start ~ partition_by.? ~ orderBy ~ limit.? ~ end ^^ { + case _ ~ id ~ _ ~ _ ~ _ ~ pb ~ ob ~ l ~ _ => + (id, pb.getOrElse(Seq.empty), ob, l) } def first_value: PackratParser[TopHitsAggregation] = FIRST_VALUE.regex ~ top_hits ^^ { case _ ~ top => - FirstValue(top._1, top._2, top._3) + FirstValue(top._1, top._2, top._3, limit = top._4) } def last_value: PackratParser[TopHitsAggregation] = LAST_VALUE.regex ~ top_hits ^^ { case _ ~ top => - LastValue(top._1, top._2, top._3) + LastValue(top._1, top._2, top._3, limit = top._4) } - def identifierWithTopHits: PackratParser[Identifier] = (first_value | last_value) ^^ { th => - th.identifier.withFunctions(th +: th.identifier.functions) - } + def array_agg: PackratParser[TopHitsAggregation] = + ARRAY_AGG.regex ~ top_hits ^^ { case _ ~ top => + ArrayAgg(top._1, top._2, top._3, limit = top._4) + } + + def identifierWithTopHits: PackratParser[Identifier] = + (first_value | last_value | array_agg) ^^ { th => + th.identifier.withFunctions(th +: th.identifier.functions) + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 47873027..a88215e2 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -166,7 +166,7 @@ object Queries { "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" val topHits: String = - "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary FROM emp" + "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary, ARRAY_AGG(name) OVER (PARTITION BY department ORDER BY hire_date ASC, salary DESC LIMIT 1000) AS employees FROM emp" val lastDay: String = "SELECT LAST_DAY(CAST(createdAt AS DATE)) AS ld, identifier FROM Table WHERE EXTRACT(DAY FROM LAST_DAY(CURRENT_TIMESTAMP)) > 28" From 07b893fbd85c72d54a6f6e31a019bbbfac2b3810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 26 Sep 2025 19:25:02 +0200 Subject: [PATCH 25/48] add support for CONVERT, TRY_CAST / SAFE_CAST functions and cast operator (::) - add more conversions within coerce utility method - add setter for out used by cast functions --- .../elastic/sql/SQLQuerySpec.scala | 32 ++++++++- .../elastic/sql/SQLQuerySpec.scala | 32 ++++++++- .../elastic/sql/function/cond/package.scala | 16 ++--- .../sql/function/convert/package.scala | 65 ++++++++++++++--- .../elastic/sql/function/package.scala | 17 +++-- .../elastic/sql/function/string/package.scala | 5 +- .../elastic/sql/function/time/package.scala | 61 ++++++++++------ .../operator/math/ArithmeticExpression.scala | 10 +-- .../app/softnetwork/elastic/sql/package.scala | 72 +++++++++++-------- .../elastic/sql/parser/Parser.scala | 14 ++-- .../sql/parser/function/convert/package.scala | 34 ++++++++- .../sql/parser/function/time/package.scala | 10 +-- .../sql/parser/operator/math/package.scala | 4 +- .../elastic/sql/parser/time/package.scala | 4 +- .../softnetwork/elastic/sql/query/Where.scala | 2 +- .../elastic/sql/type/SQLType.scala | 5 +- .../elastic/sql/type/SQLTypeUtils.scala | 42 +++++++++-- .../elastic/sql/SQLParserSpec.scala | 10 +-- 18 files changed, 322 insertions(+), 113 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index d53bb1bb..fc24d54e 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1847,7 +1847,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle cast function as script field" in { val select: ElasticSearchRequest = - SQLQuery(cast) + SQLQuery(conversion) val query = select.query println(query) query shouldBe @@ -1859,7 +1859,31 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS); }.toInstant().toEpochMilli()" + | "source": "try { def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS).toInstant().toEpochMilli(); } catch (Exception e) { return null; }" + | } + | }, + | "c2": { + | "script": { + | "lang": "painless", + | "source": "ZonedDateTime.now(ZoneId.of('Z')).toInstant().toEpochMilli()" + | } + | }, + | "c3": { + | "script": { + | "lang": "painless", + | "source": "ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()" + | } + | }, + | "c4": { + | "script": { + | "lang": "painless", + | "source": "Long.parseLong(\"125\").longValue()" + | } + | }, + | "c5": { + | "script": { + | "lang": "painless", + | "source": "LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern('yyyy-MM-dd'))" | } | } | }, @@ -1889,6 +1913,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll(",LocalDate", ", LocalDate") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") + .replaceAll("try\\{", "try {") + .replaceAll("\\}catch", "} catch ") + .replaceAll("Exceptione\\)", "Exception e) ") + .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") } it should "handle case function as script field" in { diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 2384953f..808962d9 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1836,7 +1836,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle cast function as script field" in { val select: ElasticSearchRequest = - SQLQuery(cast) + SQLQuery(conversion) val query = select.query println(query) query shouldBe @@ -1848,7 +1848,31 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "{ def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS); }.toInstant().toEpochMilli()" + | "source": "try { def v0 = ((def arg0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); (arg0 == null) ? null : arg0 == DateTimeFormatter.ofPattern('yyyy-MM-dd').parse(\"2025-09-11\", LocalDate::from) ? null : arg0));if (v0 != null) return v0; return ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().atStartOfDay(ZoneId.of('Z')).minus(2, ChronoUnit.HOURS).toInstant().toEpochMilli(); } catch (Exception e) { return null; }" + | } + | }, + | "c2": { + | "script": { + | "lang": "painless", + | "source": "ZonedDateTime.now(ZoneId.of('Z')).toInstant().toEpochMilli()" + | } + | }, + | "c3": { + | "script": { + | "lang": "painless", + | "source": "ZonedDateTime.now(ZoneId.of('Z')).toLocalDate()" + | } + | }, + | "c4": { + | "script": { + | "lang": "painless", + | "source": "Long.parseLong(\"125\").longValue()" + | } + | }, + | "c5": { + | "script": { + | "lang": "painless", + | "source": "LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern('yyyy-MM-dd'))" | } | } | }, @@ -1878,6 +1902,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll(",LocalDate", ", LocalDate") .replaceAll("=DateTimeFormatter", " = DateTimeFormatter") + .replaceAll("try\\{", "try {") + .replaceAll("\\}catch", "} catch ") + .replaceAll("Exceptione\\)", "Exception e) ") + .replaceAll(",DateTimeFormatter", ", DateTimeFormatter") } it should "handle case function as script field" in { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 10b95cad..312e2e71 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -14,7 +14,7 @@ package object cond { case object IsNull extends Expr("ISNULL") with ConditionalOp case object IsNotNull extends Expr("ISNOTNULL") with ConditionalOp case object NullIf extends Expr("NULLIF") with ConditionalOp - case object Exists extends Expr("EXISTS") with ConditionalOp + // case object Exists extends Expr("EXISTS") with ConditionalOp case object Case extends Expr("CASE") with ConditionalOp @@ -89,7 +89,7 @@ package object cond { override def sql: String = s"$Coalesce(${values.map(_.sql).mkString(", ")})" // Reprend l’idée de SQLValues mais pour n’importe quel token - override def out: SQLType = SQLTypeUtils.leastCommonSuperType(values.map(_.out).distinct) + override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(values.map(_.out).distinct) override def applyType(in: SQLType): SQLType = out @@ -129,7 +129,7 @@ package object cond { override def inputType: SQLAny = SQLTypes.Any - override def out: SQLType = expr1.out + override def baseType: SQLType = expr1.out override def applyType(in: SQLType): SQLType = out @@ -160,7 +160,7 @@ package object cond { s"$exprPart $whenThen$elsePart $END" } - override def out: SQLType = + override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType( conditions.map(_._2.out) ++ default.map(_.out).toList ) @@ -209,12 +209,12 @@ package object cond { case i: Identifier if i.name == name && cond.isInstanceOf[Identifier] => i.nullable = false if (cond.asInstanceOf[Identifier].functions.isEmpty) - s"def val$idx = $c; if (expr == val$idx) return ${SQLTypeUtils.coerce(i.toPainless(s"val$idx"), i.out, out, nullable = false)};" + s"def val$idx = $c; if (expr == val$idx) return ${SQLTypeUtils.coerce(i.toPainless(s"val$idx"), i.baseType, out, nullable = false)};" else { cond.asInstanceOf[Identifier].nullable = false s"def e$idx = ${i.checkNotNull}; def val$idx = e$idx != null ? ${SQLTypeUtils - .coerce(cond.asInstanceOf[Identifier].toPainless(s"e$idx"), cond.out, out, nullable = false)} : null; if (expr == val$idx) return ${SQLTypeUtils - .coerce(i.toPainless(s"e$idx"), i.out, out, nullable = false)};" + .coerce(cond.asInstanceOf[Identifier].toPainless(s"e$idx"), cond.baseType, out, nullable = false)} : null; if (expr == val$idx) return ${SQLTypeUtils + .coerce(i.toPainless(s"e$idx"), i.baseType, out, nullable = false)};" } case _ => s"if (expr == $c) return ${SQLTypeUtils.coerce(res, out)};" @@ -226,7 +226,7 @@ package object cond { res match { case i: Identifier if i.name == name && cond.isInstanceOf[Expression] => i.nullable = false - SQLTypeUtils.coerce(i.toPainless("left"), i.out, out, nullable = false) + SQLTypeUtils.coerce(i.toPainless("left"), i.baseType, out, nullable = false) case _ => SQLTypeUtils.coerce(res, out) } s"if ($c) return $r;" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index aeda9728..65bdd5e6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -5,25 +5,68 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} package object convert { - case object Cast extends Expr("CAST") with TokenRegex + sealed trait Conversion extends TransformFunction[SQLType, SQLType] { + override def toSQL(base: String): String = sql + + def value: PainlessScript + def targetType: SQLType + def safe: Boolean - case class Cast(value: PainlessScript, targetType: SQLType, as: Boolean = true) - extends TransformFunction[SQLType, SQLType] { - override def inputType: SQLType = value.out + override def inputType: SQLType = value.baseType override def outputType: SQLType = targetType override def args: List[PainlessScript] = List.empty - override def sql: String = - s"$Cast(${value.sql} ${if (as) s"$Alias " else ""}${targetType.typeId})" + //override def nullable: Boolean = value.nullable - override def toSQL(base: String): String = sql + override def painless: String = SQLTypeUtils.coerce(value, targetType) + + override def toPainless(base: String, idx: Int): String = { + val ret = SQLTypeUtils.coerce(base, value.baseType, targetType, value.nullable) + val bloc = ret.startsWith("{") && ret.endsWith("}") + val retWithBrackets = if (bloc) ret else s"{ return $ret; }" + if (safe) s"try $retWithBrackets catch (Exception e) { return null; }" + else ret + } + } + + case object Cast extends Expr("CAST") with TokenRegex + + case object TryCast extends Expr("TRY_CAST") with TokenRegex { + override def words: List[String] = List(sql, "SAFE_CAST") + } + + case class Cast( + value: PainlessScript, + targetType: SQLType, + as: Boolean = true, + safe: Boolean = false + ) extends Conversion { + override def sql: String = { + val ret = s"${value.sql} ${if (as) s"$Alias " else ""}$targetType" + if (safe) s"$TryCast($ret)" + else s"$Cast($ret)" + } + value.cast(targetType) + } + + case object CastOperator extends Expr("\\:\\:") with TokenRegex + + case class CastOperator(value: PainlessScript, targetType: SQLType) extends Conversion { + override def sql: String = s"${value.sql}::$targetType" - override def painless: String = - SQLTypeUtils.coerce(value, targetType) + override def safe: Boolean = false - override def toPainless(base: String, idx: Int): String = - SQLTypeUtils.coerce(base, value.out, targetType, value.nullable) + value.cast(targetType) } + case object Convert extends Expr("CONVERT") with TokenRegex + + case class Convert(value: PainlessScript, targetType: SQLType) extends Conversion { + override def sql: String = s"$Convert(${value.sql}, $targetType)" + + override def safe: Boolean = false + + value.cast(targetType) + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index a1d74b48..336cdd93 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -95,7 +95,7 @@ package object function { override def in: SQLType = functions.lastOption.map(_.in).getOrElse(super.in) - override def out: SQLType = { + override def baseType: SQLType = { val baseType = functions.lastOption.map(_.in).getOrElse(super.baseType) functions.reverse.foldLeft(baseType) { (currentType, fun) => fun.applyType(currentType) @@ -106,6 +106,15 @@ package object function { case _: ArithmeticExpression => true case _ => false } + + override def cast(targetType: SQLType): SQLType = { + functions.headOption match { + case Some(f) => + f.cast(targetType) + case None => + this.baseType + } + } } trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { @@ -121,7 +130,7 @@ package object function { def outputType: Out override def in: SQLType = inputType - override def out: SQLType = outputType + override def baseType: SQLType = outputType override def applyType(in: SQLType): SQLType = outputType @@ -143,7 +152,7 @@ package object function { .filter(_.nullable) .zipWithIndex .map { case (a, i) => - s"def arg$i = ${SQLTypeUtils.coerce(a.painless, a.out, argTypes(i), nullable = false)};" + s"def arg$i = ${SQLTypeUtils.coerce(a.painless, a.baseType, argTypes(i), nullable = false)};" } .mkString(" ") @@ -152,7 +161,7 @@ package object function { if (a.nullable) s"arg$i" else - SQLTypeUtils.coerce(a.painless, a.out, argTypes(i), nullable = false) + SQLTypeUtils.coerce(a.painless, a.baseType, argTypes(i), nullable = false) } if (args.exists(_.nullable)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 3a1180e7..13598915 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -12,6 +12,9 @@ package object string { case object Concat extends Expr("CONCAT") with StringOp { override def painless: String = " + " } + case object Pipe extends Expr("\\|\\|") with StringOp { + override def painless: String = " + " + } case object Lower extends Expr("LOWER") with StringOp case object Upper extends Expr("UPPER") with StringOp case object Trim extends Expr("TRIM") with StringOp @@ -100,7 +103,7 @@ package object string { else callArgs.zipWithIndex .map { case (arg, idx) => - SQLTypeUtils.coerce(arg, values(idx).out, SQLTypes.Varchar, nullable = false) + SQLTypeUtils.coerce(arg, values(idx).baseType, SQLTypes.Varchar, nullable = false) } .mkString(stringOp.painless) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index f7ceb182..6cf4dd77 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -48,9 +48,9 @@ package object time { override def toPainless(base: String, idx: Int): String = if (nullable) - s"(def e$idx = $base; e$idx != null ? ${SQLTypeUtils.coerce(s"e$idx", expr.out, out, nullable = false)}$painless : null)" + s"(def e$idx = $base; e$idx != null ? ${SQLTypeUtils.coerce(s"e$idx", expr.baseType, out, nullable = false)}$painless : null)" else - s"${SQLTypeUtils.coerce(base, expr.out, out, nullable = expr.nullable)}$painless" + s"${SQLTypeUtils.coerce(base, expr.baseType, out, nullable = expr.nullable)}$painless" } sealed trait AddInterval[IO <: SQLTemporal] extends IntervalFunction[IO] { @@ -73,15 +73,15 @@ package object time { sealed trait DateTimeFunction extends Function { def now: String = "ZonedDateTime.now(ZoneId.of('Z'))" - override def out: SQLType = SQLTypes.DateTime + override def baseType: SQLType = SQLTypes.DateTime } sealed trait DateFunction extends DateTimeFunction { - override def out: SQLType = SQLTypes.Date + override def baseType: SQLType = SQLTypes.Date } sealed trait TimeFunction extends DateTimeFunction { - override def out: SQLType = SQLTypes.Time + override def baseType: SQLType = SQLTypes.Time } sealed trait SystemFunction extends Function { @@ -94,44 +94,63 @@ package object time { extends DateTimeFunction with CurrentFunction with MathScript { - override def painless: String = now + override def painless: String = + SQLTypeUtils.coerce(now, this.baseType, this.out, nullable = false) override def script: String = "now" } sealed trait CurrentDateFunction extends DateFunction with CurrentFunction with MathScript { - override def painless: String = s"$now.toLocalDate()" + override def painless: String = + SQLTypeUtils.coerce(s"$now.toLocalDate()", this.baseType, this.out, nullable = false) override def script: String = "now" } sealed trait CurrentTimeFunction extends TimeFunction with CurrentFunction { - override def painless: String = s"$now.toLocalTime()" + override def painless: String = + SQLTypeUtils.coerce(s"$now.toLocalTime()", this.baseType, this.out, nullable = false) } - case object CurrentDate extends Expr("CURRENT_DATE") with CurrentDateFunction { + case object CurrentDate extends Expr("CURRENT_DATE") with TokenRegex { override lazy val words: List[String] = List(sql, "CURDATE") } - case object CurentDateWithParens extends Expr("CURRENT_DATE()") with CurrentDateFunction + case class CurrentDate(parens: Boolean = false) extends CurrentDateFunction { + override def sql: String = + if (parens) s"$CurrentDate()" + else CurrentDate.sql + } - case object CurrentTime extends Expr("CURRENT_TIME") with CurrentTimeFunction { + case object CurrentTime extends Expr("CURRENT_TIME") with TokenRegex { override lazy val words: List[String] = List(sql, "CURTIME") } - case object CurrentTimeWithParens extends Expr("CURRENT_TIME()") with CurrentTimeFunction + case class CurrentTime(parens: Boolean = false) extends CurrentTimeFunction { + override def sql: String = + if (parens) s"$CurrentTime()" + else CurrentTime.sql + } - case object CurrentTimestamp extends Expr("CURRENT_TIMESTAMP") with CurrentDateTimeFunction + case object CurrentTimestamp extends Expr("CURRENT_TIMESTAMP") with TokenRegex - case object CurrentTimestampWithParens - extends Expr("CURRENT_TIMESTAMP()") - with CurrentDateTimeFunction + case class CurrentTimestamp(parens: Boolean = false) extends CurrentDateTimeFunction { + override def sql: String = + if (parens) s"$CurrentTimestamp()" + else CurrentTimestamp.sql + } - case object Now extends Expr("NOW") with CurrentDateTimeFunction + case object Now extends Expr("NOW") with TokenRegex - case object NowWithParens extends Expr("NOW()") with CurrentDateTimeFunction + case class Now(parens: Boolean = false) extends CurrentDateTimeFunction { + override def sql: String = if (parens) s"$Now()" else Now.sql + } - case object Today extends Expr("TODAY") with CurrentDateFunction + case object Today extends Expr("TODAY") with TokenRegex - case object TodayWithParens extends Expr("TODAY()") with CurrentDateFunction + case class Today(parens: Boolean = false) extends CurrentDateFunction { + override def sql: String = + if (parens) s"$Today()" + else Today.sql + } case object DateTrunc extends Expr("DATE_TRUNC") with TokenRegex with PainlessScript { override def painless: String = ".truncatedTo" @@ -240,7 +259,7 @@ package object time { } override def toPainless(base: String, idx: Int): String = { - val arg = SQLTypeUtils.coerce(base, identifier.out, SQLTypes.Date, nullable = false) + val arg = SQLTypeUtils.coerce(base, identifier.baseType, SQLTypes.Date, nullable = false) if (nullable && base.nonEmpty) s"(def e$idx = $arg; e$idx != null ? ${toPainlessCall(List(s"e$idx"))} : null)" else diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 9a571a91..7d8fe919 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -28,7 +28,7 @@ case class ArithmeticExpression( expr } - override def out: SQLType = + override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(List(left.out, right.out)) override def validate(): Either[String, Unit] = { @@ -45,13 +45,13 @@ case class ArithmeticExpression( if (nullable) { val l = left match { case t: TransformFunction[_, _] => - SQLTypeUtils.coerce(t.toPainless("", idx + 1), left.out, out, nullable = false) - case _ => SQLTypeUtils.coerce(left.painless, left.out, out, nullable = false) + SQLTypeUtils.coerce(t.toPainless("", idx + 1), left.baseType, out, nullable = false) + case _ => SQLTypeUtils.coerce(left.painless, left.baseType, out, nullable = false) } val r = right match { case t: TransformFunction[_, _] => - SQLTypeUtils.coerce(t.toPainless("", idx + 1), right.out, out, nullable = false) - case _ => SQLTypeUtils.coerce(right.painless, right.out, out, nullable = false) + SQLTypeUtils.coerce(t.toPainless("", idx + 1), right.baseType, out, nullable = false) + case _ => SQLTypeUtils.coerce(right.painless, right.baseType, out, nullable = false) } var expr = "" if (left.nullable) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 5611c609..1e85067e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -30,7 +30,15 @@ package object sql { override def toString: String = sql def baseType: SQLType = SQLTypes.Any def in: SQLType = baseType - def out: SQLType = baseType + private[this] var _out: SQLType = SQLTypes.Null + def out: SQLType = if (_out == SQLTypes.Null) baseType else _out + def out_=(t: SQLType): Unit = { + _out = t + } + def cast(targetType: SQLType): SQLType = { + this.out = targetType + this.out + } def system: Boolean = false def nullable: Boolean = !system } @@ -74,12 +82,18 @@ package object sql { case _ => values.headOption } } - override def painless: String = value match { - case s: String => s""""$s"""" - case b: Boolean => b.toString - case n: Number => n.toString - case _ => value.toString - } + override def painless: String = + SQLTypeUtils.coerce( + value match { + case s: String => s""""$s"""" + case b: Boolean => b.toString + case n: Number => n.toString + case _ => value.toString + }, + this.baseType, + this.out, + nullable = false + ) override def nullable: Boolean = false } @@ -88,17 +102,17 @@ package object sql { override def sql: String = "NULL" override def painless: String = "null" override def nullable: Boolean = true - override def out: SQLType = SQLTypes.Null + override def baseType: SQLType = SQLTypes.Null } case class BooleanValue(override val value: Boolean) extends Value[Boolean](value) { override def sql: String = value.toString - override def out: SQLType = SQLTypes.Boolean + override def baseType: SQLType = SQLTypes.Boolean } case class CharValue(override val value: Char) extends Value[Char](value) { override def sql: String = s"""'$value'""" - override def out: SQLType = SQLTypes.Char + override def baseType: SQLType = SQLTypes.Char } case class StringValue(override val value: String) extends Value[String](value) { @@ -127,7 +141,7 @@ package object sql { case _ => super.choose(values, operator, separator) } } - override def out: SQLType = SQLTypes.Varchar + override def baseType: SQLType = SQLTypes.Varchar } sealed abstract class NumericValue[T: Numeric](override val value: T)(implicit @@ -161,43 +175,43 @@ package object sql { def ne: Seq[T] => Boolean = { _.forall { _ != value } } - override def out: SQLNumeric = SQLTypes.Numeric + override def baseType: SQLNumeric = SQLTypes.Numeric } case class ByteValue(override val value: Byte) extends NumericValue[Byte](value) { - override def out: SQLNumeric = SQLTypes.TinyInt + override def baseType: SQLNumeric = SQLTypes.TinyInt } case class ShortValue(override val value: Short) extends NumericValue[Short](value) { - override def out: SQLNumeric = SQLTypes.SmallInt + override def baseType: SQLNumeric = SQLTypes.SmallInt } case class IntValue(override val value: Int) extends NumericValue[Int](value) { - override def out: SQLNumeric = SQLTypes.Int + override def baseType: SQLNumeric = SQLTypes.Int } case class LongValue(override val value: Long) extends NumericValue[Long](value) { - override def out: SQLNumeric = SQLTypes.BigInt + override def baseType: SQLNumeric = SQLTypes.BigInt } case class FloatValue(override val value: Float) extends NumericValue[Float](value) { - override def out: SQLNumeric = SQLTypes.Real + override def baseType: SQLNumeric = SQLTypes.Real } case class DoubleValue(override val value: Double) extends NumericValue[Double](value) { - override def out: SQLNumeric = SQLTypes.Double + override def baseType: SQLNumeric = SQLTypes.Double } case object PiValue extends Value[Double](Math.PI) with TokenRegex { override def sql: String = "PI" override def painless: String = "Math.PI" - override def out: SQLNumeric = SQLTypes.Double + override def baseType: SQLNumeric = SQLTypes.Double } case object EValue extends Value[Double](Math.E) with TokenRegex { override def sql: String = "E" override def painless: String = "Math.E" - override def out: SQLNumeric = SQLTypes.Double + override def baseType: SQLNumeric = SQLTypes.Double } sealed abstract class FromTo[+T](val from: Value[T], val to: Value[T]) extends Token { @@ -241,7 +255,7 @@ package object sql { override def painless: String = s"[${values.map(_.painless).mkString(",")}]" lazy val innerValues: Seq[R] = values.map(_.value) override def nullable: Boolean = values.exists(_.nullable) - override def out: SQLArray = SQLTypes.Array(SQLTypes.Any) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Any) } case class StringValues(override val values: Seq[StringValue]) @@ -252,7 +266,7 @@ package object sql { def ne: Seq[String] => Boolean = { _.forall { s => innerValues.forall(!_.contentEquals(s)) } } - override def out: SQLArray = SQLTypes.Array(SQLTypes.Varchar) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Varchar) } class NumericValues[R: TypeTag](override val values: Seq[NumericValue[R]]) @@ -263,34 +277,34 @@ package object sql { def ne: Seq[R] => Boolean = { _.forall { n => !innerValues.contains(n) } } - override def out: SQLArray = SQLTypes.Array(SQLTypes.Numeric) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Numeric) } case class ByteValues(override val values: Seq[ByteValue]) extends NumericValues[Byte](values) { - override def out: SQLArray = SQLTypes.Array(SQLTypes.TinyInt) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.TinyInt) } case class ShortValues(override val values: Seq[ShortValue]) extends NumericValues[Short](values) { - override def out: SQLArray = SQLTypes.Array(SQLTypes.SmallInt) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.SmallInt) } case class IntValues(override val values: Seq[IntValue]) extends NumericValues[Int](values) { - override def out: SQLArray = SQLTypes.Array(SQLTypes.Int) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Int) } case class LongValues(override val values: Seq[LongValue]) extends NumericValues[Long](values) { - override def out: SQLArray = SQLTypes.Array(SQLTypes.BigInt) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.BigInt) } case class FloatValues(override val values: Seq[FloatValue]) extends NumericValues[Float](values) { - override def out: SQLArray = SQLTypes.Array(SQLTypes.Real) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Real) } case class DoubleValues(override val values: Seq[DoubleValue]) extends NumericValues[Double](values) { - override def out: SQLArray = SQLTypes.Array(SQLTypes.Double) + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Double) } def choose[T]( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 9ba1c688..7422e838 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -235,17 +235,17 @@ trait Parser private val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex def identifier: PackratParser[Identifier] = - Distinct.regex.? ~ identifierRegex ^^ { case d ~ i => + (Distinct.regex.? ~ identifierRegex ^^ { case d ~ i => GenericIdentifier( i, None, d.isDefined ) - } + }) >> castOperator def identifierWithTransformation: PackratParser[Identifier] = - mathematicalFunctionWithIdentifier | - castFunctionWithIdentifier | + (mathematicalFunctionWithIdentifier | + conversionFunctionWithIdentifier | conditionalFunctionWithIdentifier | systemFunctionWithIdentifier | dateFunctionWithIdentifier | @@ -253,10 +253,10 @@ trait Parser stringFunctionWithIdentifier | date_diff_identifier | extract_identifier | - case_when_identifier + case_when_identifier) >> castOperator def identifierWithFunction: PackratParser[Identifier] = - rep1sep( + (rep1sep( sql_functions, start ) ~ start.? ~ (identifierWithTransformation | identifierWithIntervalFunction | identifier).? ~ rep1( @@ -271,7 +271,7 @@ trait Parser } case Some(id) => id.withFunctions(id.functions ++ f) } - } + }) >> castOperator private val regexAlias = s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_]*""".stripMargin diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala index 8164a8ae..e195b5f7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.parser.function -import app.softnetwork.elastic.sql.function.convert.Cast +import app.softnetwork.elastic.sql.function.convert.{Cast, CastOperator, Convert, TryCast} import app.softnetwork.elastic.sql.{Alias, Identifier} import app.softnetwork.elastic.sql.parser.Parser @@ -9,7 +9,7 @@ package object convert { trait ConvertParser { self: Parser => def castFunctionWithIdentifier: PackratParser[Identifier] = - "(?i)cast".r ~ start ~ (identifierWithTransformation | + Cast.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { @@ -17,5 +17,35 @@ package object convert { i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) } + def tryCastFunctionWithIdentifier: PackratParser[Identifier] = + TryCast.regex ~ start ~ (identifierWithTransformation | + identifierWithIntervalFunction | + identifierWithFunction | + identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { + case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => + i.withFunctions( + a.toList ++ (Cast(i, targetType = t, as = as.isDefined, safe = true) +: i.functions) + ) + } + + def castOperator: Identifier => PackratParser[Identifier] = i => + (CastOperator.regex ~ sql_type).? ^^ { + case None => i + case Some(_ ~ t) => + i.withFunctions(CastOperator(i, targetType = t) +: i.functions) + } + + def convertFunctionWithIdentifier: PackratParser[Identifier] = + Convert.regex ~ start ~ (identifierWithTransformation | + identifierWithIntervalFunction | + identifierWithFunction | + identifier) ~ separator ~ sql_type ~ end ~ intervalFunction.? ^^ { + case _ ~ _ ~ i ~ _ ~ t ~ _ ~ a => + i.withFunctions(a.toList ++ (Convert(i, targetType = t) +: i.functions)) + } + + def conversionFunctionWithIdentifier: PackratParser[Identifier] = + castFunctionWithIdentifier | tryCastFunctionWithIdentifier | convertFunctionWithIdentifier + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index 046ba0ac..957ba3d2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -21,25 +21,25 @@ package object time { def current_date: PackratParser[CurrentFunction] = CurrentDate.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) CurentDateWithParens else CurrentDate + CurrentDate(p.isDefined) } def current_time: PackratParser[CurrentFunction] = CurrentTime.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) CurrentTimeWithParens else CurrentTime + CurrentTime(p.isDefined) } def current_timestamp: PackratParser[CurrentFunction] = CurrentTimestamp.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) CurrentTimestampWithParens else CurrentTimestamp + CurrentTimestamp(p.isDefined) } def now: PackratParser[CurrentFunction] = Now.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) NowWithParens else Now + Now(p.isDefined) } def today: PackratParser[CurrentFunction] = Today.regex ~ parens.? ^^ { case _ ~ p => - if (p.isDefined) TodayWithParens else Today + Today(p.isDefined) } def systemFunctions: PackratParser[CurrentFunction] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala index 77d3e3c8..7a211d3b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala @@ -49,13 +49,13 @@ package object math { } def identifierWithArithmeticExpression: Parser[Identifier] = - arithmeticExpressionLevel2 ^^ { + (arithmeticExpressionLevel2 ^^ { case af: ArithmeticExpression => Identifier(af) case id: Identifier => id case f: FunctionWithIdentifier => f.identifier case f: Function => Identifier(f) case other => throw new Exception(s"Unexpected expression $other") - } + }) >> castOperator } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala index f04abfa3..5d6f0c8a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala @@ -93,11 +93,11 @@ package object time { add_interval | substract_interval def identifierWithIntervalFunction: PackratParser[Identifier] = - (identifierWithTransformation | + ((identifierWithTransformation | identifierWithFunction | identifier) ~ intervalFunction ^^ { case i ~ f => i.withFunctions(f +: i.functions) - } + }) >> castOperator } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 66be42ee..5de5f0b4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -367,7 +367,7 @@ case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditiona case class IsNotNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { - override val conditionalFunction: ConditionalFunction[SQLAny] = IsNotNull( + override lazy val conditionalFunction: ConditionalFunction[SQLAny] = IsNotNull( identifier ) override val operator: Operator = IS_NOT_NULL diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index 2c91a959..a8a7db8f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -1,6 +1,9 @@ package app.softnetwork.elastic.sql.`type` -sealed trait SQLType { def typeId: String } +sealed trait SQLType { + def typeId: String + override def toString: String = typeId +} trait SQLAny extends SQLType diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 50c244ec..b0967826 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -89,7 +89,7 @@ object SQLTypeUtils { def coerce(in: PainlessScript, to: SQLType): String = { val expr = in.painless - val from = in.out + val from = in.baseType val nullable = in.nullable coerce(expr, from, to, nullable) } @@ -114,14 +114,46 @@ object SQLTypeUtils { s"((double) $expr)" // ---- NUMERIC <-> TEMPORAL ---- - case (SQLTypes.BigInt, SQLTypes.Timestamp) => + case (SQLTypes.BigInt, SQLTypes.Timestamp | SQLTypes.DateTime) => s"Instant.ofEpochMilli($expr).atZone(ZoneId.of('Z'))" - case (SQLTypes.Timestamp, SQLTypes.BigInt) => + case (SQLTypes.Timestamp | SQLTypes.DateTime, SQLTypes.BigInt) => s"$expr.toInstant().toEpochMilli()" - // ---- BOOLEEN -> NUMERIC ---- - case (SQLTypes.Boolean, SQLTypes.Numeric) => + // ---- BOOLEAN -> NUMERIC ---- + case (SQLTypes.Boolean, SQLTypes.Numeric | SQLTypes.Int) => s"($expr ? 1 : 0)" + case (SQLTypes.Boolean, SQLTypes.BigInt) => + s"($expr ? 1L : 0L)" + case (SQLTypes.Boolean, SQLTypes.Double) => + s"($expr ? 1.0 : 0.0)" + case (SQLTypes.Boolean, SQLTypes.Real) => + s"($expr ? 1.0f : 0.0f)" + case (SQLTypes.Boolean, SQLTypes.SmallInt) => + s"(short)($expr ? 1 : 0)" + case (SQLTypes.Boolean, SQLTypes.TinyInt) => + s"(byte)($expr ? 1 : 0)" + + // ---- VARCHAR -> TEMPORAL ---- + case (SQLTypes.Varchar, SQLTypes.Int) => + s"Integer.parseInt($expr).intValue()" + case (SQLTypes.Varchar, SQLTypes.BigInt) => + s"Long.parseLong($expr).longValue()" + case (SQLTypes.Varchar, SQLTypes.Double) => + s"Double.parseDouble($expr).doubleValue()" + case (SQLTypes.Varchar, SQLTypes.Real) => + s"Float.parseFloat($expr).floatValue()" + case (SQLTypes.Varchar, SQLTypes.SmallInt) => + s"Short.parseShort($expr).shortValue()" + case (SQLTypes.Varchar, SQLTypes.TinyInt) => + s"Byte.parseByte($expr).byteValue()" + + // ---- VARCHAR -> DATE ---- + case (SQLTypes.Varchar, SQLTypes.Date) => + s"LocalDate.parse($expr, DateTimeFormatter.ofPattern('yyyy-MM-dd'))" + case (SQLTypes.Varchar, SQLTypes.Time) => + s"LocalTime.parse($expr, DateTimeFormatter.ofPattern('HH:mm:ss'))" + case (SQLTypes.Varchar, SQLTypes.DateTime | SQLTypes.Timestamp) => + s"ZonedDateTime.parse($expr, DateTimeFormatter.ISO_ZONED_DATE_TIME)" // ---- IDENTITY ---- case (_, _) if from == to => diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index a88215e2..f436a4e0 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -144,8 +144,8 @@ object Queries { "SELECT COALESCE(createdAt - INTERVAL 35 MINUTE, CURRENT_DATE) AS c, identifier FROM Table" val nullif: String = "SELECT COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 DAY), CURRENT_DATE) AS c, identifier FROM Table" - val cast: String = - "SELECT CAST(COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 HOUR) BIGINT) AS c, identifier FROM Table" + val conversion: String = + "SELECT TRY_CAST(COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 HOUR) BIGINT) AS c, CONVERT(CURRENT_TIMESTAMP, BIGINT) AS c2, CURRENT_TIMESTAMP::DATE AS c3, '125'::BIGINT AS c4, '2025-09-11'::DATE AS c5, identifier FROM Table" val allCasts = "SELECT CAST(identifier AS int) AS c1, CAST(identifier AS bigint) AS c2, CAST(identifier AS double) AS c3, CAST(identifier AS real) AS c4, CAST(identifier AS boolean) AS c5, CAST(identifier AS char) AS c6, CAST(identifier AS varchar) AS c7, CAST(createdAt AS date) AS c8, CAST(createdAt AS time) AS c9, CAST(createdAt AS datetime) AS c10, CAST(createdAt AS timestamp) AS c11, CAST(identifier AS smallint) AS c12, CAST(identifier AS tinyint) AS c13 FROM Table" val caseWhen: String = @@ -745,11 +745,11 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(nullif) shouldBe true } - it should "parse CAST function" in { - val result = Parser(cast) + it should "parse conversion function" in { + val result = Parser(conversion) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") shouldBe cast + .getOrElse("") shouldBe conversion } it should "parse all casts function" in { From 16fe45aaf0985bcd2316357d6838e6c9c32595cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 26 Sep 2025 19:38:47 +0200 Subject: [PATCH 26/48] add support for OFFSET with LIMIT --- .../elastic/sql/bridge/ElasticSearchRequest.scala | 1 + .../app/softnetwork/elastic/sql/bridge/package.scala | 3 ++- .../elastic/sql/bridge/ElasticSearchRequest.scala | 1 + .../app/softnetwork/elastic/sql/bridge/package.scala | 3 ++- .../softnetwork/elastic/sql/parser/LimitParser.scala | 10 +++++++--- .../app/softnetwork/elastic/sql/query/Limit.scala | 7 ++++++- .../app/softnetwork/elastic/sql/SQLParserSpec.scala | 5 ++--- 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 569d392d..bac7afb9 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -10,6 +10,7 @@ case class ElasticSearchRequest( sources: Seq[String], criteria: Option[Criteria], limit: Option[Int], + offset: Option[Int], search: SearchRequest, buckets: Seq[Bucket] = Seq.empty, aggregations: Seq[ElasticAggregation] = Seq.empty diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 827aef7a..a8998c15 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -24,6 +24,7 @@ package object bridge { request.sources, request.where.flatMap(_.criteria), request.limit.map(_.limit), + request.limit.flatMap(_.offset.map(_.offset)).orElse(Some(0)), request, request.buckets, request.aggregates.map( @@ -126,7 +127,7 @@ package object bridge { _search size 0 } else { limit match { - case Some(l) => _search limit l.limit from 0 + case Some(l) => _search limit l.limit from l.offset.map(_.offset).getOrElse(0) case _ => _search } } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index dc5e6cc4..5535e71c 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -9,6 +9,7 @@ case class ElasticSearchRequest( sources: Seq[String], criteria: Option[Criteria], limit: Option[Int], + offset: Option[Int], search: SearchRequest, buckets: Seq[Bucket] = Seq.empty, aggregations: Seq[ElasticAggregation] = Seq.empty diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 76eae43c..6131ceb8 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -27,6 +27,7 @@ package object bridge { request.sources, request.where.flatMap(_.criteria), request.limit.map(_.limit), + request.limit.flatMap(_.offset.map(_.offset)), request, request.buckets, request.aggregates.map( @@ -128,7 +129,7 @@ package object bridge { _search size 0 } else { limit match { - case Some(l) => _search limit l.limit from 0 + case Some(l) => _search limit l.limit from l.offset.map(_.offset).getOrElse(0) case _ => _search } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala index 043000e4..2bc2a0cb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/LimitParser.scala @@ -1,12 +1,16 @@ package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.sql.query.Limit +import app.softnetwork.elastic.sql.query.{Limit, Offset} trait LimitParser { self: Parser => - def limit: PackratParser[Limit] = Limit.regex ~ long ^^ { case _ ~ i => - Limit(i.value.toInt) + def offset: PackratParser[Offset] = Offset.regex ~ long ^^ { case _ ~ i => + Offset(i.value.toInt) + } + + def limit: PackratParser[Limit] = Limit.regex ~ long ~ offset.? ^^ { case _ ~ i ~ o => + Limit(i.value.toInt, o) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala index f1e421ab..3cb1f3af 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Limit.scala @@ -4,4 +4,9 @@ import app.softnetwork.elastic.sql.{Expr, TokenRegex} case object Limit extends Expr("LIMIT") with TokenRegex -case class Limit(limit: Int) extends Expr(s" LIMIT $limit") +case class Limit(limit: Int, offset: Option[Offset]) + extends Expr(s" LIMIT $limit${offset.map(_.sql).getOrElse("")}") + +case object Offset extends Expr("OFFSET") with TokenRegex + +case class Offset(offset: Int) extends Expr(s" OFFSET $offset") diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index f436a4e0..bf4928bb 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -68,7 +68,7 @@ object Queries { val groupBy = "SELECT identifier, COUNT(identifier2) FROM Table WHERE identifier2 is NOT null GROUP BY identifier" val orderBy = "SELECT * FROM Table ORDER BY identifier DESC" - val limit = "SELECT * FROM Table limit 10" + val limit = "SELECT * FROM Table LIMIT 10 OFFSET 2" val groupByWithOrderByAndLimit: String = """SELECT identifier, COUNT(identifier2) |FROM Table @@ -550,8 +550,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { val result = Parser(limit) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") - .equalsIgnoreCase(limit) shouldBe true + .getOrElse("") shouldBe limit } it should "parse GROUP BY with ORDER BY and LIMIT" in { From ac9cfb3884563276635938380abc5a5390ff1ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 27 Sep 2025 05:50:11 +0200 Subject: [PATCH 27/48] fix baseTypes and TimeFieldExtract --- .../elastic/sql/function/cond/package.scala | 9 +++--- .../elastic/sql/function/time/package.scala | 32 +++++++++---------- .../operator/math/ArithmeticExpression.scala | 2 +- .../sql/parser/function/time/package.scala | 30 ++++++++--------- .../sql/SQLDateTimeFunctionSuite.scala | 12 +++---- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 312e2e71..c6a27d5b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -80,7 +80,7 @@ package object cond { override def args: List[PainlessScript] = values - override def outputType: SQLType = SQLTypeUtils.leastCommonSuperType(args.map(_.out)) + override def outputType: SQLType = SQLTypeUtils.leastCommonSuperType(args.map(_.baseType)) override def identifier: Identifier = Identifier() @@ -89,7 +89,8 @@ package object cond { override def sql: String = s"$Coalesce(${values.map(_.sql).mkString(", ")})" // Reprend l’idée de SQLValues mais pour n’importe quel token - override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(values.map(_.out).distinct) + override def baseType: SQLType = + SQLTypeUtils.leastCommonSuperType(values.map(_.baseType).distinct) override def applyType(in: SQLType): SQLType = out @@ -162,10 +163,10 @@ package object cond { override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType( - conditions.map(_._2.out) ++ default.map(_.out).toList + conditions.map(_._2.baseType) ++ default.map(_.baseType).toList ) - override def applyType(in: SQLType): SQLType = out + override def applyType(in: SQLType): SQLType = baseType override def validate(): Either[String, Unit] = { if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 6cf4dd77..d7588d59 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -197,42 +197,42 @@ package object time { import TimeField._ - sealed trait TimeFieldExtract extends Extract { + sealed abstract class TimeFieldExtract(field: TimeField) extends Extract(field) { override val sql: String = field.sql override def toSQL(base: String): String = s"$sql($base)" } - object Year extends Extract(YEAR) with TimeFieldExtract + class Year extends TimeFieldExtract(YEAR) - object MonthOfYear extends Extract(MONTH_OF_YEAR) with TimeFieldExtract + class MonthOfYear extends TimeFieldExtract(MONTH_OF_YEAR) - object DayOfMonth extends Extract(DAY_OF_MONTH) with TimeFieldExtract + class DayOfMonth extends TimeFieldExtract(DAY_OF_MONTH) - object DayOfWeek extends Extract(DAY_OF_WEEK) with TimeFieldExtract + class DayOfWeek extends TimeFieldExtract(DAY_OF_WEEK) - object DayOfYear extends Extract(DAY_OF_YEAR) with TimeFieldExtract + class DayOfYear extends TimeFieldExtract(DAY_OF_YEAR) - object HourOfDay extends Extract(HOUR_OF_DAY) with TimeFieldExtract + class HourOfDay extends TimeFieldExtract(HOUR_OF_DAY) - object MinuteOfHour extends Extract(MINUTE_OF_HOUR) with TimeFieldExtract + class MinuteOfHour extends TimeFieldExtract(MINUTE_OF_HOUR) - object SecondOfMinute extends Extract(SECOND_OF_MINUTE) with TimeFieldExtract + class SecondOfMinute extends TimeFieldExtract(SECOND_OF_MINUTE) - object NanoOfSecond extends Extract(NANO_OF_SECOND) with TimeFieldExtract + class NanoOfSecond extends TimeFieldExtract(NANO_OF_SECOND) - object MicroOfSecond extends Extract(MICRO_OF_SECOND) with TimeFieldExtract + class MicroOfSecond extends TimeFieldExtract(MICRO_OF_SECOND) - object MilliOfSecond extends Extract(MILLI_OF_SECOND) with TimeFieldExtract + class MilliOfSecond extends TimeFieldExtract(MILLI_OF_SECOND) - object EpochDay extends Extract(EPOCH_DAY) with TimeFieldExtract + class EpochDay extends TimeFieldExtract(EPOCH_DAY) - object OffsetSeconds extends Extract(OFFSET_SECONDS) with TimeFieldExtract + class OffsetSeconds extends TimeFieldExtract(OFFSET_SECONDS) import IsoField._ - object QuarterOfYear extends Extract(QUARTER_OF_YEAR) with TimeFieldExtract + class QuarterOfYear extends TimeFieldExtract(QUARTER_OF_YEAR) - object WeekOfWeekBasedYear extends Extract(WEEK_OF_WEEK_BASED_YEAR) with TimeFieldExtract + class WeekOfWeekBasedYear extends TimeFieldExtract(WEEK_OF_WEEK_BASED_YEAR) case object LastDayOfMonth extends Expr("LAST_DAY") with TokenRegex with PainlessScript { override def painless: String = ".withDayOfMonth" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 7d8fe919..15ea7ca3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -29,7 +29,7 @@ case class ArithmeticExpression( } override def baseType: SQLType = - SQLTypeUtils.leastCommonSuperType(List(left.out, right.out)) + SQLTypeUtils.leastCommonSuperType(List(left.baseType, right.baseType)) override def validate(): Either[String, Unit] = { for { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index 957ba3d2..ba7e7e31 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -177,37 +177,37 @@ package object time { import TimeField._ def year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - YEAR.regex ^^ (_ => Year) + YEAR.regex ^^ (_ => new Year) def month_of_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - MONTH_OF_YEAR.regex ^^ (_ => MonthOfYear) + MONTH_OF_YEAR.regex ^^ (_ => new MonthOfYear) def day_of_month_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - DAY_OF_MONTH.regex ^^ (_ => DayOfMonth) + DAY_OF_MONTH.regex ^^ (_ => new DayOfMonth) def day_of_week_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - DAY_OF_WEEK.regex ^^ (_ => DayOfWeek) + DAY_OF_WEEK.regex ^^ (_ => new DayOfWeek) def day_of_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - DAY_OF_YEAR.regex ^^ (_ => DayOfYear) + DAY_OF_YEAR.regex ^^ (_ => new DayOfYear) def hour_of_day_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - HOUR_OF_DAY.regex ^^ (_ => HourOfDay) + HOUR_OF_DAY.regex ^^ (_ => new HourOfDay) def minute_of_hour_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - MINUTE_OF_HOUR.regex ^^ (_ => MinuteOfHour) + MINUTE_OF_HOUR.regex ^^ (_ => new MinuteOfHour) def second_of_minute_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - SECOND_OF_MINUTE.regex ^^ (_ => SecondOfMinute) + SECOND_OF_MINUTE.regex ^^ (_ => new SecondOfMinute) def nano_of_second_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - NANO_OF_SECOND.regex ^^ (_ => NanoOfSecond) + NANO_OF_SECOND.regex ^^ (_ => new NanoOfSecond) def micro_of_second_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - MICRO_OF_SECOND.regex ^^ (_ => MicroOfSecond) + MICRO_OF_SECOND.regex ^^ (_ => new MicroOfSecond) def milli_of_second_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - MILLI_OF_SECOND.regex ^^ (_ => MilliOfSecond) + MILLI_OF_SECOND.regex ^^ (_ => new MilliOfSecond) def epoch_day_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - EPOCH_DAY.regex ^^ (_ => EpochDay) + EPOCH_DAY.regex ^^ (_ => new EpochDay) def offset_seconds_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - OFFSET_SECONDS.regex ^^ (_ => OffsetSeconds) + OFFSET_SECONDS.regex ^^ (_ => new OffsetSeconds) def quarter_of_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - IsoField.QUARTER_OF_YEAR.regex ^^ (_ => QuarterOfYear) + IsoField.QUARTER_OF_YEAR.regex ^^ (_ => new QuarterOfYear) def week_of_week_based_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = - IsoField.WEEK_OF_WEEK_BASED_YEAR.regex ^^ (_ => WeekOfWeekBasedYear) + IsoField.WEEK_OF_WEEK_BASED_YEAR.regex ^^ (_ => new WeekOfWeekBasedYear) def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = year_tr | diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala index 84335c8e..d0834bdf 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala @@ -24,12 +24,12 @@ class SQLDateTimeFunctionSuite extends AnyFunSuite { Extract(TimeField.DAY_OF_MONTH), DateFormat(Identifier(), "yyyy-MM-dd"), DateTimeFormat(Identifier(), "yyyy-MM-dd HH:mm:ss"), - Year, - MonthOfYear, - DayOfYear, - HourOfDay, - MinuteOfHour, - SecondOfMinute + new Year, + new MonthOfYear, + new DayOfYear, + new HourOfDay, + new MinuteOfHour, + new SecondOfMinute ) // Fonction pour chaîner une séquence de transformations en vérifiant les types From b338e0f35282f6fe12f560460b9553838438ca9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 27 Sep 2025 11:19:21 +0200 Subject: [PATCH 28/48] add support for geo distance as script field --- .../elastic/sql/bridge/package.scala | 10 +- .../elastic/sql/SQLQuerySpec.scala | 84 +++++++++++++- .../elastic/sql/bridge/package.scala | 6 +- .../elastic/sql/SQLQuerySpec.scala | 85 +++++++++++++- .../elastic/sql/function/geo/package.scala | 104 +++++++++++++++++- .../app/softnetwork/elastic/sql/package.scala | 4 + .../elastic/sql/parser/Parser.scala | 2 +- .../elastic/sql/parser/SelectParser.scala | 2 +- .../elastic/sql/parser/WhereParser.scala | 5 +- .../sql/parser/function/geo/package.scala | 20 +++- .../softnetwork/elastic/sql/query/Where.scala | 8 +- .../elastic/sql/SQLParserSpec.scala | 15 ++- 12 files changed, 323 insertions(+), 22 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index a8998c15..33efee8e 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -107,7 +107,13 @@ package object bridge { _search scriptfields scriptFields.map { field => scriptField( field.scriptName, - Script(script = field.painless).lang("painless").scriptType("source") + Script(script = field.painless) + .lang("painless") + .scriptType("source") + .params(field.identifier.functions.headOption match { + case Some(f: PainlessParams) => f.params + case _ => Map.empty[String, Any] + }) ) } } @@ -412,7 +418,7 @@ package object bridge { geoDistance: ElasticGeoDistance ): Query = { import geoDistance._ - geoDistanceQuery(identifier.name, lat.value, lon.value) distance distance.value + geoDistanceQuery(identifier.name, point.lat.value, point.lon.value) distance distance.value } implicit def matchToQuery( diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index fc24d54e..bb7b8bb2 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -632,8 +632,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | blockedCustomers not like "%uuid%" AND | NOT receiptOfOrdersDisabled=true AND | ( - | distance(pickup.location,(0.0,0.0)) <= "7000m" OR - | distance(withdrawals.location,(0.0,0.0)) <= "7000m" + | distance(pickup.location, POINT(0.0, 0.0)) <= "7000m" OR + | distance(withdrawals.location, POINT(0.0, 0.0)) <= "7000m" | ) | ) |GROUP BY @@ -2827,4 +2827,84 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll("(\\d)=", "$1 = ") } + + it should "handle geo distance as script field" in { + val select: ElasticSearchRequest = + SQLQuery(geoDistance) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "script_fields": { + | "d1": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | }, + | "d2": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | }, + | "d3": { + | "script": { + | "lang": "painless", + | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2)", + | "params": { + | "lat1": -70.0, + | "lon1": 40.0, + | "lat2": 0.0, + | "lon2": 0.0 + | } + | } + | } + | }, + | "_source": true + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",params", ", params") + .replaceAll("GeoPoint", " GeoPoint") + } + } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 6131ceb8..a1997cad 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -110,6 +110,10 @@ package object bridge { scriptField( field.scriptName, Script(script = field.painless).lang("painless").scriptType("source") + .params(field.identifier.functions.headOption match { + case Some(f: PainlessParams) => f.params + case _ => Map.empty[String, Any] + }) ) } } @@ -410,7 +414,7 @@ package object bridge { geoDistance: ElasticGeoDistance ): Query = { import geoDistance._ - geoDistanceQuery(identifier.name, lat.value, lon.value) distance distance.value + geoDistanceQuery(identifier.name, point.lat.value, point.lon.value) distance distance.value } implicit def matchToQuery( diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 808962d9..1ecebaa9 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -632,8 +632,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | blockedCustomers not like "%uuid%" AND | NOT receiptOfOrdersDisabled=true AND | ( - | distance(pickup.location,(0.0,0.0)) <= "7000m" OR - | distance(withdrawals.location,(0.0,0.0)) <= "7000m" + | distance(pickup.location, POINT(0.0, 0.0)) <= "7000m" OR + | distance(withdrawals.location, POINT(0.0, 0.0)) <= "7000m" | ) | ) |GROUP BY @@ -2816,4 +2816,85 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll("(\\d)=", "$1 = ") } + + + it should "handle geo distance as script field" in { + val select: ElasticSearchRequest = + SQLQuery(geoDistance) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "match_all": {} + | }, + | "script_fields": { + | "d1": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | }, + | "d2": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | }, + | "d3": { + | "script": { + | "lang": "painless", + | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2)", + | "params": { + | "lat1": -70.0, + | "lon1": 40.0, + | "lat2": 0.0, + | "lon2": 0.0 + | } + | } + | } + | }, + | "_source": true + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll("\\*", " * ") + .replaceAll("/", " / ") + .replaceAll(">", " > ") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",params", ", params") + .replaceAll("GeoPoint", " GeoPoint") + } + } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 115ba8c4..ca50abac 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -1,10 +1,110 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.Expr +import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLDouble, SQLTypes} +import app.softnetwork.elastic.sql.{ + DoubleValue, + Expr, + Identifier, + PainlessParams, + PainlessScript, + Token, + TokenRegex +} import app.softnetwork.elastic.sql.operator.Operator package object geo { - case object Distance extends Expr("DISTANCE") with Function with Operator + case object Point extends Expr("POINT") with TokenRegex + + case class Point(lat: DoubleValue, lon: DoubleValue) extends Token { + override def sql: String = s"POINT($lat, $lon)" + } + + case object Distance extends Expr("ST_DISTANCE") with Function with Operator { + override def words: List[String] = List(sql, "DISTANCE") + + override def painless: String = ".arcDistance" + } + + case class Distance(from: Either[Identifier, Point], to: Either[Identifier, Point]) + extends FunctionN[SQLAny, SQLDouble] + with PainlessParams { + + override def fun: Option[PainlessScript] = Some(Distance) + + override def inputType: SQLAny = SQLTypes.Any + override def outputType: SQLDouble = SQLTypes.Double + + override def args: List[PainlessScript] = List.empty + + override def sql: String = + s"$Distance(${from.fold(identity, identity)}, ${to.fold(identity, identity)})" + + private[this] lazy val (fromId, toId) = { + val fromId = from.fold(Some(_), _ => None) + val toId = to.fold(Some(_), _ => None) + (fromId, toId) + } + + private[this] lazy val identifiers: List[Identifier] = List(fromId, toId).flatten + + private[this] lazy val (fromPoint, toPoint) = { + val fromPoint = from.fold(_ => None, Some(_)) + val toPoint = to.fold(_ => None, Some(_)) + (fromPoint, toPoint) + } + + private[this] lazy val points: List[Point] = List(fromPoint, toPoint).flatten + + private[this] lazy val oneIdentifier: Boolean = identifiers.size == 1 + + override def nullable: Boolean = + from.fold(identity, identity).nullable || to.fold(identity, identity).nullable + + override def params: Map[String, Any] = + if (oneIdentifier) + Map( + "lat" -> points.head.lat.value, + "lon" -> points.head.lon.value + ) + else if (identifiers.isEmpty) + Map( + "lat1" -> fromPoint.get.lat.value, + "lon1" -> fromPoint.get.lon.value, + "lat2" -> toPoint.get.lat.value, + "lon2" -> toPoint.get.lon.value + ) + else + Map.empty + + override def painless: String = { + val nullCheck = + identifiers.zipWithIndex + .map { case (_, i) => s"arg$i == null" } + .mkString(" || ") + + val assignments = + identifiers.zipWithIndex + .map { case (a, i) => + val name = a.name + s"def arg$i = (!doc.containsKey('$name') || doc['$name'].empty ? ${a.nullValue} : doc['$name']);" + } + .mkString(" ") + + val ret = + if (oneIdentifier) { + s"arg0${fun.map(_.painless).getOrElse("")}(params.lat, params.lon)" + } else if (identifiers.isEmpty) { + s"new GeoPoint(params.lat1, params.lon1)${fun.map(_.painless).getOrElse("")}(params.lat2, params.lon2)" + } else { + s"arg0${fun.map(_.painless).getOrElse("")}(arg1.lat, arg1.lon)" + } + + if (identifiers.nonEmpty) + s"($assignments ($nullCheck) ? null : $ret)" + else + ret + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 1e85067e..db55104f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -48,6 +48,10 @@ package object sql { def nullValue: String = "null" } + trait PainlessParams extends PainlessScript { + def params: Map[String, Any] + } + trait MathScript extends Token { def script: String } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 7422e838..6bf2b7f0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -109,7 +109,7 @@ trait Parser } def sql_functions: PackratParser[Function] = - aggregates | distance | date_diff | date_trunc | extractors | date_functions | datetime_functions | conditional_functions | string_functions + aggregates | date_diff | date_trunc | extractors | date_functions | datetime_functions | conditional_functions | string_functions private val reservedKeywords = Seq( "select", diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala index 6746f1ee..273c7608 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -6,7 +6,7 @@ trait SelectParser { self: Parser with WhereParser => def field: PackratParser[Field] = - (identifierWithTopHits | + (distance_identifier >> castOperator | identifierWithTopHits | identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index e4c7d454..00ea32c3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.function.geo.Distance import app.softnetwork.elastic.sql.{ DoubleFromTo, DoubleValues, @@ -159,8 +160,8 @@ trait WhereParser { } def sql_distance: PackratParser[Criteria] = - distance ~ start ~ identifier ~ separator ~ start ~ double ~ separator ~ double ~ end ~ end ~ le ~ literal ^^ { - case _ ~ _ ~ i ~ _ ~ _ ~ lat ~ _ ~ lon ~ _ ~ _ ~ _ ~ d => ElasticGeoDistance(i, d, lat, lon) + Distance.regex ~ start ~ identifier ~ separator ~ point ~ end ~ le ~ literal ^^ { + case _ ~ _ ~ i ~ _ ~ p ~ _ ~ _ ~ d => ElasticGeoDistance(i, d, p) } def matchCriteria: PackratParser[MatchCriteria] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala index afe2af03..5d5295ba 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala @@ -1,14 +1,30 @@ package app.softnetwork.elastic.sql.parser.function +import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.function.Function -import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.function.geo.{Distance, Point} import app.softnetwork.elastic.sql.parser.Parser package object geo { trait GeoParser { self: Parser => - def distance: PackratParser[Function] = Distance.regex ^^ (_ => Distance) + def point: PackratParser[Point] = + Point.regex ~> start ~> double ~ separator ~ double <~ end ^^ { case lat ~ _ ~ lon => + Point(lat, lon) + } + def pointOrIdentifier: PackratParser[Either[Identifier, Point]] = + (point | identifier) ^^ { + case id: Identifier => Left(id) + case p: Point => Right(p) + } + + def distance: PackratParser[Function] = + Distance.regex ~> start ~> pointOrIdentifier ~ separator ~ pointOrIdentifier <~ end ^^ { + case from ~ _ ~ to => Distance(from, to) + } + + def distance_identifier: PackratParser[Identifier] = distance ^^ functionAsIdentifier } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 5de5f0b4..837ac4b2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.function.cond.{ConditionalFunction, IsNotNull, IsNull} -import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.function.geo.{Distance, Point} import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql._ @@ -443,10 +443,10 @@ case class BetweenExpr[+T]( case class ElasticGeoDistance( identifier: Identifier, distance: StringValue, - lat: DoubleValue, - lon: DoubleValue + point: Point ) extends Expression { - override def sql = s"$Distance($identifier,($lat,$lon)) $operator $distance" + override def sql = + s"$Distance($identifier, POINT(${point.lat}, ${point.lon})) $operator $distance" override val functions: List[Function] = List(Distance) override def operator: Operator = LE override def update(request: SQLSearchRequest): ElasticGeoDistance = diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index bf4928bb..11bcb377 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -61,7 +61,7 @@ object Queries { val isNull = "SELECT * FROM Table WHERE identifier is null" val isNotNull = "SELECT * FROM Table WHERE identifier is NOT null" val geoDistanceCriteria = - "SELECT * FROM Table WHERE distance(profile.location,(-70.0,40.0)) <= '5km'" + "SELECT * FROM Table WHERE ST_DISTANCE(profile.location, POINT(-70.0, 40.0)) <= '5km'" val except = "SELECT * except(col1,col2) FROM Table" val matchCriteria = "SELECT * FROM Table WHERE match (identifier1,identifier2,identifier3) against ('value')" @@ -173,6 +173,10 @@ object Queries { val extractors: String = "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" + + val geoDistance = + "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table" + } /** Created by smanciot on 15/02/17. @@ -510,8 +514,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { val result = Parser(geoDistanceCriteria) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") - .equalsIgnoreCase(geoDistanceCriteria) shouldBe true + .getOrElse("") shouldBe geoDistanceCriteria } it should "parse EXCEPT fields" in { @@ -827,4 +830,10 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .getOrElse("") shouldBe extractors } + it should "parse geo distance field" in { + val result = Parser(geoDistance) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") shouldBe geoDistance + } } From d76065e3f67a92316bbfa527b985175f20c0f455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 28 Sep 2025 09:01:23 +0200 Subject: [PATCH 29/48] add support for geo distance as script query --- .../elastic/sql/bridge/ElasticQuery.scala | 20 +++--- .../elastic/sql/bridge/package.scala | 25 +++++++- .../elastic/sql/SQLQuerySpec.scala | 57 +++++++++++++++-- .../elastic/sql/bridge/ElasticQuery.scala | 4 +- .../elastic/sql/bridge/package.scala | 27 ++++++-- .../elastic/sql/SQLQuerySpec.scala | 58 ++++++++++++++++-- .../elastic/sql/function/geo/package.scala | 61 ++++++++++++++++--- .../elastic/sql/parser/WhereParser.scala | 12 ++-- .../sql/parser/function/geo/package.scala | 17 +++++- .../softnetwork/elastic/sql/query/Where.scala | 43 ++++++------- .../elastic/sql/SQLParserSpec.scala | 4 +- 11 files changed, 260 insertions(+), 68 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 4cfc2c76..b773c5b5 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -2,10 +2,10 @@ package app.softnetwork.elastic.sql.bridge import app.softnetwork.elastic.sql.query.{ BetweenExpr, + DistanceCriteria, ElasticBoolQuery, ElasticChild, ElasticFilter, - ElasticGeoDistance, ElasticMatch, ElasticNested, ElasticParent, @@ -62,15 +62,15 @@ case class ElasticQuery(filter: ElasticFilter) { criteria.asQuery(group = group, innerHitsNames = innerHitsNames), score = false ) - case expression: GenericExpression => expression - case isNull: IsNullExpr => isNull - case isNotNull: IsNotNullExpr => isNotNull - case in: InExpr[_, _] => in - case between: BetweenExpr[_] => between - case geoDistance: ElasticGeoDistance => geoDistance - case matchExpression: ElasticMatch => matchExpression - case isNull: IsNullCriteria => isNull - case isNotNull: IsNotNullCriteria => isNotNull + case expression: GenericExpression => expression + case isNull: IsNullExpr => isNull + case isNotNull: IsNotNullExpr => isNotNull + case in: InExpr[_, _] => in + case between: BetweenExpr[_] => between + case geoDistance: DistanceCriteria => geoDistance + case matchExpression: ElasticMatch => matchExpression + case isNull: IsNullCriteria => isNull + case isNotNull: IsNotNullCriteria => isNotNull case other => throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") } diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 33efee8e..148824a9 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -9,6 +9,7 @@ import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn import com.sksamuel.elastic4s.script.Script +import com.sksamuel.elastic4s.script.ScriptType.Source import com.sksamuel.elastic4s.searches.aggs.Aggregation import com.sksamuel.elastic4s.searches.queries.Query import com.sksamuel.elastic4s.searches.{MultiSearchRequest, SearchRequest} @@ -415,10 +416,28 @@ package object bridge { } implicit def geoDistanceToQuery( - geoDistance: ElasticGeoDistance + distanceCriteria: DistanceCriteria ): Query = { - import geoDistance._ - geoDistanceQuery(identifier.name, point.lat.value, point.lon.value) distance distance.value + import distanceCriteria._ + operator match { + case LE | LT if distance.oneIdentifier => + val identifier = distance.identifiers.head + val point = distance.points.head + geoDistanceQuery( + identifier.name, + point.lat.value, + point.lon.value + ) distance geoDistance + case _ => + scriptQuery( + Script( + script = distanceCriteria.painless, + lang = Some("painless"), + scriptType = Source, + params = distance.params + ) + ) + } } implicit def matchToQuery( diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index bb7b8bb2..6f347709 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -632,8 +632,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | blockedCustomers not like "%uuid%" AND | NOT receiptOfOrdersDisabled=true AND | ( - | distance(pickup.location, POINT(0.0, 0.0)) <= "7000m" OR - | distance(withdrawals.location, POINT(0.0, 0.0)) <= "7000m" + | distance(pickup.location, POINT(0.0, 0.0)) <= 7000 m OR + | distance(withdrawals.location, POINT(0.0, 0.0)) <= 7000 m | ) | ) |GROUP BY @@ -2836,7 +2836,53 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { query shouldBe """{ | "query": { - | "match_all": {} + | "bool": { + | "filter": [ + | { + | "geo_distance": { + | "distance": "5000km", + | "toLocation": [ + | 40.0, + | -70.0 + | ] + | } + | }, + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | } + | }, + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); def arg1 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | } + | } + | }, + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2) < 1000000.0", + | "params": { + | "lat1": -70.0, + | "lon1": 40.0, + | "lat2": 0.0, + | "lon2": 0.0 + | } + | } + | } + | } + | ] + | } | }, | "script_fields": { | "d1": { @@ -2897,7 +2943,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\+", " + ") .replaceAll("\\*", " * ") .replaceAll("/", " / ") - .replaceAll(">", " > ") + .replaceAll(">(\\d)", " > $1") + .replaceAll("=(\\d)", "= $1") + .replaceAll(">=", " >=") .replaceAll("<", " < ") .replaceAll("!=", " != ") .replaceAll("&&", " && ") @@ -2905,6 +2953,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") .replaceAll(",params", ", params") .replaceAll("GeoPoint", " GeoPoint") + .replaceAll("lat,arg", "lat, arg") } } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index f8308229..559530af 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -4,7 +4,7 @@ import app.softnetwork.elastic.sql.query.{ ElasticBoolQuery, ElasticChild, ElasticFilter, - ElasticGeoDistance, + DistanceCriteria, ElasticMatch, ElasticNested, ElasticParent, @@ -67,7 +67,7 @@ case class ElasticQuery(filter: ElasticFilter) { case isNotNull: IsNotNullExpr => isNotNull case in: InExpr[_, _] => in case between: BetweenExpr[_] => between - case geoDistance: ElasticGeoDistance => geoDistance + case geoDistance: DistanceCriteria => geoDistance case matchExpression: ElasticMatch => matchExpression case isNull: IsNullCriteria => isNull case isNotNull: IsNotNullCriteria => isNotNull diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index a1997cad..a43f54b1 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -8,6 +8,7 @@ import app.softnetwork.elastic.sql.query._ import com.sksamuel.elastic4s.ElasticApi import com.sksamuel.elastic4s.ElasticApi._ import com.sksamuel.elastic4s.requests.script.Script +import com.sksamuel.elastic4s.requests.script.ScriptType.Source import com.sksamuel.elastic4s.requests.searches.aggs.Aggregation import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.requests.searches.sort.FieldSort @@ -411,10 +412,28 @@ package object bridge { } implicit def geoDistanceToQuery( - geoDistance: ElasticGeoDistance - ): Query = { - import geoDistance._ - geoDistanceQuery(identifier.name, point.lat.value, point.lon.value) distance distance.value + distanceCriteria: DistanceCriteria + ): Query = { + import distanceCriteria._ + operator match { + case LE | LT if distance.oneIdentifier => + val identifier = distance.identifiers.head + val point = distance.points.head + geoDistanceQuery( + identifier.name, + point.lat.value, + point.lon.value + ) distance geoDistance + case _ => + scriptQuery( + Script( + script = distanceCriteria.painless, + lang = Some("painless"), + scriptType = Source, + params = distance.params + ) + ) + } } implicit def matchToQuery( diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 1ecebaa9..dfaf910d 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -632,8 +632,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | blockedCustomers not like "%uuid%" AND | NOT receiptOfOrdersDisabled=true AND | ( - | distance(pickup.location, POINT(0.0, 0.0)) <= "7000m" OR - | distance(withdrawals.location, POINT(0.0, 0.0)) <= "7000m" + | distance(pickup.location, POINT(0.0, 0.0)) <= 7000 m OR + | distance(withdrawals.location, POINT(0.0, 0.0)) <= 7000 m | ) | ) |GROUP BY @@ -2817,7 +2817,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") } - it should "handle geo distance as script field" in { val select: ElasticSearchRequest = SQLQuery(geoDistance) @@ -2826,7 +2825,53 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { query shouldBe """{ | "query": { - | "match_all": {} + | "bool": { + | "filter": [ + | { + | "geo_distance": { + | "distance": "5000km", + | "toLocation": [ + | 40.0, + | -70.0 + | ] + | } + | }, + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | } + | }, + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('fromLocation') || doc['fromLocation'].empty ? null : doc['fromLocation']); def arg1 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | } + | } + | }, + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2) < 1000000.0", + | "params": { + | "lat1": -70.0, + | "lon1": 40.0, + | "lat2": 0.0, + | "lon2": 0.0 + | } + | } + | } + | } + | ] + | } | }, | "script_fields": { | "d1": { @@ -2887,7 +2932,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\+", " + ") .replaceAll("\\*", " * ") .replaceAll("/", " / ") - .replaceAll(">", " > ") + .replaceAll(">(\\d)", " > $1") + .replaceAll("=(\\d)", "= $1") + .replaceAll(">=", " >=") .replaceAll("<", " < ") .replaceAll("!=", " != ") .replaceAll("&&", " && ") @@ -2895,6 +2942,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") .replaceAll(",params", ", params") .replaceAll("GeoPoint", " GeoPoint") + .replaceAll("lat,arg", "lat, arg") } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index ca50abac..d3849326 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -1,16 +1,19 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLDouble, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLDouble, SQLType, SQLTypes} import app.softnetwork.elastic.sql.{ DoubleValue, Expr, Identifier, + LongValue, PainlessParams, PainlessScript, Token, - TokenRegex + TokenRegex, + Updateable } import app.softnetwork.elastic.sql.operator.Operator +import app.softnetwork.elastic.sql.query.SQLSearchRequest package object geo { @@ -20,6 +23,44 @@ package object geo { override def sql: String = s"POINT($lat, $lon)" } + sealed trait DistanceUnit extends TokenRegex + + object DistanceUnit { + def convertToMeters(value: Double, unit: DistanceUnit): Double = + unit match { + case Kilometers => value * 1000.0 + case Meters => value + case Centimeters => value / 100.0 + case Millimeters => value / 1000.0 + case Miles => value * 1609.34 + case Yards => value * 0.9144 + case Feet => value * 0.3048 + case Inches => value * 0.0254 + case NauticalMiles => value * 1852.0 + } + } + + sealed trait MetricUnit extends DistanceUnit + + case object Kilometers extends Expr("km") with MetricUnit + case object Meters extends Expr("m") with MetricUnit + case object Centimeters extends Expr("cm") with MetricUnit + case object Millimeters extends Expr("mm") with MetricUnit + + sealed trait ImperialUnit extends DistanceUnit + case object Miles extends Expr("mi") with ImperialUnit + case object Yards extends Expr("yd") with ImperialUnit + case object Feet extends Expr("ft") with ImperialUnit + case object Inches extends Expr("in") with ImperialUnit + + case object NauticalMiles extends Expr("nmi") with DistanceUnit + + case class GeoDistance(value: LongValue, unit: DistanceUnit) extends PainlessScript { + override def baseType: SQLType = SQLTypes.BigInt + override def sql: String = s"$value$unit" + override def painless: String = s"${DistanceUnit.convertToMeters(value.value, unit)}" + } + case object Distance extends Expr("ST_DISTANCE") with Function with Operator { override def words: List[String] = List(sql, "DISTANCE") @@ -28,7 +69,13 @@ package object geo { case class Distance(from: Either[Identifier, Point], to: Either[Identifier, Point]) extends FunctionN[SQLAny, SQLDouble] - with PainlessParams { + with PainlessParams + with Updateable { + + override def update(request: SQLSearchRequest): Distance = this.copy( + from = from.fold(id => Left(id.update(request)), p => Right(p)), + to = to.fold(id => Left(id.update(request)), p => Right(p)) + ) override def fun: Option[PainlessScript] = Some(Distance) @@ -46,7 +93,9 @@ package object geo { (fromId, toId) } - private[this] lazy val identifiers: List[Identifier] = List(fromId, toId).flatten + lazy val identifiers: List[Identifier] = List(fromId, toId).flatten + + lazy val oneIdentifier: Boolean = identifiers.size == 1 private[this] lazy val (fromPoint, toPoint) = { val fromPoint = from.fold(_ => None, Some(_)) @@ -54,9 +103,7 @@ package object geo { (fromPoint, toPoint) } - private[this] lazy val points: List[Point] = List(fromPoint, toPoint).flatten - - private[this] lazy val oneIdentifier: Boolean = identifiers.size == 1 + lazy val points: List[Point] = List(fromPoint, toPoint).flatten override def nullable: Boolean = from.fold(identity, identity).nullable || to.fold(identity, identity).nullable diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index 00ea32c3..eb970bc1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.sql.function.geo.Distance +import app.softnetwork.elastic.sql.function.geo.Meters import app.softnetwork.elastic.sql.{ DoubleFromTo, DoubleValues, @@ -41,8 +41,8 @@ import app.softnetwork.elastic.sql.query.{ BetweenExpr, ConditionalFunctionAsCriteria, Criteria, + DistanceCriteria, ElasticChild, - ElasticGeoDistance, ElasticNested, ElasticParent, ElasticRelation, @@ -159,9 +159,9 @@ trait WhereParser { case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) } - def sql_distance: PackratParser[Criteria] = - Distance.regex ~ start ~ identifier ~ separator ~ point ~ end ~ le ~ literal ^^ { - case _ ~ _ ~ i ~ _ ~ p ~ _ ~ _ ~ d => ElasticGeoDistance(i, d, p) + def distanceCriteria: PackratParser[Criteria] = + distance ~ (ge | gt | le | lt) ~ long ~ distance_unit.? ^^ { case d ~ o ~ v ~ u => + DistanceCriteria(d, o, v, u.getOrElse(Meters)) } def matchCriteria: PackratParser[MatchCriteria] = @@ -184,7 +184,7 @@ trait WhereParser { } def criteria: PackratParser[Criteria] = - (equality | like | rlike | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ sql_distance | matchCriteria | logical_criteria) ^^ ( + (equality | like | rlike | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ distanceCriteria | matchCriteria | logical_criteria) ^^ ( c => c ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala index 5d5295ba..b7f1a0e3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.sql.parser.function import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.function.Function -import app.softnetwork.elastic.sql.function.geo.{Distance, Point} +import app.softnetwork.elastic.sql.function.geo._ import app.softnetwork.elastic.sql.parser.Parser package object geo { @@ -20,11 +20,24 @@ package object geo { case p: Point => Right(p) } - def distance: PackratParser[Function] = + def distance: PackratParser[Distance] = Distance.regex ~> start ~> pointOrIdentifier ~ separator ~ pointOrIdentifier <~ end ^^ { case from ~ _ ~ to => Distance(from, to) } + def kilometers: PackratParser[DistanceUnit] = Kilometers.regex ^^ (_ => Kilometers) + def meters: PackratParser[DistanceUnit] = Meters.regex ^^ (_ => Meters) + def centimeters: PackratParser[DistanceUnit] = Centimeters.regex ^^ (_ => Centimeters) + def millimeters: PackratParser[DistanceUnit] = Millimeters.regex ^^ (_ => Millimeters) + def miles: PackratParser[DistanceUnit] = Miles.regex ^^ (_ => Miles) + def yards: PackratParser[DistanceUnit] = Yards.regex ^^ (_ => Yards) + def feet: PackratParser[DistanceUnit] = Feet.regex ^^ (_ => Feet) + def inches: PackratParser[DistanceUnit] = Inches.regex ^^ (_ => Inches) + def nauticalMiles: PackratParser[DistanceUnit] = NauticalMiles.regex ^^ (_ => NauticalMiles) + + def distance_unit: PackratParser[DistanceUnit] = + kilometers | meters | centimeters | millimeters | miles | yards | feet | inches | nauticalMiles + def distance_identifier: PackratParser[Identifier] = distance ^^ functionAsIdentifier } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 837ac4b2..d86d1b51 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.function.cond.{ConditionalFunction, IsNotNull, IsNull} -import app.softnetwork.elastic.sql.function.geo.{Distance, Point} +import app.softnetwork.elastic.sql.function.geo.{Distance, DistanceUnit, GeoDistance, Point} import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql._ @@ -209,10 +209,8 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def painlessValue: String = maybeValue .map { - case v: Value[_] => v.painless - case v: Values[_, _] => v.painless - case v: Identifier => v.painless - case v => v.sql + case v: PainlessScript => v.painless + case v => v.sql } .getOrElse("") /*{ operator match { @@ -223,13 +221,8 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { protected lazy val left: String = { val targetedType = maybeValue match { - case Some(v) => - v match { - case value: Value[_] => value.out - case values: Values[_, _] => values.out - case other => other.out - } - case None => identifier.out + case Some(v) => v.out + case None => identifier.out } SQLTypeUtils.coerce(identifier, targetedType) } @@ -440,19 +433,23 @@ case class BetweenExpr[+T]( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this } -case class ElasticGeoDistance( - identifier: Identifier, - distance: StringValue, - point: Point +case class DistanceCriteria( + distance: Distance, + operator: ComparisonOperator, + distanceValue: LongValue, + distanceUnit: DistanceUnit ) extends Expression { - override def sql = - s"$Distance($identifier, POINT(${point.lat}, ${point.lon})) $operator $distance" - override val functions: List[Function] = List(Distance) - override def operator: Operator = LE - override def update(request: SQLSearchRequest): ElasticGeoDistance = - this.copy(identifier = identifier.update(request)) - override def maybeValue: Option[Token] = Some(distance) + def geoDistance: String = s"$distanceValue$distanceUnit" + + override def identifier: Identifier = Identifier(distance) + + override def sql = s"$distance $operator $distanceValue $distanceUnit" + + override def update(request: SQLSearchRequest): DistanceCriteria = + this.copy(distance = distance.update(request)) + + override def maybeValue: Option[Token] = Some(GeoDistance(distanceValue, distanceUnit)) override def maybeNot: Option[NOT.type] = None diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 11bcb377..6d4d4193 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -61,7 +61,7 @@ object Queries { val isNull = "SELECT * FROM Table WHERE identifier is null" val isNotNull = "SELECT * FROM Table WHERE identifier is NOT null" val geoDistanceCriteria = - "SELECT * FROM Table WHERE ST_DISTANCE(profile.location, POINT(-70.0, 40.0)) <= '5km'" + "SELECT * FROM Table WHERE ST_DISTANCE(profile.location, POINT(-70.0, 40.0)) <= 5 km" val except = "SELECT * except(col1,col2) FROM Table" val matchCriteria = "SELECT * FROM Table WHERE match (identifier1,identifier2,identifier3) against ('value')" @@ -175,7 +175,7 @@ object Queries { "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" val geoDistance = - "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table" + "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) < 5000 km AND ST_DISTANCE(POINT(-70.0, 40.0), toLocation) >= 4000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) < 1000 km" } From 8f900636b73e2fbf6c5dc9c2a3d6eaad78820c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 09:53:13 +0200 Subject: [PATCH 30/48] add support for geo distance with between operator --- .../elastic/sql/bridge/ElasticQuery.scala | 2 +- .../elastic/sql/bridge/package.scala | 75 ++++++++++++++++++- .../elastic/sql/SQLQuerySpec.scala | 42 ++++++----- .../elastic/sql/bridge/ElasticQuery.scala | 2 +- .../elastic/sql/bridge/package.scala | 75 ++++++++++++++++++- .../elastic/sql/SQLQuerySpec.scala | 42 ++++++----- .../elastic/sql/function/geo/package.scala | 9 +-- .../app/softnetwork/elastic/sql/package.scala | 30 +++++++- .../elastic/sql/parser/Parser.scala | 3 +- .../elastic/sql/parser/SelectParser.scala | 2 +- .../elastic/sql/parser/WhereParser.scala | 50 +++++++++++-- .../sql/parser/function/geo/package.scala | 6 +- .../softnetwork/elastic/sql/query/Where.scala | 29 +++---- .../elastic/sql/SQLParserSpec.scala | 2 +- 14 files changed, 291 insertions(+), 78 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index b773c5b5..bfba52f6 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -67,7 +67,7 @@ case class ElasticQuery(filter: ElasticFilter) { case isNotNull: IsNotNullExpr => isNotNull case in: InExpr[_, _] => in case between: BetweenExpr[_] => between - case geoDistance: DistanceCriteria => geoDistance + // case geoDistance: DistanceCriteria => geoDistance case matchExpression: ElasticMatch => matchExpression case isNull: IsNullCriteria => isNull case isNotNull: IsNotNullCriteria => isNotNull diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 148824a9..073158ae 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} import app.softnetwork.elastic.sql.function.aggregate.COUNT +import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ import com.sksamuel.elastic4s.ElasticApi @@ -157,9 +158,50 @@ package object bridge { import expression._ if (aggregation) return matchAllQuery() - if (identifier.functions.nonEmpty) { + if ( + identifier.functions.nonEmpty && (identifier.functions.size > 1 || (identifier.functions.head match { + case _: Distance => false + case _ => true + })) + ) { return scriptQuery(Script(script = painless).lang("painless").scriptType("source")) } + // Geo distance special case + identifier.functions.headOption match { + case Some(d: Distance) => + operator match { + case o: ComparisonOperator => + (value match { + case l: LongValue => + Some(GeoDistance(l, Meters)) + case g: GeoDistance => + Some(g) + case _ => None + }) match { + case Some(g) => + maybeNot match { + case Some(_) => + return geoDistanceToQuery( + DistanceCriteria( + d, + o.not, + g + ) + ) + case _ => + return geoDistanceToQuery( + DistanceCriteria( + d, + o, + g + ) + ) + } + case _ => + } + } + case _ => + } value match { case n: NumericValue[_] => operator match { @@ -396,6 +438,35 @@ package object bridge { between: BetweenExpr[_] ): Query = { import between._ + // Geo distance special case + identifier.functions.headOption match { + case Some(d: Distance) => + fromTo match { + case ft: GeoDistanceFromTo => + val fq = + geoDistanceToQuery( + DistanceCriteria( + d, + GE, + ft.from + ) + ) + val tq = + geoDistanceToQuery( + DistanceCriteria( + d, + LE, + ft.to + ) + ) + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + case _ => + } + case _ => + } val r = out match { case _: SQLDouble => @@ -427,7 +498,7 @@ package object bridge { identifier.name, point.lat.value, point.lon.value - ) distance geoDistance + ) distance geoDistance.geoDistance case _ => scriptQuery( Script( diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 6f347709..c38dbcd1 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2828,7 +2828,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") } - it should "handle geo distance as script field" in { + it should "handle geo distance as script fields and criteria" in { val select: ElasticSearchRequest = SQLQuery(geoDistance) val query = select.query @@ -2839,24 +2839,30 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "bool": { | "filter": [ | { - | "geo_distance": { - | "distance": "5000km", - | "toLocation": [ - | 40.0, - | -70.0 - | ] - | } - | }, - | { - | "script": { - | "script": { - | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", - | "params": { - | "lat": -70.0, - | "lon": 40.0 + | "bool": { + | "must": [ + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | } + | }, + | { + | "geo_distance": { + | "distance": "5000km", + | "toLocation": [ + | 40.0, + | -70.0 + | ] + | } | } - | } + | ] | } | }, | { diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 559530af..7ac0a6e3 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -67,7 +67,7 @@ case class ElasticQuery(filter: ElasticFilter) { case isNotNull: IsNotNullExpr => isNotNull case in: InExpr[_, _] => in case between: BetweenExpr[_] => between - case geoDistance: DistanceCriteria => geoDistance + // case geoDistance: DistanceCriteria => geoDistance case matchExpression: ElasticMatch => matchExpression case isNull: IsNullCriteria => isNull case isNotNull: IsNotNullCriteria => isNotNull diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index a43f54b1..628224c4 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} import app.softnetwork.elastic.sql.function.aggregate.COUNT +import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.query._ @@ -157,9 +158,50 @@ package object bridge { import expression._ if (aggregation) return matchAllQuery() - if (identifier.functions.nonEmpty) { + if ( + identifier.functions.nonEmpty && (identifier.functions.size > 1 || (identifier.functions.head match { + case _: Distance => false + case _ => true + })) + ) { return scriptQuery(Script(script = painless).lang("painless").scriptType("source")) } + // Geo distance special case + identifier.functions.headOption match { + case Some(d: Distance) => + operator match { + case o: ComparisonOperator => + (value match { + case l: LongValue => + Some(GeoDistance(l, Meters)) + case g: GeoDistance => + Some(g) + case _ => None + }) match { + case Some(g) => + maybeNot match { + case Some(_) => + return geoDistanceToQuery( + DistanceCriteria( + d, + o.not, + g + ) + ) + case _ => + return geoDistanceToQuery( + DistanceCriteria( + d, + o, + g + ) + ) + } + case _ => + } + } + case _ => + } value match { case n: NumericValue[_] => operator match { @@ -396,6 +438,35 @@ package object bridge { between: BetweenExpr[_] ): Query = { import between._ + // Geo distance special case + identifier.functions.headOption match { + case Some(d: Distance) => + fromTo match { + case ft: GeoDistanceFromTo => + val fq = + geoDistanceToQuery( + DistanceCriteria( + d, + GE, + ft.from + ) + ) + val tq = + geoDistanceToQuery( + DistanceCriteria( + d, + LE, + ft.to + ) + ) + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + case _ => + } + case _ => + } val r = out match { case _: SQLDouble => @@ -423,7 +494,7 @@ package object bridge { identifier.name, point.lat.value, point.lon.value - ) distance geoDistance + ) distance geoDistance.geoDistance case _ => scriptQuery( Script( diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index dfaf910d..10311d0d 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2817,7 +2817,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("(\\d)=", "$1 = ") } - it should "handle geo distance as script field" in { + it should "handle geo distance as script fields and criteria" in { val select: ElasticSearchRequest = SQLQuery(geoDistance) val query = select.query @@ -2828,24 +2828,30 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "bool": { | "filter": [ | { - | "geo_distance": { - | "distance": "5000km", - | "toLocation": [ - | 40.0, - | -70.0 - | ] - | } - | }, - | { - | "script": { - | "script": { - | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", - | "params": { - | "lat": -70.0, - | "lon": 40.0 + | "bool": { + | "must": [ + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('toLocation') || doc['toLocation'].empty ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "params": { + | "lat": -70.0, + | "lon": 40.0 + | } + | } + | } + | }, + | { + | "geo_distance": { + | "distance": "5000km", + | "toLocation": [ + | 40.0, + | -70.0 + | ] + | } | } - | } + | ] | } | }, | { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index d3849326..a451a184 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -1,11 +1,10 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLDouble, SQLType, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLDouble, SQLTypes} import app.softnetwork.elastic.sql.{ DoubleValue, Expr, Identifier, - LongValue, PainlessParams, PainlessScript, Token, @@ -55,12 +54,6 @@ package object geo { case object NauticalMiles extends Expr("nmi") with DistanceUnit - case class GeoDistance(value: LongValue, unit: DistanceUnit) extends PainlessScript { - override def baseType: SQLType = SQLTypes.BigInt - override def sql: String = s"$value$unit" - override def painless: String = s"${DistanceUnit.convertToMeters(value.value, unit)}" - } - case object Distance extends Expr("ST_DISTANCE") with Function with Operator { override def words: List[String] = List(sql, "DISTANCE") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index db55104f..94f90791 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1,8 +1,9 @@ package app.softnetwork.elastic import app.softnetwork.elastic.sql.function.aggregate.{MAX, MIN} +import app.softnetwork.elastic.sql.function.geo.DistanceUnit import app.softnetwork.elastic.sql.operator._ -import app.softnetwork.elastic.sql.parser.Validation +import app.softnetwork.elastic.sql.parser.{Validation, Validator} import app.softnetwork.elastic.sql.query._ import java.security.MessageDigest @@ -218,8 +219,23 @@ package object sql { override def baseType: SQLNumeric = SQLTypes.Double } + case class GeoDistance(longValue: LongValue, unit: DistanceUnit) + extends NumericValue[Double](DistanceUnit.convertToMeters(longValue.value, unit)) + with PainlessScript { + override def baseType: SQLNumeric = SQLTypes.Double + override def sql: String = s"$longValue $unit" + def geoDistance: String = s"$longValue$unit" + override def painless: String = s"$value" + } + sealed abstract class FromTo[+T](val from: Value[T], val to: Value[T]) extends Token { - override def sql = s"${from.sql} and ${to.sql}" + override def sql = s"${from.sql} AND ${to.sql}" + + override def baseType: SQLType = + SQLTypeUtils.leastCommonSuperType(List(from.baseType, to.baseType)) + + override def validate(): Either[String, Unit] = + Validator.validateTypesMatching(from.out, to.out) } case class LiteralFromTo(override val from: StringValue, override val to: StringValue) @@ -252,6 +268,16 @@ package object sql { } } + case class GeoDistanceFromTo(override val from: GeoDistance, override val to: GeoDistance) + extends FromTo[Double](from, to) { + def between: Seq[Double] => Boolean = { + _.exists { n => n >= from.value && n <= to.value } + } + def notBetween: Seq[Double] => Boolean = { + _.forall { n => n < from.value || n > to.value } + } + } + sealed abstract class Values[+R: TypeTag, +T <: Value[R]](val values: Seq[T]) extends Token with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 6bf2b7f0..3b552f40 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -253,7 +253,8 @@ trait Parser stringFunctionWithIdentifier | date_diff_identifier | extract_identifier | - case_when_identifier) >> castOperator + case_when_identifier | + distance_identifier) >> castOperator def identifierWithFunction: PackratParser[Identifier] = (rep1sep( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala index 273c7608..6746f1ee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/SelectParser.scala @@ -6,7 +6,7 @@ trait SelectParser { self: Parser with WhereParser => def field: PackratParser[Field] = - (distance_identifier >> castOperator | identifierWithTopHits | + (identifierWithTopHits | identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index eb970bc1..c79c5668 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -4,9 +4,12 @@ import app.softnetwork.elastic.sql.function.geo.Meters import app.softnetwork.elastic.sql.{ DoubleFromTo, DoubleValues, + GeoDistance, + GeoDistanceFromTo, Identifier, LiteralFromTo, LongFromTo, + LongValue, LongValues, StringValues, Token @@ -81,7 +84,7 @@ trait WhereParser { identifier private def equality: PackratParser[GenericExpression] = - not.? ~ any_identifier ~ (eq | ne | diff) ~ (boolean | literal | double | pi | long | any_identifier) ^^ { + not.? ~ any_identifier ~ (eq | ne | diff) ~ (boolean | literal | double | pi | geo_distance | long | any_identifier) ^^ { case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) } @@ -104,7 +107,7 @@ trait WhereParser { def lt: PackratParser[ComparisonOperator] = LT.sql ^^ (_ => LT) private def comparison: PackratParser[GenericExpression] = - not.? ~ any_identifier ~ (ge | gt | le | lt) ~ (double | pi | long | literal | any_identifier) ^^ { + not.? ~ any_identifier ~ (ge | gt | le | lt) ~ (double | pi | geo_distance | long | literal | any_identifier) ^^ { case n ~ i ~ o ~ v => GenericExpression(i, o, v, n) } @@ -159,11 +162,30 @@ trait WhereParser { case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) } - def distanceCriteria: PackratParser[Criteria] = - distance ~ (ge | gt | le | lt) ~ long ~ distance_unit.? ^^ { case d ~ o ~ v ~ u => - DistanceCriteria(d, o, v, u.getOrElse(Meters)) + def betweenDistances: PackratParser[Criteria] = + distance_identifier ~ not.? ~ BETWEEN.regex ~ (geo_distance | long) ~ and ~ (geo_distance | long) ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => + BetweenExpr( + i, + GeoDistanceFromTo( + from match { + case gd: GeoDistance => gd + case l: LongValue => GeoDistance(l, Meters) + }, + to match { + case gd: GeoDistance => gd + case l: LongValue => GeoDistance(l, Meters) + } + ), + n + ) } + /*def distanceCriteria: PackratParser[Criteria] = + distance ~ (ge | gt | le | lt) ~ geo_distance ^^ { case d ~ o ~ g => + DistanceCriteria(d, o, g) + }*/ + def matchCriteria: PackratParser[MatchCriteria] = MATCH.regex ~ start ~ rep1sep( any_identifier, @@ -184,9 +206,21 @@ trait WhereParser { } def criteria: PackratParser[Criteria] = - (equality | like | rlike | comparison | inLiteral | inLongs | inDoubles | between | betweenLongs | betweenDoubles | isNotNull | isNull | /*coalesce | nullif |*/ distanceCriteria | matchCriteria | logical_criteria) ^^ ( - c => c - ) + (equality | + like | + rlike | + comparison | + inLiteral | + inLongs | + inDoubles | + between | + betweenDistances | + betweenLongs | + betweenDoubles | + isNotNull | + isNull | /*coalesce | nullif | distanceCriteria | */ + matchCriteria | + logical_criteria) ^^ (c => c) def predicate: PackratParser[Predicate] = criteria ~ (and | or) ~ not.? ~ criteria ^^ { case l ~ o ~ n ~ r => Predicate(l, o, r, n) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala index b7f1a0e3..7736fc4c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala @@ -1,7 +1,6 @@ package app.softnetwork.elastic.sql.parser.function -import app.softnetwork.elastic.sql.Identifier -import app.softnetwork.elastic.sql.function.Function +import app.softnetwork.elastic.sql.{GeoDistance, Identifier} import app.softnetwork.elastic.sql.function.geo._ import app.softnetwork.elastic.sql.parser.Parser @@ -38,6 +37,9 @@ package object geo { def distance_unit: PackratParser[DistanceUnit] = kilometers | meters | centimeters | millimeters | miles | yards | feet | inches | nauticalMiles + def geo_distance: PackratParser[GeoDistance] = + long ~ distance_unit ^^ { case value ~ unit => GeoDistance(value, unit) } + def distance_identifier: PackratParser[Identifier] = distance ^^ functionAsIdentifier } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index d86d1b51..a8c6b6dd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.`type`.{SQLAny, SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.function.cond.{ConditionalFunction, IsNotNull, IsNull} -import app.softnetwork.elastic.sql.function.geo.{Distance, DistanceUnit, GeoDistance, Point} +import app.softnetwork.elastic.sql.function.geo.Distance import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql._ @@ -413,12 +413,7 @@ case class BetweenExpr[+T]( fromTo: FromTo[T], maybeNot: Option[NOT.type] ) extends Expression { - private[this] lazy val id = functions.headOption match { - case Some(f) => s"$f($identifier)" - case _ => s"$identifier" - } - override def sql = - s"$id $notAsString$operator $fromTo" + override def sql = s"$identifier $notAsString$operator $fromTo" override def operator: Operator = BETWEEN override def update(request: SQLSearchRequest): Criteria = { val updated = this.copy(identifier = identifier.update(request)) @@ -431,25 +426,33 @@ case class BetweenExpr[+T]( override def maybeValue: Option[Token] = Some(fromTo) override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this + + override def validate(): Either[String, Unit] = + Validator.validateTypesMatching(identifier.out, fromTo.out) + + override def painless: String = { + if (identifier.nullable) { + return s"def left = $left; left == null ? false : $painlessNot(${fromTo.from} <= left <= ${fromTo.to})" + } + s"$painlessNot(${fromTo.from} <= $left <= ${fromTo.to})" + } + } case class DistanceCriteria( distance: Distance, operator: ComparisonOperator, - distanceValue: LongValue, - distanceUnit: DistanceUnit + geoDistance: GeoDistance ) extends Expression { - def geoDistance: String = s"$distanceValue$distanceUnit" - override def identifier: Identifier = Identifier(distance) - override def sql = s"$distance $operator $distanceValue $distanceUnit" + override def sql = s"$distance $operator $geoDistance" override def update(request: SQLSearchRequest): DistanceCriteria = this.copy(distance = distance.update(request)) - override def maybeValue: Option[Token] = Some(GeoDistance(distanceValue, distanceUnit)) + override def maybeValue: Option[Token] = Some(geoDistance) override def maybeNot: Option[NOT.type] = None diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 6d4d4193..e7eb9fd9 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -175,7 +175,7 @@ object Queries { "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" val geoDistance = - "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) < 5000 km AND ST_DISTANCE(POINT(-70.0, 40.0), toLocation) >= 4000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) < 1000 km" + "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) < 1000 km" } From 00cac3797bcc9a79e926f2554a3424808a6fb812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 10:11:49 +0200 Subject: [PATCH 31/48] fix validation error message for expression, update validation for between expression --- .../app/softnetwork/elastic/sql/query/Where.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index a8c6b6dd..97722750 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -251,7 +251,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { Validator.validateTypesMatching(identifier.out, v.out) match { case Left(_) => Left( - s"Type mismatch: '${out.typeId}' is not compatible with '${v.out.typeId}' in expression: $this" + s"Type mismatch: '${identifier.out.typeId}' is not compatible with '${v.out.typeId}' in expression: $this" ) case Right(_) => Right(()) } @@ -427,8 +427,13 @@ case class BetweenExpr[+T]( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - override def validate(): Either[String, Unit] = - Validator.validateTypesMatching(identifier.out, fromTo.out) + override def validate(): Either[String, Unit] = { + for { + _ <- identifier.validate() + _ <- fromTo.validate() + _ <- Validator.validateTypesMatching(identifier.out, fromTo.from.out) + } yield () + } override def painless: String = { if (identifier.nullable) { From 8bca4a1d7ce0bc41d083899532da96831bae7594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 11:06:06 +0200 Subject: [PATCH 32/48] add haversine support for distance with 2 geo points --- .../elastic/sql/SQLQuerySpec.scala | 16 ++-------- .../elastic/sql/SQLQuerySpec.scala | 16 ++-------- .../elastic/sql/function/geo/package.scala | 29 ++++++++++++++----- .../elastic/sql/SQLParserSpec.scala | 2 +- 4 files changed, 26 insertions(+), 37 deletions(-) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index c38dbcd1..6dd17883 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2877,13 +2877,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2) < 1000000.0", - | "params": { - | "lat1": -70.0, - | "lon1": 40.0, - | "lat2": 0.0, - | "lon2": 0.0 - | } + | "source": "0.0 < 1000000.0" | } | } | } @@ -2914,13 +2908,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d3": { | "script": { | "lang": "painless", - | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2)", - | "params": { - | "lat1": -70.0, - | "lon1": 40.0, - | "lat2": 0.0, - | "lon2": 0.0 - | } + | "source": "8318612.0" | } | } | }, diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 10311d0d..1572497d 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2866,13 +2866,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2) < 1000000.0", - | "params": { - | "lat1": -70.0, - | "lon1": 40.0, - | "lat2": 0.0, - | "lon2": 0.0 - | } + | "source": "0.0 < 1000000.0" | } | } | } @@ -2903,13 +2897,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d3": { | "script": { | "lang": "painless", - | "source": "new GeoPoint(params.lat1, params.lon1).arcDistance(params.lat2, params.lon2)", - | "params": { - | "lat1": -70.0, - | "lon1": 40.0, - | "lat2": 0.0, - | "lon2": 0.0 - | } + | "source": "8318612.0" | } | } | }, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index a451a184..5d8c8201 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -107,16 +107,24 @@ package object geo { "lat" -> points.head.lat.value, "lon" -> points.head.lon.value ) - else if (identifiers.isEmpty) - Map( - "lat1" -> fromPoint.get.lat.value, - "lon1" -> fromPoint.get.lon.value, - "lat2" -> toPoint.get.lat.value, - "lon2" -> toPoint.get.lon.value - ) else Map.empty + def haversine(lat1: Double, lon1: Double, lat2: Double, lon2: Double): Double = { + val R = 6371e3 // Radius of the earth in meters + val r1 = lat1.toRadians + val r2 = lat2.toRadians + val rlat = (lat2 - lat1).toRadians + val rlon = (lon2 - lon1).toRadians + + val a = Math.sin(rlat / 2) * Math.sin(rlat / 2) + + Math.cos(r1) * Math.cos(r2) * + Math.sin(rlon / 2) * Math.sin(rlon / 2) + val c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)) + + (R * c).round.toDouble // in meters + } + override def painless: String = { val nullCheck = identifiers.zipWithIndex @@ -135,7 +143,12 @@ package object geo { if (oneIdentifier) { s"arg0${fun.map(_.painless).getOrElse("")}(params.lat, params.lon)" } else if (identifiers.isEmpty) { - s"new GeoPoint(params.lat1, params.lon1)${fun.map(_.painless).getOrElse("")}(params.lat2, params.lon2)" + s"${haversine( + fromPoint.get.lat.value, + fromPoint.get.lon.value, + toPoint.get.lat.value, + toPoint.get.lon.value + )}" } else { s"arg0${fun.map(_.painless).getOrElse("")}(arg1.lat, arg1.lon)" } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index e7eb9fd9..bf125f84 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -175,7 +175,7 @@ object Queries { "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" val geoDistance = - "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) < 1000 km" + "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(-70.0, 40.0)) < 1000 km" } From 2210ffe042970fda881140a2ec62952b2e2b1b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 11:09:54 +0200 Subject: [PATCH 33/48] move haversine to Distance companion object --- .../elastic/sql/function/geo/package.scala | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 5d8c8201..3cff8405 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -58,6 +58,21 @@ package object geo { override def words: List[String] = List(sql, "DISTANCE") override def painless: String = ".arcDistance" + + def haversine(lat1: Double, lon1: Double, lat2: Double, lon2: Double): Double = { + val R = 6371e3 // Radius of the earth in meters + val r1 = lat1.toRadians + val r2 = lat2.toRadians + val rlat = (lat2 - lat1).toRadians + val rlon = (lon2 - lon1).toRadians + + val a = Math.sin(rlat / 2) * Math.sin(rlat / 2) + + Math.cos(r1) * Math.cos(r2) * + Math.sin(rlon / 2) * Math.sin(rlon / 2) + val c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)) + + (R * c).round.toDouble // in meters + } } case class Distance(from: Either[Identifier, Point], to: Either[Identifier, Point]) @@ -110,21 +125,6 @@ package object geo { else Map.empty - def haversine(lat1: Double, lon1: Double, lat2: Double, lon2: Double): Double = { - val R = 6371e3 // Radius of the earth in meters - val r1 = lat1.toRadians - val r2 = lat2.toRadians - val rlat = (lat2 - lat1).toRadians - val rlon = (lon2 - lon1).toRadians - - val a = Math.sin(rlat / 2) * Math.sin(rlat / 2) + - Math.cos(r1) * Math.cos(r2) * - Math.sin(rlon / 2) * Math.sin(rlon / 2) - val c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)) - - (R * c).round.toDouble // in meters - } - override def painless: String = { val nullCheck = identifiers.zipWithIndex @@ -143,7 +143,7 @@ package object geo { if (oneIdentifier) { s"arg0${fun.map(_.painless).getOrElse("")}(params.lat, params.lon)" } else if (identifiers.isEmpty) { - s"${haversine( + s"${Distance.haversine( fromPoint.get.lat.value, fromPoint.get.lon.value, toPoint.get.lat.value, From 0636e34205351b1251d8218d020eec470ffff8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 11:15:02 +0200 Subject: [PATCH 34/48] init sql engine documentation --- documentation/README.md | 15 ++ documentation/functions_aggregate.md | 106 +++++++++ documentation/functions_conditional.md | 95 ++++++++ documentation/functions_date_time.md | 244 +++++++++++++++++++++ documentation/functions_geo.md | 26 +++ documentation/functions_math.md | 119 ++++++++++ documentation/functions_string.md | 95 ++++++++ documentation/functions_system.md | 26 +++ documentation/functions_type_conversion.md | 27 +++ documentation/keywords.md | 20 ++ documentation/operator_precedence.md | 24 ++ documentation/operators.md | 155 +++++++++++++ documentation/request_structure.md | 104 +++++++++ documentation/type_conversion.md | 92 ++++++++ 14 files changed, 1148 insertions(+) create mode 100644 documentation/README.md create mode 100644 documentation/functions_aggregate.md create mode 100644 documentation/functions_conditional.md create mode 100644 documentation/functions_date_time.md create mode 100644 documentation/functions_geo.md create mode 100644 documentation/functions_math.md create mode 100644 documentation/functions_string.md create mode 100644 documentation/functions_system.md create mode 100644 documentation/functions_type_conversion.md create mode 100644 documentation/keywords.md create mode 100644 documentation/operator_precedence.md create mode 100644 documentation/operators.md create mode 100644 documentation/request_structure.md create mode 100644 documentation/type_conversion.md diff --git a/documentation/README.md b/documentation/README.md new file mode 100644 index 00000000..0bf373cc --- /dev/null +++ b/documentation/README.md @@ -0,0 +1,15 @@ +# SQL Engine Documentation + +Welcome to the SQL Engine Documentation. Navigate through the sections below: + +- [Query Structure](request_structure.md) +- [Operators](operators.md) +- [Operator Precedence](operator_precedence.md) +- [Aggregate Functions](functions_aggregate.md) +- [Date/Time Functions](functions_date_time.md) +- [Math Functions](functions_math.md) +- [String Functions](functions_string.md) +- [Type Conversion](type_conversion.md) +- [Conditional Functions](functions_conditional.md) +- [Geo Functions](functions_geo.md) +- [Keywords](keywords.md) diff --git a/documentation/functions_aggregate.md b/documentation/functions_aggregate.md new file mode 100644 index 00000000..8f402a8c --- /dev/null +++ b/documentation/functions_aggregate.md @@ -0,0 +1,106 @@ +[Back to index](./README.md) + +# Aggregate Functions + +**Navigation:** [Functions — Date / Time](./functions_date_time.md) · [Functions — Conditional](./functions_conditional.md) + +Each function below follows the uniform documentation format. + +--- + +### Function: COUNT (Aliases: COUNT(*)) +**Description:** Count rows or non-null expressions. With `DISTINCT` counts distinct values. +**Inputs:** `expr` or `*`; optional `DISTINCT` +**Output:** `BIGINT` +**Example:** +```sql +SELECT COUNT(*) AS total FROM emp; +-- Result: total = 42 + +SELECT COUNT(DISTINCT salary) AS distinct_salaries FROM emp; +-- Result: 8 +``` + +--- + +### Function: SUM +**Description:** Sum of values. +**Inputs:** `expr` (NUMERIC) +**Output:** NUMERIC +**Example:** +```sql +SELECT SUM(salary) AS total_salary FROM emp; +``` + +--- + +### Function: AVG +**Description:** Average of values. +**Inputs:** `expr` (NUMERIC) +**Output:** DOUBLE +**Example:** +```sql +SELECT AVG(salary) AS avg_salary FROM emp; +``` + +--- + +### Function: MIN +**Description:** Minimum value in group. +**Inputs:** `expr` (comparable) +**Output:** same as input +**Example:** +```sql +SELECT MIN(hire_date) AS earliest FROM emp; +``` + +--- + +### Function: MAX +**Description:** Maximum value in group. +**Inputs:** `expr` (comparable) +**Output:** same as input +**Example:** +```sql +SELECT MAX(salary) AS top_salary FROM emp; +``` + +--- + +### Function: FIRST_VALUE +**Description:** Window: first value ordered by ORDER BY. Pushed as `top_hits size=1` to ES when possible. +**Inputs:** `expr` with `OVER (PARTITION BY ... ORDER BY ...)` +**Output:** same as input +**Example:** +```sql +SELECT FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary +FROM emp; +``` + +--- + +### Function: LAST_VALUE +**Description:** Window: last value ordered by ORDER BY. Pushed to ES by flipping sort order in `top_hits`. +**Inputs:** `expr` with `OVER (PARTITION BY ... ORDER BY ...)` +**Output:** same as input +**Example:** +```sql +SELECT LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary +FROM emp; +``` + +--- + +### Function: ARRAY_AGG +**Description:** Collect values into an array for each partition. Implemented using `OVER` and pushed to ES as `top_hits`. Post-processing converts hits to an array of scalars. +**Inputs:** `expr` with optional `OVER (PARTITION BY ... ORDER BY ... LIMIT n)` +**Output:** `ARRAY` +**Example:** +```sql +SELECT department, + ARRAY_AGG(name) OVER (PARTITION BY department ORDER BY hire_date ASC LIMIT 100) AS employees +FROM emp; +-- Result: employees is an array of name values per department (sorted and limited) +``` + +[Back to index](./README.md) diff --git a/documentation/functions_conditional.md b/documentation/functions_conditional.md new file mode 100644 index 00000000..40b830ab --- /dev/null +++ b/documentation/functions_conditional.md @@ -0,0 +1,95 @@ +[Back to index](./README.md) + +# Conditional Functions + +This page documents conditional expressions. Two syntaxes for `CASE` are supported and described below. + +--- + +### Function: CASE (searched form) +**Name & Aliases:** `CASE WHEN ... THEN ... ELSE ... END` (searched CASE form) + +**Description:** +Evaluates boolean WHEN expressions in order; returns the result expression corresponding to the first true condition; if none match, returns the ELSE expression (or NULL if ELSE omitted). + +**Inputs:** +- One or more `WHEN condition THEN result` pairs. Optional `ELSE result`. + +**Output:** +- Type coerced from result expressions (THEN/ELSE). + +**Example:** +```sql +SELECT CASE + WHEN salary > 100000 THEN 'very_high' + WHEN salary > 50000 THEN 'high' + ELSE 'normal' + END AS salary_band +FROM emp; +-- Result: 'very_high' / 'high' / 'normal' +``` + +--- + +### Function: CASE (simple / expression form) +**Name & Aliases:** `CASE expr WHEN val1 THEN r1 WHEN val2 THEN r2 ... ELSE rN END` (simple CASE) + +**Description:** +Compare `expr` to `valN` sequentially using equality; returns corresponding `rN` for first match; else `ELSE` result or NULL. + +**Inputs:** +- `expr` (any comparable type) and pairs `WHEN value THEN result`. + +**Output:** +- Type coerced from result expressions. + +**Example:** +```sql +SELECT CASE department + WHEN 'IT' THEN 'tech' + WHEN 'Sales' THEN 'revenue' + ELSE 'other' + END AS dept_category +FROM emp; +-- Result: 'tech', 'revenue', or 'other' depending on department +``` + +**Implementation notes:** +- The simple form evaluates by comparing `expr = value` for each WHEN. +- Both CASE forms are parsed and translated into nested conditional Painless scripts for `script_fields` when used outside an aggregation push-down. + +--- + +### Function: COALESCE +**Description:** Return first non-null argument. +**Inputs:** `expr1, expr2, ...` +**Output:** Value of first non-null expression (coerced) +**Example:** +```sql +SELECT COALESCE(nickname, firstname, 'N/A') AS display FROM users; +-- Result: 'Jo' or 'John' or 'N/A' +``` + +--- + +### Function: NULLIF +**Description:** Return NULL if expr1 = expr2; otherwise return expr1. +**Inputs:** `expr1, expr2` +**Output:** Type of `expr1` +**Example:** +```sql +SELECT NULLIF(status, 'unknown') AS status_norm FROM events; +``` + +--- + +### Predicate: IS NULL / IS NOT NULL (also exposed as functions ISNULL / ISNOTNULL) +**Description:** Test nullness. +**Inputs:** `expr` +**Output:** BOOLEAN +**Example:** +```sql +SELECT ISNULL(manager) AS manager_missing FROM emp; +``` + +[Back to index](./README.md) diff --git a/documentation/functions_date_time.md b/documentation/functions_date_time.md new file mode 100644 index 00000000..8ece3c58 --- /dev/null +++ b/documentation/functions_date_time.md @@ -0,0 +1,244 @@ +[Back to index](./README.md) + +# Date / Time / Datetime / Interval Functions + +**Navigation:** [Aggregate functions](./functions_aggregate.md) · [Operator Precedence](./operator_precedence.md) + +This page documents DATE vs DATETIME functions clearly, with DATETIME_* variants separated. + +--- + +### Function: CURRENT_TIMESTAMP (Aliases: NOW, CURRENT_DATETIME) +**Description:** Returns current datetime (ZonedDateTime) in UTC. +**Inputs:** none +**Output:** TIMESTAMP / DATETIME +**Example:** +```sql +SELECT CURRENT_TIMESTAMP AS now; +-- Result: 2025-09-26T12:34:56Z +``` + +--- + +### Function: CURRENT_DATE (Aliases: CURDATE, TODAY) +**Description:** Returns current date as DATE. +**Inputs:** none +**Output:** DATE +**Example:** +```sql +SELECT CURRENT_DATE AS today; +-- Result: 2025-09-26 +``` + +--- + +### Function: CURRENT_TIME (Aliases: CURTIME) +**Description:** Returns current time-of-day. +**Inputs:** none +**Output:** TIME +**Example:** +```sql +SELECT CURRENT_TIME AS t; +``` + +--- + +### Function: DATE_ADD / DATEADD (Aliases: DATEADD) +**Description:** Adds interval to DATE (LocalDate arithmetic). Use for DATE only. +**Inputs:** `date_expr` (DATE), `INTERVAL n UNIT` (YEAR|MONTH|WEEK|DAY) +**Output:** DATE +**Example:** +```sql +SELECT DATE_ADD(DATE '2025-01-10', INTERVAL 1 MONTH) AS next_month; +-- Result: 2025-02-10 +``` + +--- + +### Function: DATE_SUB / DATESUB +**Description:** Subtract interval from DATE. +**Inputs:** `date_expr` (DATE), `INTERVAL n UNIT` +**Output:** DATE +**Example:** +```sql +SELECT DATE_SUB(DATE '2025-01-10', INTERVAL 7 DAY) AS week_before; +-- Result: 2025-01-03 +``` + +--- + +### Function: DATETIME_ADD / DATETIMEADD +**Description:** Adds interval to DATETIME/TIMESTAMP (ZonedDateTime arithmetic). Use for DATETIME only. +**Inputs:** `datetime_expr` (DATETIME), `INTERVAL n UNIT` (including HOUR|MINUTE|SECOND) +**Output:** DATETIME +**Example:** +```sql +SELECT DATETIME_ADD(TIMESTAMP '2025-01-10T12:00:00Z', INTERVAL 1 DAY) AS tomorrow; +-- Result: 2025-01-11T12:00:00Z +``` + +--- + +### Function: DATETIME_SUB / DATETIMESUB +**Description:** Subtract interval from DATETIME/TIMESTAMP. +**Inputs:** `datetime_expr`, `INTERVAL n UNIT` +**Output:** DATETIME +**Example:** +```sql +SELECT DATETIME_SUB(TIMESTAMP '2025-01-10T12:00:00Z', INTERVAL 2 HOUR) AS earlier; +``` + +--- + +### Function: DATEDIFF / DATE_DIFF +**Description:** Difference in days (date1 - date2). +**Inputs:** `date1`, `date2` (DATE or DATETIME) +**Output:** BIGINT +**Example:** +```sql +SELECT DATEDIFF(DATE '2025-01-10', DATE '2025-01-01') AS diff; +-- Result: 9 +``` + +--- + +### Function: DATE_FORMAT +**Description:** Format DATE/DATETIME to string using Java DateTimeFormatter. +**Inputs:** `date_expr`, `pattern` +**Output:** VARCHAR +**Example:** +```sql +SELECT DATE_FORMAT(DATE '2025-01-10', 'yyyy-MM-dd') AS fmt; +-- Result: '2025-01-10' +``` + +--- + +### Function: DATE_PARSE +**Description:** Parse string into DATE. +**Inputs:** `string`, `pattern` +**Output:** DATE +**Example:** +```sql +SELECT DATE_PARSE('2025-01-10','yyyy-MM-dd') AS d; +``` + +--- + +### Function: DATETIME_PARSE +**Description:** Parse string into DATETIME/TIMESTAMP. +**Inputs:** `string`, `pattern` +**Output:** DATETIME (ZonedDateTime) +**Example:** +```sql +SELECT DATETIME_PARSE('2025-01-10T12:00:00Z','yyyy-MM-dd''T''HH:mm:ssX') AS dt; +``` + +--- + +### Function: DATETIME_FORMAT +**Description:** Format DATETIME/TIMESTAMP to string with pattern. +**Inputs:** `datetime_expr`, `pattern` +**Output:** VARCHAR +**Example:** +```sql +SELECT DATETIME_FORMAT(TIMESTAMP '2025-01-10T12:00:00Z','yyyy-MM-dd HH:mm:ss') AS s; +``` + +--- + +### Function: DATE_TRUNC +**Description:** Truncate date/datetime to a unit. +**Inputs:** `date_or_datetime_expr`, `unit` +**Output:** DATE or DATETIME +**Example:** +```sql +SELECT DATE_TRUNC(DATE '2025-01-15', MONTH) AS start_month; +-- Result: 2025-01-01 +``` + +--- + +### Function: EXTRACT +**Description:** Extract field from date or datetime. +**Inputs:** `unit FROM date_expr` +**Output:** INTEGER / BIGINT +**Example:** +```sql +SELECT EXTRACT(YEAR FROM TIMESTAMP '2025-01-10T12:00:00Z') AS y; +-- Result: 2025 +``` + +--- + +### Function: LAST_DAY +**Description:** Last day of month for a date. +**Inputs:** `date_expr` +**Output:** DATE +**Example:** +```sql +SELECT LAST_DAY(DATE '2025-02-15') AS ld; +-- Result: 2025-02-28 +``` + +--- + +### Function: WEEK +**Description:** ISO week number (1..53) +**Inputs:** `date_expr` +**Output:** INTEGER +**Example:** +```sql +SELECT WEEK(DATE '2025-01-01') AS w; +-- Result: 1 +``` + +--- + +### Function: QUARTER +**Description:** Quarter number (1..4) +**Inputs:** `date_expr` +**Output:** INTEGER +**Example:** +```sql +SELECT QUARTER(DATE '2025-05-10') AS q; +-- Result: 2 +``` + +--- + +### Function: NANOSECOND / MICROSECOND / MILLISECOND +**Description:** Sub-second extraction. +**Inputs:** `datetime_expr` +**Output:** INTEGER +**Example:** +```sql +SELECT MILLISECOND(TIMESTAMP '2025-01-01T12:00:00.123Z') AS ms; +-- Result: 123 +``` + +--- + +### Function: EPOCHDAY +**Description:** Days since epoch. +**Inputs:** `date_expr` +**Output:** BIGINT +**Example:** +```sql +SELECT EPOCHDAY(DATE '1970-01-02') AS d; +-- Result: 1 +``` + +--- + +### Function: OFFSET / OFFSET_SECONDS +**Description:** Timezone offset in seconds. +**Inputs:** `timestamp_expr` +**Output:** INTEGER +**Example:** +```sql +SELECT OFFSET(TIMESTAMP '2025-01-01T12:00:00+02:00') AS off; +-- Result: 7200 +``` + +[Back to index](./README.md) diff --git a/documentation/functions_geo.md b/documentation/functions_geo.md new file mode 100644 index 00000000..96de8e6f --- /dev/null +++ b/documentation/functions_geo.md @@ -0,0 +1,26 @@ +[Back to index](./README.md) + +# Geo Functions + +--- + +### Function: ST_DISTANCE (Aliases: DISTANCE) +**Description:** Compute distance between two geo points. Under the hood may use Haversine or ES geo-distance depending on index mapping and push-down capability. +**Inputs:** `point1`, `point2` (WKT or field references) +**Output:** DOUBLE (distance in meters or configured units) +**Example:** +```sql +SELECT ST_DISTANCE(location, 'POINT(2.3522 48.8566)') AS dist FROM places; +-- Result: e.g., 1234.56 +``` + +### Function: ST_WITHIN +**Description:** Test whether point/geometry is within another geometry. +**Inputs:** `geom1`, `geom2` +**Output:** BOOLEAN +**Example:** +```sql +SELECT ST_WITHIN(location, 'POLYGON((...))') FROM places; +``` + +[Back to index](./README.md) diff --git a/documentation/functions_math.md b/documentation/functions_math.md new file mode 100644 index 00000000..d7988931 --- /dev/null +++ b/documentation/functions_math.md @@ -0,0 +1,119 @@ +[Back to index](./README.md) + +# Mathematical Functions + +**Navigation:** [Functions — Aggregate](./functions_aggregate.md) · [Functions — String](./functions_string.md) + +--- + +### Function: ABS +**Description:** Absolute value. +**Inputs:** `x` (NUMERIC) +**Output:** NUMERIC +**Example:** +```sql +SELECT ABS(-5) AS a; +-- Result: 5 +``` + +### Function: ROUND +**Description:** Round to n decimals (optional). +**Inputs:** `x` (NUMERIC), optional `n` (INTEGER) +**Output:** NUMERIC +**Example:** +```sql +SELECT ROUND(123.456, 2) AS r; +-- Result: 123.46 +``` + +### Function: FLOOR +**Description:** Greatest integer ≤ x. +**Inputs:** `x` (NUMERIC) +**Output:** INTEGER +**Example:** +```sql +SELECT FLOOR(3.9) AS f; +-- Result: 3 +``` + +### Function: CEIL / CEILING +**Description:** Smallest integer ≥ x. +**Inputs:** `x` (NUMERIC) +**Output:** INTEGER +**Example:** +```sql +SELECT CEIL(3.1) AS c; +-- Result: 4 +``` + +### Function: POWER / POW +**Description:** x^y. +**Inputs:** `x` (NUMERIC), `y` (NUMERIC) +**Output:** NUMERIC +**Example:** +```sql +SELECT POWER(2, 10) AS p; +-- Result: 1024 +``` + +### Function: SQRT +**Description:** Square root. +**Inputs:** `x` (NUMERIC >= 0) +**Output:** NUMERIC +**Example:** +```sql +SELECT SQRT(16) AS s; +-- Result: 4 +``` + +### Function: LOG / LN +**Description:** Natural logarithm. +**Inputs:** `x` (NUMERIC > 0) +**Output:** NUMERIC +**Example:** +```sql +SELECT LOG(EXP(1)) AS l; +-- Result: 1 +``` + +### Function: LOG10 +**Description:** Base-10 logarithm. +**Inputs:** `x` (NUMERIC > 0) +**Output:** NUMERIC +**Example:** +```sql +SELECT LOG10(1000) AS l10; +-- Result: 3 +``` + +### Function: EXP +**Description:** e^x. +**Inputs:** `x` (NUMERIC) +**Output:** NUMERIC +**Example:** +```sql +SELECT EXP(1) AS e; +-- Result: 2.71828... +``` + +### Function: SIGN / SGN +**Description:** Returns -1, 0, or 1 according to sign. +**Inputs:** `x` (NUMERIC) +**Output:** INTEGER +**Example:** +```sql +SELECT SIGN(-10) AS s; +-- Result: -1 +``` + +### Trigonometric functions: COS, ACOS, SIN, ASIN, TAN, ATAN, ATAN2 +**Description:** Standard trig functions. Inputs in radians. +**Inputs:** `x` or (`y`, `x` for ATAN2) +**Output:** NUMERIC +**Example:** +```sql +SELECT COS(PI()/3) AS c; +-- Result: 0.5 +``` + +[Back to index](./README.md) diff --git a/documentation/functions_string.md b/documentation/functions_string.md new file mode 100644 index 00000000..8960e652 --- /dev/null +++ b/documentation/functions_string.md @@ -0,0 +1,95 @@ +[Back to index](./README.md) + +# String Functions + +--- + +### Function: UPPER / UCASE +**Description:** Convert string to upper case. +**Inputs:** `str` (VARCHAR) +**Output:** `VARCHAR` +**Example:** +```sql +SELECT UPPER('hello') AS up; +-- Result: 'HELLO' +``` + +### Function: LOWER / LCASE +**Description:** Convert string to lower case. +**Inputs:** `str` (VARCHAR) +**Output:** `VARCHAR` +**Example:** +```sql +SELECT LOWER('Hello') AS lo; +-- Result: 'hello' +``` + +### Function: TRIM +**Description:** Trim whitespace both sides. +**Inputs:** `str` (VARCHAR) +**Output:** `VARCHAR` +**Example:** +```sql +SELECT TRIM(' abc ') AS t; +-- Result: 'abc' +``` + +### Function: LENGTH / LEN +**Description:** Character length. +**Inputs:** `str` (VARCHAR) +**Output:** INTEGER +**Example:** +```sql +SELECT LENGTH('abc') AS l; +-- Result: 3 +``` + +### Function: SUBSTRING / SUBSTR +**Description:** SQL 1-based substring. +**Inputs:** `str` (VARCHAR), `start` (INT >=1), optional `length` (INT) +**Output:** `VARCHAR` +**Example:** +```sql +SELECT SUBSTRING('abcdef', 2, 3) AS s; +-- Result: 'bcd' +``` + +### Function: CONCAT +**Description:** Concatenate values into a string. +**Inputs:** `expr1, expr2, ...` (coercible to VARCHAR) +**Output:** VARCHAR +**Example:** +```sql +SELECT CONCAT(firstName, ' ', lastName) AS full FROM users; +``` + +### Function: REPLACE +**Description:** Replace substring occurrences. +**Inputs:** `str, search, replace` +**Output:** VARCHAR +**Example:** +```sql +SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; +-- Result: 'John' +``` + +### Function: POSITION / STRPOS +**Description:** 1-based index, 0 if not found. +**Inputs:** `substr, str` +**Output:** INTEGER +**Example:** +```sql +SELECT POSITION('lo' IN 'hello') AS pos; +-- Result: 4 +``` + +### Function: REGEXP_LIKE / RLIKE +**Description:** Regex match predicate. +**Inputs:** `str, pattern` +**Output:** BOOLEAN +**Example:** +```sql +SELECT REGEXP_LIKE(email, '.*@example\.com') AS ok FROM users; +``` + +[Back to index](./README.md) diff --git a/documentation/functions_system.md b/documentation/functions_system.md new file mode 100644 index 00000000..f4120f3f --- /dev/null +++ b/documentation/functions_system.md @@ -0,0 +1,26 @@ +[Back to index](./README.md) + +# System Functions + +--- + +### Function: VERSION +**Description:** Return engine version string. +**Inputs:** none +**Output:** VARCHAR +**Example:** +```sql +SELECT VERSION() AS v; +-- Result: 'sql-elasticsearch-engine 1.0.0' +``` + +### Function: USER +**Description:** Return current user (context-specific). +**Inputs:** none +**Output:** VARCHAR +**Example:** +```sql +SELECT USER() AS current_user; +``` + +[Back to index](./README.md) diff --git a/documentation/functions_type_conversion.md b/documentation/functions_type_conversion.md new file mode 100644 index 00000000..79aa334e --- /dev/null +++ b/documentation/functions_type_conversion.md @@ -0,0 +1,27 @@ +[Back to index](./README.md) + +# Type Conversion Functions + +--- + +### Function: CAST (Aliases: CONVERT) +**Description:** Cast expression to a target SQL type. +**Inputs:** `expr`, `TYPE` (DATE, TIMESTAMP, VARCHAR, INT, DOUBLE, etc.) +**Output:** `TYPE` +**Example:** +```sql +SELECT CAST(salary AS DOUBLE) AS s FROM emp; +-- Result: 12345.0 +``` + +### Function: TRY_CAST (Aliases: none) +**Description:** Attempt a cast and return NULL on failure (safer alternative). +**Inputs:** `expr`, `TYPE` +**Output:** `TYPE` or NULL +**Example:** +```sql +SELECT TRY_CAST('not-a-number' AS INT) AS maybe_null; +-- Result: NULL +``` + +[Back to index](./README.md) diff --git a/documentation/keywords.md b/documentation/keywords.md new file mode 100644 index 00000000..eff9a6ac --- /dev/null +++ b/documentation/keywords.md @@ -0,0 +1,20 @@ +[Back to index](./README.md) + +# Keywords + +A list of reserved words recognized by the parser for this engine. + +``` +SELECT, FROM, WHERE, GROUP BY, HAVING, ORDER BY, UNNEST, AS, CAST, +COUNT, SUM, AVG, MIN, MAX, FIRST_VALUE, LAST_VALUE, ARRAY_AGG, +ROW_NUMBER, RANK, LAG, LEAD, DAY, MONTH, YEAR, HOUR, MINUTE, SECOND, +LAST_DAY, DAYOFWEEK, DAYOFYEAR, WEEK, QUARTER, NANOSECOND, MICROSECOND, +MILLISECOND, EPOCHDAY, OFFSET, UPPER, LOWER, TRIM, LENGTH, SUBSTRING, +CONCAT, POSITION, REGEXP_LIKE, REPLACE, ABS, ROUND, FLOOR, CEIL, POWER, +SQRT, LOG, LOG10, EXP, SIGN, COS, ACOS, SIN, ASIN, TAN, ATAN, ATAN2, +CASE, WHEN, THEN, ELSE, END, COALESCE, ISNULL, ISNOTNULL, NULLIF, +CURRENT_DATE, CURDATE, TODAY, NOW, CURRENT_TIME, CURTIME, CURRENT_TIMESTAMP, +LIKE, RLIKE, IN, NOT IN, BETWEEN, IS NULL, IS NOT NULL, AND, OR, NOT, INTERVAL +``` + +[Back to index](./README.md) diff --git a/documentation/operator_precedence.md b/documentation/operator_precedence.md new file mode 100644 index 00000000..8c3697db --- /dev/null +++ b/documentation/operator_precedence.md @@ -0,0 +1,24 @@ +[Back to index](./README.md) + +# Operator Precedence + +This page lists operator precedence used by the parser and evaluator (highest precedence at top). + +1. Parentheses `(...)` +2. Unary operators: `-` (negation), `+` (unary plus), `NOT` +3. Multiplicative: `*`, `/`, `%` +4. Additive: `+`, `-` +5. Comparison: `<`, `<=`, `>`, `>=` +6. Equality: `=`, `!=`, `<>` +7. Membership & pattern: `BETWEEN`, `IN`, `LIKE`, `RLIKE` +8. Logical `AND` +9. Logical `OR` + +**Notes and examples** +```sql +SELECT 1 + 2 * 3 AS v; -- v = 7 +SELECT (1 + 2) * 3 AS v; -- v = 9 +SELECT a BETWEEN 1 AND 3 OR b = 5; -- interpreted as (a BETWEEN 1 AND 3) OR (b = 5) +``` + +[Back to index](./README.md) diff --git a/documentation/operators.md b/documentation/operators.md new file mode 100644 index 00000000..e2b68f02 --- /dev/null +++ b/documentation/operators.md @@ -0,0 +1,155 @@ +[Back to index](./README.md) + +# Operators (detailed) + +**Navigation:** [Query Structure](./query-structure.md) · [Operator Precedence](./operator_precedence.md) · [Keywords](./keywords.md) + +This file provides a per-operator description and a concrete SQL example for each operator supported by the engine. + +--- + +### Operator: `+` +**Description:** Arithmetic addition. +**Example:** +```sql +SELECT salary + bonus AS total_comp FROM emp; +-- result example: if salary=50000 and bonus=10000 -> total_comp = 60000 +``` + +### Operator: `-` +**Description:** Arithmetic subtraction or unary negation when used with single operand. +**Example:** +```sql +SELECT salary - tax AS net FROM emp; +SELECT -balance AS negative_balance FROM accounts; +``` + +### Operator: `*` +**Description:** Multiplication. +**Example:** +```sql +SELECT quantity * price AS revenue FROM sales; +``` + +### Operator: `/` +**Description:** Division; division by zero must be guarded (NULLIF), engine returns NULL for invalid arithmetic. +**Example:** +```sql +SELECT total / NULLIF(count,0) AS avg FROM table; +``` + +### Operator: `%` (MOD) +**Description:** Remainder/modulo operator. +**Example:** +```sql +SELECT id % 10 AS bucket FROM users; +``` + +--- + +### Operator: `=` +**Description:** Equality comparison. +**Example:** +```sql +SELECT * FROM emp WHERE department = 'IT'; +``` + +### Operator: `<>`, `!=` +**Description:** Inequality comparison (both synonyms supported). +**Example:** +```sql +SELECT * FROM emp WHERE status <> 'terminated'; +``` + +### Operator: `<`, `<=`, `>`, `>=` +**Description:** Relational comparisons. +**Example:** +```sql +SELECT * FROM emp WHERE age >= 21 AND age < 65; +``` + +### Operator: `IN` +**Description:** Membership in a set of literal or numeric values or results of subquery (subquery support depends on implementation). +**Example:** +```sql +SELECT * FROM emp WHERE department IN ('Sales', 'IT', 'HR'); +SELECT * FROM emp WHERE status IN (1, 2); +``` + +### Operator: `NOT IN` +**Description:** Negated membership. +**Example:** +```sql +SELECT * FROM emp WHERE department NOT IN ('HR','Legal'); +``` + +### Operator: `BETWEEN ... AND ...` +**Description:** Inclusive range test: `expr BETWEEN a AND b` ⇔ `expr >= a AND expr <= b`. +**Example:** +```sql +SELECT * FROM emp WHERE salary BETWEEN 40000 AND 80000; +``` + +### Operator: `IS NULL` +**Description:** Null check predicate. +**Example:** +```sql +SELECT * FROM emp WHERE manager IS NULL; +``` + +### Operator: `IS NOT NULL` +**Description:** Negated null check. +**Example:** +```sql +SELECT * FROM emp WHERE manager IS NOT NULL; +``` + +--- + +### Operator: `LIKE` +**Description:** Pattern match using `%` and `_`. Engine converts `%` → `.*` and `_` → `.` for underlying regex matching. +**Example:** +```sql +SELECT * FROM emp WHERE name LIKE 'Jo%'; +``` + +### Operator: `RLIKE` +**Description:** Regular-expression match (Java regex semantics). +**Example:** +```sql +SELECT * FROM users WHERE email RLIKE '.*@example\.com$'; +``` + +--- + +### Operator: `AND` +**Description:** Logical conjunction. +**Example:** +```sql +SELECT * FROM emp WHERE dept = 'IT' AND salary > 50000; +``` + +### Operator: `OR` +**Description:** Logical disjunction. +**Example:** +```sql +SELECT * FROM emp WHERE dept = 'IT' OR dept = 'Sales'; +``` + +### Operator: `NOT` +**Description:** Logical negation. +**Example:** +```sql +SELECT * FROM emp WHERE NOT active; +``` + +--- + +### Operator-like: `CAST(...)` / `CONVERT(...)` +**Description:** Type conversion operator/function. See Type Conversion functions page for details and examples. +**Example:** +```sql +SELECT CAST(hire_date AS DATE) FROM emp; +``` + +[Back to index](./README.md) diff --git a/documentation/request_structure.md b/documentation/request_structure.md new file mode 100644 index 00000000..6f12ef3d --- /dev/null +++ b/documentation/request_structure.md @@ -0,0 +1,104 @@ +[Back to index](./README.md) + +# Query Structure + +**Navigation:** [Operators](./operators.md) · [Functions — Aggregate](./functions_aggregate.md) · [Keywords](./keywords.md) + +This page documents the SQL clauses supported by the engine and how they map to Elasticsearch. + +--- + +## SELECT +**Description:** Projection of fields, expressions and computed values. + +**Behavior:** +- `_source` includes for plain fields. +- Computed expressions are translated into `script_fields` (Painless) when push-down is not otherwise possible. +- Aggregates are translated to ES aggregations and the top-level `size` is often set to `0` for aggregation-only queries. + +**Example:** +```sql +SELECT department, COUNT(*) AS cnt +FROM emp +GROUP BY department; +``` + +--- + +## FROM +**Description:** Source index (one or more). Translates to the Elasticsearch index parameter. + +**Example:** +```sql +SELECT * FROM employees; +``` + +--- + +## UNNEST +**Description:** Expand an array / nested field into rows. Mapped to Elasticsearch `nested` and inner hits where necessary. + +**Example:** +```sql +SELECT id, phone +FROM customers +UNNEST(phones) AS phone; +``` + +--- + +## WHERE +**Description:** Row-level predicates. Mapped to `bool` queries; complex expressions become `script` queries (Painless). + +**Example:** +```sql +SELECT * FROM emp WHERE salary > 50000 AND department = 'IT'; +``` + +--- + +## GROUP BY +**Description:** Aggregation buckets. Mapped to `terms`/`date_histogram` and nested sub-aggregations. +Non-aggregated selected fields are disallowed unless included in the `GROUP BY` (standard SQL semantics). + +**Example:** +```sql +SELECT department, AVG(salary) AS avg_salary +FROM emp +GROUP BY department; +``` + +--- + +## HAVING +**Description:** Filter groups using aggregate expressions. Implemented with pipeline aggregations and `bucket_selector` where possible, or client-side filtering if required. + +**Example:** +```sql +SELECT department, COUNT(*) AS cnt +FROM emp +GROUP BY department +HAVING COUNT(*) > 10; +``` + +--- + +## ORDER BY +**Description:** Sorting of final rows or ordering used inside window/aggregations (pushed to `sort` or `top_hits`). + +**Example:** +```sql +SELECT name, salary FROM emp ORDER BY salary DESC; +``` + +--- + +## LIMIT / OFFSET +**Description:** Limit and paging. For pure aggregations, `size` is typically set to 0 and `limit` applies to aggregations or outer rows. + +**Example:** +```sql +SELECT * FROM emp ORDER BY hire_date DESC LIMIT 10 OFFSET 20; +``` + +[Back to index](./README.md) diff --git a/documentation/type_conversion.md b/documentation/type_conversion.md new file mode 100644 index 00000000..cb9b89bc --- /dev/null +++ b/documentation/type_conversion.md @@ -0,0 +1,92 @@ +[Back to index](./README.md) + +# Type Conversion Functions and Operators + +## Function: CAST (Alias: NONE) + +**Description:** +Converts a value to a specified SQL type. Fails if the conversion is invalid. + +**Inputs:** +- `value` (ANY type) +- `targetType` (SQLType: `INT`, `BIGINT`, `DOUBLE`, `DATE`, `DATETIME`, `TIMESTAMP`, `VARCHAR`, etc.) + +**Output:** +- `targetType` + +**Example:** +```sql +SELECT CAST('2025-09-11' AS DATE) AS d; +-- Result: 2025-09-11 +``` + +--- + +## Function: TRY_CAST (Aliases: SAFE_CAST) + +**Description:** +Attempts to convert a value to a specified SQL type. Returns `NULL` if the conversion fails instead of raising an error. + +**Inputs:** +- `value` (ANY type) +- `targetType` (SQLType: `INT`, `BIGINT`, `DOUBLE`, `DATE`, `DATETIME`, etc.) + +**Output:** +- `targetType` (nullable) + +**Example:** +```sql +SELECT TRY_CAST('invalid-date' AS DATE) AS d; +-- Result: NULL +``` + +--- + +## Function: CONVERT (Alias: NONE) + +**Description:** +Converts a value to a specified SQL type. Equivalent to `CAST`, but uses function syntax instead of `CAST ... AS ...`. + +**Inputs:** +- `value` (ANY type) +- `targetType` (SQLType) + +**Output:** +- `targetType` + +**Example:** +```sql +SELECT CONVERT('125', BIGINT) AS b; +-- Result: 125 +``` + +--- + +## Operator: `::` (Cast Operator) + +**Description:** +Shorthand operator for casting. Equivalent to `CAST(value AS type)`. + +**Inputs:** +- `value` (ANY type) +- `targetType` (SQLType) + +**Output:** +- `targetType` + +**Example:** +```sql +SELECT '2025-09-11'::DATE AS d, '125'::BIGINT AS b; +-- Result: 2025-09-11, 125 +``` + +--- + +## Behavior Notes + +- `CAST` and `CONVERT` will raise errors on invalid conversions. +- `TRY_CAST` (`SAFE_CAST`) returns `NULL` instead of failing. +- `::` is syntactic sugar, easier to read in queries. +- Type inference relies on `baseType`, and explicit `CAST`/`CONVERT`/`::` updates the type context for following functions. + +[Back to index](./README.md) From b057505036cabaa3ba619eef9334c9745c027456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 11:17:00 +0200 Subject: [PATCH 35/48] fix link to Query structure --- documentation/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/operators.md b/documentation/operators.md index e2b68f02..89b7fe28 100644 --- a/documentation/operators.md +++ b/documentation/operators.md @@ -2,7 +2,7 @@ # Operators (detailed) -**Navigation:** [Query Structure](./query-structure.md) · [Operator Precedence](./operator_precedence.md) · [Keywords](./keywords.md) +**Navigation:** [Query Structure](./request_structure.md) · [Operator Precedence](./operator_precedence.md) · [Keywords](./keywords.md) This file provides a per-operator description and a concrete SQL example for each operator supported by the engine. From a8d2cff95e2e13f678eef23fa431ba7631915fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 14:40:37 +0200 Subject: [PATCH 36/48] update sql engine documentation, update window functions parsing in order to use the column name for the sorting if no OVER has been defined --- documentation/functions_aggregate.md | 125 ++++++-- documentation/functions_conditional.md | 60 +++- documentation/functions_date_time.md | 277 +++++++++++++----- documentation/functions_geo.md | 44 ++- documentation/functions_math.md | 131 ++++++--- documentation/functions_string.md | 108 +++++-- documentation/functions_system.md | 21 +- documentation/functions_type_conversion.md | 29 +- documentation/operators.md | 230 ++++++++++++--- documentation/request_structure.md | 24 +- documentation/type_conversion.md | 6 +- .../elastic/sql/function/math/package.scala | 24 +- .../elastic/sql/function/string/package.scala | 12 +- .../parser/function/aggregate/package.scala | 15 +- 14 files changed, 827 insertions(+), 279 deletions(-) diff --git a/documentation/functions_aggregate.md b/documentation/functions_aggregate.md index 8f402a8c..2c8b458b 100644 --- a/documentation/functions_aggregate.md +++ b/documentation/functions_aggregate.md @@ -4,14 +4,21 @@ **Navigation:** [Functions — Date / Time](./functions_date_time.md) · [Functions — Conditional](./functions_conditional.md) -Each function below follows the uniform documentation format. +This page documents aggregate functions. --- ### Function: COUNT (Aliases: COUNT(*)) -**Description:** Count rows or non-null expressions. With `DISTINCT` counts distinct values. -**Inputs:** `expr` or `*`; optional `DISTINCT` -**Output:** `BIGINT` +**Description:** +Count rows or non-null expressions. +With `DISTINCT` counts distinct values. + +**Inputs:** +- `expr` or `*`; optional `DISTINCT` + +**Output:** +- `BIGINT` + **Example:** ```sql SELECT COUNT(*) AS total FROM emp; @@ -24,9 +31,15 @@ SELECT COUNT(DISTINCT salary) AS distinct_salaries FROM emp; --- ### Function: SUM -**Description:** Sum of values. -**Inputs:** `expr` (NUMERIC) -**Output:** NUMERIC +**Description:** +Sum of values. + +**Inputs:** +- `expr` (`NUMERIC`) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT SUM(salary) AS total_salary FROM emp; @@ -35,9 +48,15 @@ SELECT SUM(salary) AS total_salary FROM emp; --- ### Function: AVG -**Description:** Average of values. -**Inputs:** `expr` (NUMERIC) -**Output:** DOUBLE +**Description:** +Average of values. + +**Inputs:** +- `expr` (`NUMERIC`) + +**Output:** +- `DOUBLE` + **Example:** ```sql SELECT AVG(salary) AS avg_salary FROM emp; @@ -46,9 +65,15 @@ SELECT AVG(salary) AS avg_salary FROM emp; --- ### Function: MIN -**Description:** Minimum value in group. -**Inputs:** `expr` (comparable) -**Output:** same as input +**Description:** +Minimum value in group. + +**Inputs:** +- `expr` (comparable) + +**Output:** +- same as input + **Example:** ```sql SELECT MIN(hire_date) AS earliest FROM emp; @@ -57,9 +82,15 @@ SELECT MIN(hire_date) AS earliest FROM emp; --- ### Function: MAX -**Description:** Maximum value in group. -**Inputs:** `expr` (comparable) -**Output:** same as input +**Description:** +Maximum value in group. + +**Inputs:** +- `expr` (comparable) + +**Output:** +- same as input + **Example:** ```sql SELECT MAX(salary) AS top_salary FROM emp; @@ -68,39 +99,73 @@ SELECT MAX(salary) AS top_salary FROM emp; --- ### Function: FIRST_VALUE -**Description:** Window: first value ordered by ORDER BY. Pushed as `top_hits size=1` to ES when possible. -**Inputs:** `expr` with `OVER (PARTITION BY ... ORDER BY ...)` -**Output:** same as input +**Description:** +Window: first value ordered by `ORDER BY`. Pushed as `top_hits size=1` to ES when possible. + +**Inputs:** +- `expr` with optional `OVER (PARTITION BY ... ORDER BY ...)` +If `OVER` is not provided, only the expr column name is used for the sorting. + +**Output:** +- same as input + **Example:** ```sql -SELECT FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary +SELECT FIRST_VALUE(salary) +OVER ( + PARTITION BY department + ORDER BY hire_date ASC +) AS first_salary FROM emp; ``` --- ### Function: LAST_VALUE -**Description:** Window: last value ordered by ORDER BY. Pushed to ES by flipping sort order in `top_hits`. -**Inputs:** `expr` with `OVER (PARTITION BY ... ORDER BY ...)` -**Output:** same as input +**Description:** +Window: last value ordered by `ORDER BY. Pushed to ES by flipping sort order in `top_hits`. + +**Inputs:** +- `expr` with optional `OVER (PARTITION BY ... ORDER BY ...)` +If `OVER` is not provided, only the expr column name is used for the sorting. + +**Output:** +- same as input + **Example:** ```sql -SELECT LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary +SELECT LAST_VALUE(salary) +OVER ( + PARTITION BY department + ORDER BY hire_date ASC +) AS last_salary FROM emp; ``` --- ### Function: ARRAY_AGG -**Description:** Collect values into an array for each partition. Implemented using `OVER` and pushed to ES as `top_hits`. Post-processing converts hits to an array of scalars. -**Inputs:** `expr` with optional `OVER (PARTITION BY ... ORDER BY ... LIMIT n)` -**Output:** `ARRAY` +**Description:** +Collect values into an array for each partition. Implemented using `OVER` and pushed to ES as `top_hits`. Post-processing converts hits to an array of scalars. + +**Inputs:** +- `expr` with optional `OVER (PARTITION BY ... ORDER BY ... LIMIT n)` +If `OVER` is not provided, only the expr column name is used for the sorting. + +**Output:** +- `ARRAY` + **Example:** ```sql -SELECT department, - ARRAY_AGG(name) OVER (PARTITION BY department ORDER BY hire_date ASC LIMIT 100) AS employees +SELECT department, +ARRAY_AGG(name) OVER ( + PARTITION BY department + ORDER BY hire_date ASC + LIMIT 100 +) AS employees FROM emp; --- Result: employees is an array of name values per department (sorted and limited) +-- Result: employees as an array of name values +-- per department (sorted and limited) ``` [Back to index](./README.md) diff --git a/documentation/functions_conditional.md b/documentation/functions_conditional.md index 40b830ab..f02a19fd 100644 --- a/documentation/functions_conditional.md +++ b/documentation/functions_conditional.md @@ -2,7 +2,7 @@ # Conditional Functions -This page documents conditional expressions. Two syntaxes for `CASE` are supported and described below. +This page documents conditional expressions. --- @@ -61,9 +61,15 @@ FROM emp; --- ### Function: COALESCE -**Description:** Return first non-null argument. -**Inputs:** `expr1, expr2, ...` -**Output:** Value of first non-null expression (coerced) +**Description:** +Return first non-null argument. + +**Inputs:** +- `expr1, expr2, ...` + +**Output:** +- Value of first non-null expression (coerced) + **Example:** ```sql SELECT COALESCE(nickname, firstname, 'N/A') AS display FROM users; @@ -73,23 +79,55 @@ SELECT COALESCE(nickname, firstname, 'N/A') AS display FROM users; --- ### Function: NULLIF -**Description:** Return NULL if expr1 = expr2; otherwise return expr1. -**Inputs:** `expr1, expr2` -**Output:** Type of `expr1` +**Description:** +Return NULL if expr1 = expr2; otherwise return expr1. + +**Inputs:** +- `expr1, expr2` + +**Output:** +- Type of `expr1` + **Example:** ```sql SELECT NULLIF(status, 'unknown') AS status_norm FROM events; +-- Result: NULL if status is 'unknown', else original status ``` --- -### Predicate: IS NULL / IS NOT NULL (also exposed as functions ISNULL / ISNOTNULL) -**Description:** Test nullness. -**Inputs:** `expr` -**Output:** BOOLEAN +### Function: ISNULL +**Description:** +Test nullness. + +**Inputs:** +- `expr` + +**Output:** +- `BOOLEAN` + **Example:** ```sql SELECT ISNULL(manager) AS manager_missing FROM emp; +-- Result: TRUE if manager is NULL, else FALSE +``` + +--- + +### Function: ISNOTNULL +**Description:** +Test non-nullness. + +**Inputs:** +- `expr` + +**Output:** +- `BOOLEAN` + +**Example:** +```sql +SELECT ISNOTNULL(manager) AS manager_missing FROM emp; +-- Result: TRUE if manager is NOT NULL, else FALSE ``` [Back to index](./README.md) diff --git a/documentation/functions_date_time.md b/documentation/functions_date_time.md index 8ece3c58..40ee21cb 100644 --- a/documentation/functions_date_time.md +++ b/documentation/functions_date_time.md @@ -1,17 +1,23 @@ [Back to index](./README.md) -# Date / Time / Datetime / Interval Functions +# Date / Time / Datetime / Timestamp / Interval Functions **Navigation:** [Aggregate functions](./functions_aggregate.md) · [Operator Precedence](./operator_precedence.md) -This page documents DATE vs DATETIME functions clearly, with DATETIME_* variants separated. +This page documents TEMPORAL functions. --- ### Function: CURRENT_TIMESTAMP (Aliases: NOW, CURRENT_DATETIME) -**Description:** Returns current datetime (ZonedDateTime) in UTC. -**Inputs:** none -**Output:** TIMESTAMP / DATETIME +**Description:** +Returns current datetime (ZonedDateTime) in UTC. + +**Inputs:** +- none + +**Output:** +- `TIMESTAMP` / `DATETIME` + **Example:** ```sql SELECT CURRENT_TIMESTAMP AS now; @@ -21,9 +27,15 @@ SELECT CURRENT_TIMESTAMP AS now; --- ### Function: CURRENT_DATE (Aliases: CURDATE, TODAY) -**Description:** Returns current date as DATE. -**Inputs:** none -**Output:** DATE +**Description:** +Returns current date as `DATE`. + +**Inputs:** +- none + +**Output:** +- `DATE` + **Example:** ```sql SELECT CURRENT_DATE AS today; @@ -33,211 +45,322 @@ SELECT CURRENT_DATE AS today; --- ### Function: CURRENT_TIME (Aliases: CURTIME) -**Description:** Returns current time-of-day. -**Inputs:** none -**Output:** TIME +**Description:** +Returns current time-of-day. + +**Inputs:** +- none + +**Output:** +- `TIME` + **Example:** ```sql SELECT CURRENT_TIME AS t; +-- Result: 12:34:56 ``` --- ### Function: DATE_ADD / DATEADD (Aliases: DATEADD) -**Description:** Adds interval to DATE (LocalDate arithmetic). Use for DATE only. -**Inputs:** `date_expr` (DATE), `INTERVAL n UNIT` (YEAR|MONTH|WEEK|DAY) -**Output:** DATE +**Description:** +Adds interval to `DATE`. + +**Inputs:** +- `date_expr` (`DATE`), `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`) + +**Output:** +- `DATE` + **Example:** ```sql -SELECT DATE_ADD(DATE '2025-01-10', INTERVAL 1 MONTH) AS next_month; +SELECT DATE_ADD('2025-01-10'::DATE, INTERVAL 1 MONTH) AS next_month; -- Result: 2025-02-10 ``` --- ### Function: DATE_SUB / DATESUB -**Description:** Subtract interval from DATE. -**Inputs:** `date_expr` (DATE), `INTERVAL n UNIT` -**Output:** DATE +**Description:** +Subtract interval from `DATE`. + +**Inputs:** +- `date_expr` (`DATE`), `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`) + +**Output:** +- `DATE` + **Example:** ```sql -SELECT DATE_SUB(DATE '2025-01-10', INTERVAL 7 DAY) AS week_before; +SELECT DATE_SUB('2025-01-10'::DATE, INTERVAL 7 DAY) AS week_before; -- Result: 2025-01-03 ``` --- ### Function: DATETIME_ADD / DATETIMEADD -**Description:** Adds interval to DATETIME/TIMESTAMP (ZonedDateTime arithmetic). Use for DATETIME only. -**Inputs:** `datetime_expr` (DATETIME), `INTERVAL n UNIT` (including HOUR|MINUTE|SECOND) -**Output:** DATETIME +**Description:** +Adds interval to `DATETIME` / `TIMESTAMP` + +**Inputs:** +- `datetime_expr` (`DATETIME`), `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) + +**Output:** +- `DATETIME` + **Example:** ```sql -SELECT DATETIME_ADD(TIMESTAMP '2025-01-10T12:00:00Z', INTERVAL 1 DAY) AS tomorrow; +SELECT DATETIME_ADD('2025-01-10T12:00:00Z'::TIMESTAMP, INTERVAL 1 DAY) AS tomorrow; -- Result: 2025-01-11T12:00:00Z ``` --- ### Function: DATETIME_SUB / DATETIMESUB -**Description:** Subtract interval from DATETIME/TIMESTAMP. -**Inputs:** `datetime_expr`, `INTERVAL n UNIT` -**Output:** DATETIME +**Description:** +Subtract interval from `DATETIME` / `TIMESTAMP`. + +**Inputs:** +- `datetime_expr`, `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) + +**Output:** +- `DATETIME` + **Example:** ```sql -SELECT DATETIME_SUB(TIMESTAMP '2025-01-10T12:00:00Z', INTERVAL 2 HOUR) AS earlier; +SELECT DATETIME_SUB('2025-01-10T12:00:00Z'::TIMESTAMP, INTERVAL 2 HOUR) AS earlier; +-- Result: 2025-01-10T10:00:00Z ``` --- ### Function: DATEDIFF / DATE_DIFF -**Description:** Difference in days (date1 - date2). -**Inputs:** `date1`, `date2` (DATE or DATETIME) -**Output:** BIGINT +**Description:** +Difference between 2 dates (date1 - date2) in the specified time unit. + +**Inputs:** +- `date1` (`DATE` or `DATETIME`), `date2` (`DATE` or `DATETIME`), `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) + +**Output:** +- `BIGINT` + **Example:** ```sql -SELECT DATEDIFF(DATE '2025-01-10', DATE '2025-01-01') AS diff; +SELECT DATEDIFF('2025-01-10'::DATE, '2025-01-01'::DATE) AS diff; -- Result: 9 ``` --- ### Function: DATE_FORMAT -**Description:** Format DATE/DATETIME to string using Java DateTimeFormatter. -**Inputs:** `date_expr`, `pattern` -**Output:** VARCHAR +**Description:** +Format `DATE` / `DATETIME` to `VARCHAR`. + +**Inputs:** +- `date_expr`, `pattern` + +**Output:** +- `VARCHAR` + **Example:** ```sql -SELECT DATE_FORMAT(DATE '2025-01-10', 'yyyy-MM-dd') AS fmt; +SELECT DATE_FORMAT('2025-01-10'::DATE, 'yyyy-MM-dd') AS fmt; -- Result: '2025-01-10' ``` --- ### Function: DATE_PARSE -**Description:** Parse string into DATE. -**Inputs:** `string`, `pattern` -**Output:** DATE +**Description:** +Parse `VARCHAR` into `DATE`. + +**Inputs:** +- `VARCHAR`, `pattern` + +**Output:** +- `DATE` + **Example:** ```sql SELECT DATE_PARSE('2025-01-10','yyyy-MM-dd') AS d; +-- Result: 2025-01-10 ``` --- ### Function: DATETIME_PARSE -**Description:** Parse string into DATETIME/TIMESTAMP. -**Inputs:** `string`, `pattern` -**Output:** DATETIME (ZonedDateTime) +**Description:** +Parse `VARCHAR` into `DATETIME` / `TIMESTAMP`. + +**Inputs:** +- `VARCHAR`, `pattern` + +**Output:** +- `DATETIME` + **Example:** ```sql -SELECT DATETIME_PARSE('2025-01-10T12:00:00Z','yyyy-MM-dd''T''HH:mm:ssX') AS dt; +SELECT DATETIME_PARSE('2025-01-10T12:00:00Z','yyyy-MM-dd''T''HH:mm:ssZ') AS dt; +-- Result: 2025-01-10T12:00:00Z ``` --- ### Function: DATETIME_FORMAT -**Description:** Format DATETIME/TIMESTAMP to string with pattern. +**Description:** +Format `DATETIME` / `TIMESTAMP` to `VARCHAR` with pattern. + **Inputs:** `datetime_expr`, `pattern` -**Output:** VARCHAR + +**Output:** +- `VARCHAR` + **Example:** ```sql -SELECT DATETIME_FORMAT(TIMESTAMP '2025-01-10T12:00:00Z','yyyy-MM-dd HH:mm:ss') AS s; +SELECT DATETIME_FORMAT('2025-01-10T12:00:00Z'::TIMESTAMP,'yyyy-MM-dd HH:mm:ss') AS s; ``` --- ### Function: DATE_TRUNC -**Description:** Truncate date/datetime to a unit. -**Inputs:** `date_or_datetime_expr`, `unit` -**Output:** DATE or DATETIME +**Description:** +Truncate date/datetime to a `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`). + +**Inputs:** +- `date_or_datetime_expr`, `unit` + +**Output:** +- `DATE` or `DATETIME` + **Example:** ```sql -SELECT DATE_TRUNC(DATE '2025-01-15', MONTH) AS start_month; +SELECT DATE_TRUNC('2025-01-15'::DATE, MONTH) AS start_month; -- Result: 2025-01-01 ``` --- ### Function: EXTRACT -**Description:** Extract field from date or datetime. -**Inputs:** `unit FROM date_expr` -**Output:** INTEGER / BIGINT +**Description:** +Extract field from date or datetime. + +**Inputs:** +- `unit FROM date_expr` + +**Output:** +- `INT` / `BIGINT` + **Example:** ```sql -SELECT EXTRACT(YEAR FROM TIMESTAMP '2025-01-10T12:00:00Z') AS y; +SELECT EXTRACT(YEAR FROM '2025-01-10T12:00:00Z'::TIMESTAMP) AS y; -- Result: 2025 ``` --- ### Function: LAST_DAY -**Description:** Last day of month for a date. -**Inputs:** `date_expr` -**Output:** DATE +**Description:** +Last day of month for a date. + +**Inputs:** +- `date_expr` + +**Output:** +- `DATE` + **Example:** ```sql -SELECT LAST_DAY(DATE '2025-02-15') AS ld; +SELECT LAST_DAY('2025-02-15'::DATE) AS ld; -- Result: 2025-02-28 ``` --- ### Function: WEEK -**Description:** ISO week number (1..53) -**Inputs:** `date_expr` -**Output:** INTEGER +**Description:** +ISO week number (1..53) + +**Inputs:** +- `date_expr` + +**Output:** +- `INT` + **Example:** ```sql -SELECT WEEK(DATE '2025-01-01') AS w; +SELECT WEEK('2025-01-01'::DATE) AS w; -- Result: 1 ``` --- ### Function: QUARTER -**Description:** Quarter number (1..4) -**Inputs:** `date_expr` -**Output:** INTEGER +**Description:** +Quarter number (1..4) + +**Inputs:** +- `date_expr` + +**Output:** +- `INT` + **Example:** ```sql -SELECT QUARTER(DATE '2025-05-10') AS q; +SELECT QUARTER('2025-05-10'::DATE) AS q; -- Result: 2 ``` --- ### Function: NANOSECOND / MICROSECOND / MILLISECOND -**Description:** Sub-second extraction. -**Inputs:** `datetime_expr` -**Output:** INTEGER +**Description:** +Sub-second extraction. + +**Inputs:** +- `datetime_expr` + +**Output:** +- `INT` + **Example:** ```sql -SELECT MILLISECOND(TIMESTAMP '2025-01-01T12:00:00.123Z') AS ms; +SELECT MILLISECOND('2025-01-01T12:00:00.123Z'::TIMESTAMP) AS ms; -- Result: 123 ``` --- ### Function: EPOCHDAY -**Description:** Days since epoch. -**Inputs:** `date_expr` -**Output:** BIGINT +**Description:** +Days since epoch. + +**Inputs:** +- `date_expr` + +**Output:** +- `BIGINT` + **Example:** ```sql -SELECT EPOCHDAY(DATE '1970-01-02') AS d; +SELECT EPOCHDAY('1970-01-02'::DATE) AS d; -- Result: 1 ``` --- ### Function: OFFSET / OFFSET_SECONDS -**Description:** Timezone offset in seconds. -**Inputs:** `timestamp_expr` -**Output:** INTEGER +**Description:** +Timezone offset in seconds. + +**Inputs:** +- `timestamp_expr` + +**Output:** +- `INT` + **Example:** ```sql -SELECT OFFSET(TIMESTAMP '2025-01-01T12:00:00+02:00') AS off; +SELECT OFFSET('2025-01-01T12:00:00+02:00'::TIMESTAMP) AS off; -- Result: 7200 ``` diff --git a/documentation/functions_geo.md b/documentation/functions_geo.md index 96de8e6f..5a107021 100644 --- a/documentation/functions_geo.md +++ b/documentation/functions_geo.md @@ -5,22 +5,40 @@ --- ### Function: ST_DISTANCE (Aliases: DISTANCE) -**Description:** Compute distance between two geo points. Under the hood may use Haversine or ES geo-distance depending on index mapping and push-down capability. -**Inputs:** `point1`, `point2` (WKT or field references) -**Output:** DOUBLE (distance in meters or configured units) -**Example:** +**Description:** + +Computes the geodesic distance (great-circle distance) in meters between two points. + +**Inputs:** + +Each point can be: +- A column of type `geo_point` in Elasticsearch +- A literal defined with `POINT(lat, lon)` + +If both arguments are fixed points, the distance is **precomputed at query compilation time**. + +**Output:** +- `DOUBLE` (distance in meters) + +**Examples:** + +- Distance between a fixed point and a field ```sql -SELECT ST_DISTANCE(location, 'POINT(2.3522 48.8566)') AS dist FROM places; --- Result: e.g., 1234.56 + SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d + FROM locations; ``` - -### Function: ST_WITHIN -**Description:** Test whether point/geometry is within another geometry. -**Inputs:** `geom1`, `geom2` -**Output:** BOOLEAN -**Example:** +- Distance between two fields +```sql +SELECT ST_DISTANCE(fromLocation, toLocation) AS d +FROM locations; +``` +- Distance between two fixed points (precomputed) ```sql -SELECT ST_WITHIN(location, 'POLYGON((...))') FROM places; +SELECT ST_DISTANCE( + POINT(-70.0, 40.0), + POINT(0.0, 0.0) +) AS d; + -- Precomputed result: 8318612.0 (meters) ``` [Back to index](./README.md) diff --git a/documentation/functions_math.md b/documentation/functions_math.md index d7988931..102238c4 100644 --- a/documentation/functions_math.md +++ b/documentation/functions_math.md @@ -7,9 +7,15 @@ --- ### Function: ABS -**Description:** Absolute value. -**Inputs:** `x` (NUMERIC) -**Output:** NUMERIC +**Description:** +Absolute value. + +**Inputs:** +- `x` (`NUMERIC`) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT ABS(-5) AS a; @@ -17,9 +23,14 @@ SELECT ABS(-5) AS a; ``` ### Function: ROUND -**Description:** Round to n decimals (optional). -**Inputs:** `x` (NUMERIC), optional `n` (INTEGER) -**Output:** NUMERIC +**Description:** +Round to n decimals (optional). + +**Inputs:** `x` (`NUMERIC`), optional `n` (`INT`) + +**Output:** +- `DOUBLE` + **Example:** ```sql SELECT ROUND(123.456, 2) AS r; @@ -27,9 +38,15 @@ SELECT ROUND(123.456, 2) AS r; ``` ### Function: FLOOR -**Description:** Greatest integer ≤ x. -**Inputs:** `x` (NUMERIC) -**Output:** INTEGER +**Description:** +Greatest `BIGINT` ≤ x. + +**Inputs:** +- `x` (`NUMERIC`) + +**Output:** +- `BIGINT` + **Example:** ```sql SELECT FLOOR(3.9) AS f; @@ -37,9 +54,15 @@ SELECT FLOOR(3.9) AS f; ``` ### Function: CEIL / CEILING -**Description:** Smallest integer ≥ x. -**Inputs:** `x` (NUMERIC) -**Output:** INTEGER +**Description:** +Smallest `BIGINT` ≥ x. + +**Inputs:** +- `x` (`NUMERIC`) + +**Output:** +- `BIGINT` + **Example:** ```sql SELECT CEIL(3.1) AS c; @@ -47,9 +70,15 @@ SELECT CEIL(3.1) AS c; ``` ### Function: POWER / POW -**Description:** x^y. -**Inputs:** `x` (NUMERIC), `y` (NUMERIC) -**Output:** NUMERIC +**Description:** +x^y. + +**Inputs:** +- `x` (`NUMERIC`), `y` (`NUMERIC`) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT POWER(2, 10) AS p; @@ -57,9 +86,15 @@ SELECT POWER(2, 10) AS p; ``` ### Function: SQRT -**Description:** Square root. -**Inputs:** `x` (NUMERIC >= 0) -**Output:** NUMERIC +**Description:** +Square root. + +**Inputs:** +- `x` (`NUMERIC` >= 0) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT SQRT(16) AS s; @@ -67,9 +102,15 @@ SELECT SQRT(16) AS s; ``` ### Function: LOG / LN -**Description:** Natural logarithm. -**Inputs:** `x` (NUMERIC > 0) -**Output:** NUMERIC +**Description:** +Natural logarithm. + +**Inputs:** +- `x` (`NUMERIC` > 0) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT LOG(EXP(1)) AS l; @@ -77,9 +118,15 @@ SELECT LOG(EXP(1)) AS l; ``` ### Function: LOG10 -**Description:** Base-10 logarithm. -**Inputs:** `x` (NUMERIC > 0) -**Output:** NUMERIC +**Description:** +Base-10 logarithm. + +**Inputs:** +- `x` (`NUMERIC` > 0) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT LOG10(1000) AS l10; @@ -87,9 +134,15 @@ SELECT LOG10(1000) AS l10; ``` ### Function: EXP -**Description:** e^x. -**Inputs:** `x` (NUMERIC) -**Output:** NUMERIC +**Description:** +e^x. + +**Inputs:** +- `x` (`NUMERIC`) + +**Output:** +- `NUMERIC` + **Example:** ```sql SELECT EXP(1) AS e; @@ -97,9 +150,15 @@ SELECT EXP(1) AS e; ``` ### Function: SIGN / SGN -**Description:** Returns -1, 0, or 1 according to sign. -**Inputs:** `x` (NUMERIC) -**Output:** INTEGER +**Description:** +Returns -1, 0, or 1 according to sign. + +**Inputs:** +- `x` (`NUMERIC`) + +**Output:** +- `TINYINT` + **Example:** ```sql SELECT SIGN(-10) AS s; @@ -107,9 +166,15 @@ SELECT SIGN(-10) AS s; ``` ### Trigonometric functions: COS, ACOS, SIN, ASIN, TAN, ATAN, ATAN2 -**Description:** Standard trig functions. Inputs in radians. -**Inputs:** `x` or (`y`, `x` for ATAN2) -**Output:** NUMERIC +**Description:** +Standard trigonometric functions. Inputs in radians. + +**Inputs:** +- `x` or (`y`, `x` for ATAN2) + +**Output:** +- `DOUBLE` + **Example:** ```sql SELECT COS(PI()/3) AS c; diff --git a/documentation/functions_string.md b/documentation/functions_string.md index 8960e652..7b8e1507 100644 --- a/documentation/functions_string.md +++ b/documentation/functions_string.md @@ -5,9 +5,15 @@ --- ### Function: UPPER / UCASE -**Description:** Convert string to upper case. -**Inputs:** `str` (VARCHAR) -**Output:** `VARCHAR` +**Description:** +Convert string to upper case. + +**Inputs:** +- `str` (`VARCHAR`) + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT UPPER('hello') AS up; @@ -15,9 +21,15 @@ SELECT UPPER('hello') AS up; ``` ### Function: LOWER / LCASE -**Description:** Convert string to lower case. -**Inputs:** `str` (VARCHAR) -**Output:** `VARCHAR` +**Description:** +Convert string to lower case. + +**Inputs:** +- `str` (`VARCHAR`) + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT LOWER('Hello') AS lo; @@ -25,9 +37,15 @@ SELECT LOWER('Hello') AS lo; ``` ### Function: TRIM -**Description:** Trim whitespace both sides. -**Inputs:** `str` (VARCHAR) -**Output:** `VARCHAR` +**Description:** +Trim whitespace both sides. + +**Inputs:** +- `str` (`VARCHAR`) + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT TRIM(' abc ') AS t; @@ -35,9 +53,15 @@ SELECT TRIM(' abc ') AS t; ``` ### Function: LENGTH / LEN -**Description:** Character length. -**Inputs:** `str` (VARCHAR) -**Output:** INTEGER +**Description:** +Character length. + +**Inputs:** +- `str` (`VARCHAR`) + +**Output:** +- `BIGINT` + **Example:** ```sql SELECT LENGTH('abc') AS l; @@ -45,9 +69,15 @@ SELECT LENGTH('abc') AS l; ``` ### Function: SUBSTRING / SUBSTR -**Description:** SQL 1-based substring. -**Inputs:** `str` (VARCHAR), `start` (INT >=1), optional `length` (INT) -**Output:** `VARCHAR` +**Description:** +SQL 1-based substring. + +**Inputs:** +- `str` (`VARCHAR`), `start` (`INT` >=1), optional `length` (`INT`) + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT SUBSTRING('abcdef', 2, 3) AS s; @@ -55,18 +85,30 @@ SELECT SUBSTRING('abcdef', 2, 3) AS s; ``` ### Function: CONCAT -**Description:** Concatenate values into a string. -**Inputs:** `expr1, expr2, ...` (coercible to VARCHAR) -**Output:** VARCHAR +**Description:** +Concatenate values into a string. + +**Inputs:** +- `expr1, expr2, ...` (coercible to `VARCHAR`) + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT CONCAT(firstName, ' ', lastName) AS full FROM users; ``` ### Function: REPLACE -**Description:** Replace substring occurrences. -**Inputs:** `str, search, replace` -**Output:** VARCHAR +**Description:** +Replace substring occurrences. + +**Inputs:** +- `str, search, replace` + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; @@ -74,9 +116,15 @@ SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; ``` ### Function: POSITION / STRPOS -**Description:** 1-based index, 0 if not found. -**Inputs:** `substr, str` -**Output:** INTEGER +**Description:** +1-based index, 0 if not found. + +**Inputs:** +- `substr, str` + +**Output:** +- `INT` + **Example:** ```sql SELECT POSITION('lo' IN 'hello') AS pos; @@ -84,9 +132,15 @@ SELECT POSITION('lo' IN 'hello') AS pos; ``` ### Function: REGEXP_LIKE / RLIKE -**Description:** Regex match predicate. -**Inputs:** `str, pattern` -**Output:** BOOLEAN +**Description:** +Regex match predicate. + +**Inputs:** +- `str, pattern` + +**Output:** +- `BOOLEAN` + **Example:** ```sql SELECT REGEXP_LIKE(email, '.*@example\.com') AS ok FROM users; diff --git a/documentation/functions_system.md b/documentation/functions_system.md index f4120f3f..03fccc98 100644 --- a/documentation/functions_system.md +++ b/documentation/functions_system.md @@ -5,22 +5,19 @@ --- ### Function: VERSION -**Description:** Return engine version string. -**Inputs:** none -**Output:** VARCHAR +**Description:** +Return engine version string. + +**Inputs:** +- none + +**Output:** +- `VARCHAR` + **Example:** ```sql SELECT VERSION() AS v; -- Result: 'sql-elasticsearch-engine 1.0.0' ``` -### Function: USER -**Description:** Return current user (context-specific). -**Inputs:** none -**Output:** VARCHAR -**Example:** -```sql -SELECT USER() AS current_user; -``` - [Back to index](./README.md) diff --git a/documentation/functions_type_conversion.md b/documentation/functions_type_conversion.md index 79aa334e..a17e9da2 100644 --- a/documentation/functions_type_conversion.md +++ b/documentation/functions_type_conversion.md @@ -5,9 +5,17 @@ --- ### Function: CAST (Aliases: CONVERT) -**Description:** Cast expression to a target SQL type. -**Inputs:** `expr`, `TYPE` (DATE, TIMESTAMP, VARCHAR, INT, DOUBLE, etc.) -**Output:** `TYPE` +**Description:** + +Cast expression to a target SQL type. + +**Inputs:** +- `expr` +- `TYPE` (`DATE`, `TIMESTAMP`, `VARCHAR`, `INT`, `DOUBLE`, etc.) + +**Output:** +- `TYPE` + **Example:** ```sql SELECT CAST(salary AS DOUBLE) AS s FROM emp; @@ -15,9 +23,18 @@ SELECT CAST(salary AS DOUBLE) AS s FROM emp; ``` ### Function: TRY_CAST (Aliases: none) -**Description:** Attempt a cast and return NULL on failure (safer alternative). -**Inputs:** `expr`, `TYPE` -**Output:** `TYPE` or NULL +**Description:** + +Attempt a cast and return NULL on failure (safer alternative). + +**Inputs:** +- `expr` +- `TYPE` (`DATE`, `TIMESTAMP`, `VARCHAR`, `INT`, `DOUBLE`, etc.) + +**Output:** + +- `TYPE`or `NULL` + **Example:** ```sql SELECT TRY_CAST('not-a-number' AS INT) AS maybe_null; diff --git a/documentation/operators.md b/documentation/operators.md index 89b7fe28..a2ef6134 100644 --- a/documentation/operators.md +++ b/documentation/operators.md @@ -8,38 +8,55 @@ This file provides a per-operator description and a concrete SQL example for eac --- -### Operator: `+` -**Description:** Arithmetic addition. +### Math operators + +#### Operator: `+` +**Description:** + +Arithmetic addition. + **Example:** ```sql SELECT salary + bonus AS total_comp FROM emp; -- result example: if salary=50000 and bonus=10000 -> total_comp = 60000 ``` -### Operator: `-` -**Description:** Arithmetic subtraction or unary negation when used with single operand. +#### Operator: `-` +**Description:** + +Arithmetic subtraction or unary negation when used with single operand. + **Example:** ```sql SELECT salary - tax AS net FROM emp; SELECT -balance AS negative_balance FROM accounts; ``` -### Operator: `*` -**Description:** Multiplication. +#### Operator: `*` +**Description:** + +Multiplication. + **Example:** ```sql SELECT quantity * price AS revenue FROM sales; ``` -### Operator: `/` -**Description:** Division; division by zero must be guarded (NULLIF), engine returns NULL for invalid arithmetic. +#### Operator: `/` +**Description:** + +Division; division by zero must be guarded (NULLIF), engine returns NULL for invalid arithmetic. + **Example:** ```sql -SELECT total / NULLIF(count,0) AS avg FROM table; +SELECT total / NULLIF(count, 0) AS avg FROM table; ``` -### Operator: `%` (MOD) -**Description:** Remainder/modulo operator. +#### Operator: `%` (MOD) +**Description:** + +Remainder/modulo operator. + **Example:** ```sql SELECT id % 10 AS bucket FROM users; @@ -47,74 +64,174 @@ SELECT id % 10 AS bucket FROM users; --- -### Operator: `=` -**Description:** Equality comparison. +### Comparison operators + +#### Operator: `=` +**Description:** + +Equality comparison. + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE department = 'IT'; ``` -### Operator: `<>`, `!=` -**Description:** Inequality comparison (both synonyms supported). +#### Operator: `<>`, `!=` +**Description:** + +Inequality comparison (both synonyms supported). + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE status <> 'terminated'; ``` -### Operator: `<`, `<=`, `>`, `>=` -**Description:** Relational comparisons. +#### Operator: `<`, `<=`, `>`, `>=` +**Description:** + +Relational comparisons. + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE age >= 21 AND age < 65; ``` -### Operator: `IN` -**Description:** Membership in a set of literal or numeric values or results of subquery (subquery support depends on implementation). +#### Operator: `IN` +**Description:** + +Membership in a set of literal or numeric values or results of subquery (subquery support depends on implementation). + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE department IN ('Sales', 'IT', 'HR'); SELECT * FROM emp WHERE status IN (1, 2); ``` -### Operator: `NOT IN` -**Description:** Negated membership. +#### Operator: `NOT IN` +**Description:** + +Negated membership. + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE department NOT IN ('HR','Legal'); ``` -### Operator: `BETWEEN ... AND ...` -**Description:** Inclusive range test: `expr BETWEEN a AND b` ⇔ `expr >= a AND expr <= b`. -**Example:** +#### Operator: `BETWEEN ... AND ...` + +**Description:** + +Checks if an expression lies between two boundaries (inclusive). + +For numeric expressions, `BETWEEN` works as standard SQL. + +For distance expressions (`ST_DISTANCE`), it supports units (`m`, `km`, `mi`, etc.). + +**Return type:** + +- `BOOLEAN` + +**Examples:** + +- Numeric BETWEEN ```sql -SELECT * FROM emp WHERE salary BETWEEN 40000 AND 80000; +SELECT age +FROM users +WHERE age BETWEEN 18 AND 30; ``` -### Operator: `IS NULL` -**Description:** Null check predicate. +- Distance BETWEEN (using meters) + +```sql +SELECT id +FROM locations +WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) +BETWEEN 4000 AND 5000; +``` + +- Distance BETWEEN (with explicit units) + +```sql +SELECT id +FROM locations +WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km; +``` + +👉 In Elasticsearch translation, the last 2 examples are optimized into a combination of: +- a **script filter** for the lower bound +- a `geo_distance` **query** for the upper bound (native ES optimization) + +#### Operator: `IS NULL` +**Description:** + +Null check predicate. + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE manager IS NULL; ``` -### Operator: `IS NOT NULL` -**Description:** Negated null check. +#### Operator: `IS NOT NULL` +**Description:** + +Negated null check. + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM emp WHERE manager IS NOT NULL; ``` ---- +#### Operator: `LIKE` +**Description:** + +Pattern match using `%` and `_`. Engine converts `%` → `.*` and `_` → `.` for underlying regex matching. + +**Return type:** + +- `BOOLEAN` -### Operator: `LIKE` -**Description:** Pattern match using `%` and `_`. Engine converts `%` → `.*` and `_` → `.` for underlying regex matching. **Example:** ```sql SELECT * FROM emp WHERE name LIKE 'Jo%'; ``` -### Operator: `RLIKE` -**Description:** Regular-expression match (Java regex semantics). +#### Operator: `RLIKE` +**Description:** + +Regular-expression match (Java regex semantics). + +**Return type:** + +- `BOOLEAN` + **Example:** ```sql SELECT * FROM users WHERE email RLIKE '.*@example\.com$'; @@ -122,22 +239,33 @@ SELECT * FROM users WHERE email RLIKE '.*@example\.com$'; --- -### Operator: `AND` -**Description:** Logical conjunction. +### Logical operators + +#### Operator: `AND` +**Description:** + +Logical conjunction. + **Example:** ```sql SELECT * FROM emp WHERE dept = 'IT' AND salary > 50000; ``` -### Operator: `OR` -**Description:** Logical disjunction. +#### Operator: `OR` +**Description:** + +Logical disjunction. + **Example:** ```sql SELECT * FROM emp WHERE dept = 'IT' OR dept = 'Sales'; ``` -### Operator: `NOT` -**Description:** Logical negation. +#### Operator: `NOT` +**Description:** + +Logical negation. + **Example:** ```sql SELECT * FROM emp WHERE NOT active; @@ -145,11 +273,25 @@ SELECT * FROM emp WHERE NOT active; --- -### Operator-like: `CAST(...)` / `CONVERT(...)` -**Description:** Type conversion operator/function. See Type Conversion functions page for details and examples. -**Example:** +### Cast operators + +#### Operator : `::` + +**Description:** + +Provides an alternative syntax to the [CAST](./functions_type_conversion.md#function-cast-aliases-convert) function. + +**Inputs:** +- `expr` +- `TYPE` (`DATE`, `TIMESTAMP`, `VARCHAR`, `INT`, `DOUBLE`, etc.) + +**Return type:** + +- `TYPE` + +**Examples:** ```sql -SELECT CAST(hire_date AS DATE) FROM emp; +SELECT hire_date::DATE FROM emp; ``` [Back to index](./README.md) diff --git a/documentation/request_structure.md b/documentation/request_structure.md index 6f12ef3d..8482cbfc 100644 --- a/documentation/request_structure.md +++ b/documentation/request_structure.md @@ -9,7 +9,8 @@ This page documents the SQL clauses supported by the engine and how they map to --- ## SELECT -**Description:** Projection of fields, expressions and computed values. +**Description:** +Projection of fields, expressions and computed values. **Behavior:** - `_source` includes for plain fields. @@ -26,7 +27,8 @@ GROUP BY department; --- ## FROM -**Description:** Source index (one or more). Translates to the Elasticsearch index parameter. +**Description:** +Source index (one or more). Translates to the Elasticsearch index parameter. **Example:** ```sql @@ -36,7 +38,8 @@ SELECT * FROM employees; --- ## UNNEST -**Description:** Expand an array / nested field into rows. Mapped to Elasticsearch `nested` and inner hits where necessary. +**Description:** +Expand an array / nested field into rows. Mapped to Elasticsearch `nested` and inner hits where necessary. **Example:** ```sql @@ -48,7 +51,8 @@ UNNEST(phones) AS phone; --- ## WHERE -**Description:** Row-level predicates. Mapped to `bool` queries; complex expressions become `script` queries (Painless). +**Description:** +Row-level predicates. Mapped to `bool` queries; complex expressions become `script` queries (Painless). **Example:** ```sql @@ -58,7 +62,8 @@ SELECT * FROM emp WHERE salary > 50000 AND department = 'IT'; --- ## GROUP BY -**Description:** Aggregation buckets. Mapped to `terms`/`date_histogram` and nested sub-aggregations. +**Description:** +Aggregation buckets. Mapped to `terms`/`date_histogram` and nested sub-aggregations. Non-aggregated selected fields are disallowed unless included in the `GROUP BY` (standard SQL semantics). **Example:** @@ -71,7 +76,8 @@ GROUP BY department; --- ## HAVING -**Description:** Filter groups using aggregate expressions. Implemented with pipeline aggregations and `bucket_selector` where possible, or client-side filtering if required. +**Description:** +Filter groups using aggregate expressions. Implemented with pipeline aggregations and `bucket_selector` where possible, or client-side filtering if required. **Example:** ```sql @@ -84,7 +90,8 @@ HAVING COUNT(*) > 10; --- ## ORDER BY -**Description:** Sorting of final rows or ordering used inside window/aggregations (pushed to `sort` or `top_hits`). +**Description:** +Sorting of final rows or ordering used inside window/aggregations (pushed to `sort` or `top_hits`). **Example:** ```sql @@ -94,7 +101,8 @@ SELECT name, salary FROM emp ORDER BY salary DESC; --- ## LIMIT / OFFSET -**Description:** Limit and paging. For pure aggregations, `size` is typically set to 0 and `limit` applies to aggregations or outer rows. +**Description:** +Limit and paging. For pure aggregations, `size` is typically set to 0 and `limit` applies to aggregations or outer rows. **Example:** ```sql diff --git a/documentation/type_conversion.md b/documentation/type_conversion.md index cb9b89bc..dbd5da68 100644 --- a/documentation/type_conversion.md +++ b/documentation/type_conversion.md @@ -9,7 +9,7 @@ Converts a value to a specified SQL type. Fails if the conversion is invalid. **Inputs:** - `value` (ANY type) -- `targetType` (SQLType: `INT`, `BIGINT`, `DOUBLE`, `DATE`, `DATETIME`, `TIMESTAMP`, `VARCHAR`, etc.) +- `targetType` (SQL type: `INT`, `BIGINT`, `DOUBLE`, `DATE`, `DATETIME`, `TIMESTAMP`, `VARCHAR`, etc.) **Output:** - `targetType` @@ -29,7 +29,7 @@ Attempts to convert a value to a specified SQL type. Returns `NULL` if the conve **Inputs:** - `value` (ANY type) -- `targetType` (SQLType: `INT`, `BIGINT`, `DOUBLE`, `DATE`, `DATETIME`, etc.) +- `targetType` (SQL type: `INT`, `BIGINT`, `DOUBLE`, `DATE`, `DATETIME`, etc.) **Output:** - `targetType` (nullable) @@ -49,7 +49,7 @@ Converts a value to a specified SQL type. Equivalent to `CAST`, but uses functio **Inputs:** - `value` (ANY type) -- `targetType` (SQLType) +- `targetType` (SQL type) **Output:** - `targetType` diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 18b21008..fea5f7df 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -1,27 +1,37 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{Expr, Identifier, IntValue, PainlessScript, TokenRegex} -import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLType, SQLTypes} package object math { sealed trait MathOp extends PainlessScript with TokenRegex { override def painless: String = s"Math.${sql.toLowerCase()}" override def toString: String = s" $sql " + + override def baseType: SQLNumeric = SQLTypes.Numeric } case object Abs extends Expr("ABS") with MathOp - case object Ceil extends Expr("CEIL") with MathOp - case object Floor extends Expr("FLOOR") with MathOp + case object Ceil extends Expr("CEIL") with MathOp{ + override def baseType: SQLNumeric = SQLTypes.BigInt + } + case object Floor extends Expr("FLOOR") with MathOp{ + override def baseType: SQLNumeric = SQLTypes.BigInt + } case object Round extends Expr("ROUND") with MathOp case object Exp extends Expr("EXP") with MathOp case object Log extends Expr("LOG") with MathOp case object Log10 extends Expr("LOG10") with MathOp case object Pow extends Expr("POW") with MathOp case object Sqrt extends Expr("SQRT") with MathOp - case object Sign extends Expr("SIGN") with MathOp + case object Sign extends Expr("SIGN") with MathOp { + override def baseType: SQLNumeric = SQLTypes.TinyInt + } - sealed trait Trigonometric extends MathOp + sealed trait Trigonometric extends MathOp { + override def baseType: SQLNumeric = SQLTypes.Double + } case object Sin extends Expr("SIN") with Trigonometric case object Asin extends Expr("ASIN") with Trigonometric @@ -36,7 +46,7 @@ package object math { with FunctionWithIdentifier { override def inputType: SQLNumeric = SQLTypes.Numeric - override def outputType: SQLNumeric = SQLTypes.Double + override def outputType: SQLNumeric = mathOp.baseType def mathOp: MathOp @@ -74,8 +84,6 @@ package object math { override def args: List[PainlessScript] = List(arg) - override def outputType: SQLNumeric = SQLTypes.Int - override def painless: String = { val ret = "arg0 > 0 ? 1 : (arg0 < 0 ? -1 : 0)" if (arg.nullable) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 13598915..ac4b2777 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -15,8 +15,12 @@ package object string { case object Pipe extends Expr("\\|\\|") with StringOp { override def painless: String = " + " } - case object Lower extends Expr("LOWER") with StringOp - case object Upper extends Expr("UPPER") with StringOp + case object Lower extends Expr("LOWER") with StringOp { + override lazy val words: List[String] = List(sql, "LCASE") + } + case object Upper extends Expr("UPPER") with StringOp { + override lazy val words: List[String] = List(sql, "UCASE") + } case object Trim extends Expr("TRIM") with StringOp //case object LTrim extends SQLExpr("LTRIM") with SQLStringOperator //case object RTrim extends SQLExpr("RTRIM") with SQLStringOperator @@ -25,7 +29,9 @@ package object string { override lazy val words: List[String] = List(sql, "SUBSTR") } case object TO extends Expr("TO") with TokenRegex - case object Length extends Expr("LENGTH") with StringOp + case object Length extends Expr("LENGTH") with StringOp{ + override lazy val words: List[String] = List(sql, "LEN") + } sealed trait StringFunction[Out <: SQLType] extends TransformFunction[SQLVarchar, Out] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index 01cf69c9..da496a5f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql.parser.function import app.softnetwork.elastic.sql.Identifier import app.softnetwork.elastic.sql.function.aggregate._ import app.softnetwork.elastic.sql.parser.{LimitParser, OrderByParser, Parser} -import app.softnetwork.elastic.sql.query.{Limit, OrderBy} +import app.softnetwork.elastic.sql.query.{FieldSort, Limit, OrderBy} package object aggregate { @@ -30,11 +30,18 @@ package object aggregate { def partition_by: PackratParser[Seq[Identifier]] = PARTITION_BY.regex ~> rep1sep(identifier, separator) + private[this] def over: Parser[(Seq[Identifier], OrderBy, Option[Limit])] = + OVER.regex ~> start ~ partition_by.? ~ orderBy ~ limit.? <~ end ^^ { case _ ~ pb ~ ob ~ l => + (pb.getOrElse(Seq.empty), ob, l) + } + private[this] def top_hits : PackratParser[(Identifier, Seq[Identifier], OrderBy, Option[Limit])] = - start ~ identifier ~ end ~ OVER.regex ~ start ~ partition_by.? ~ orderBy ~ limit.? ~ end ^^ { - case _ ~ id ~ _ ~ _ ~ _ ~ pb ~ ob ~ l ~ _ => - (id, pb.getOrElse(Seq.empty), ob, l) + start ~ identifier ~ end ~ over.? ^^ { case _ ~ id ~ _ ~ o => + o match { + case Some((pb, ob, l)) => (id, pb, ob, l) + case None => (id, Seq.empty, OrderBy(Seq(FieldSort(id.name, order = None))), None) + } } def first_value: PackratParser[TopHitsAggregation] = From 012070ca1f43e6d8ec8d2c2adba5274b66764bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 15:30:31 +0200 Subject: [PATCH 37/48] to fix lint failures + rename Length to LengthOp and SQLLength to Length --- .../elastic/sql/function/math/package.scala | 4 ++-- .../elastic/sql/function/string/package.scala | 8 ++++---- .../elastic/sql/parser/function/string/package.scala | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index fea5f7df..d8ef52c9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -13,10 +13,10 @@ package object math { } case object Abs extends Expr("ABS") with MathOp - case object Ceil extends Expr("CEIL") with MathOp{ + case object Ceil extends Expr("CEIL") with MathOp { override def baseType: SQLNumeric = SQLTypes.BigInt } - case object Floor extends Expr("FLOOR") with MathOp{ + case object Floor extends Expr("FLOOR") with MathOp { override def baseType: SQLNumeric = SQLTypes.BigInt } case object Round extends Expr("ROUND") with MathOp diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index ac4b2777..b965eeca 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -28,8 +28,8 @@ package object string { override def painless: String = ".substring" override lazy val words: List[String] = List(sql, "SUBSTR") } - case object TO extends Expr("TO") with TokenRegex - case object Length extends Expr("LENGTH") with StringOp{ + case object To extends Expr("TO") with TokenRegex + case object LengthOp extends Expr("LENGTH") with StringOp { override lazy val words: List[String] = List(sql, "LEN") } @@ -121,9 +121,9 @@ package object string { override def toSQL(base: String): String = sql } - case object SQLLength extends StringFunction[SQLBigInt] { + case object Length extends StringFunction[SQLBigInt] { override def outputType: SQLBigInt = SQLTypes.BigInt - override def stringOp: StringOp = Length + override def stringOp: StringOp = LengthOp override def args: List[PainlessScript] = List.empty } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index a39155b9..db519a4e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -5,12 +5,12 @@ import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLVarchar} import app.softnetwork.elastic.sql.function.string.{ Concat, Length, + LengthOp, Lower, - SQLLength, StringFunction, StringFunctionWithOp, Substring, - TO, + To, Trim, Upper } @@ -27,7 +27,7 @@ package object string { } def substringFunction: PackratParser[StringFunction[SQLVarchar]] = - Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((TO.regex | separator) ~ long).? ~ end ^^ { + Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((To.regex | separator) ~ long).? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ eOpt ~ _ => Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) } @@ -38,8 +38,8 @@ package object string { } def length: PackratParser[StringFunction[SQLBigInt]] = - Length.regex ^^ { _ => - SQLLength + LengthOp.regex ^^ { _ => + Length } def lower: PackratParser[StringFunction[SQLVarchar]] = From 8d83cfc4560be76a9bf03765f2a02536f5d1c89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Sep 2025 15:56:15 +0200 Subject: [PATCH 38/48] add support for LTRIM and RTRIM string functions --- documentation/functions_string.md | 37 ++++++++++++++++++- .../elastic/sql/SQLQuerySpec.scala | 16 +++++++- .../elastic/sql/SQLQuerySpec.scala | 16 +++++++- .../elastic/sql/function/string/package.scala | 12 ++++-- .../sql/parser/function/string/package.scala | 19 ++++++++-- .../elastic/sql/SQLParserSpec.scala | 2 +- 6 files changed, 89 insertions(+), 13 deletions(-) diff --git a/documentation/functions_string.md b/documentation/functions_string.md index 7b8e1507..ea20e181 100644 --- a/documentation/functions_string.md +++ b/documentation/functions_string.md @@ -52,6 +52,38 @@ SELECT TRIM(' abc ') AS t; -- Result: 'abc' ``` +### Function: LTRIM +**Description:** +Trim whitespace left side. + +**Inputs:** +- `str` (`VARCHAR`) + +**Output:** +- `VARCHAR` + +**Example:** +```sql +SELECT LTRIM(' abc ') AS t; +-- Result: 'abc ' +``` + +### Function: RTRIM +**Description:** +Trim whitespace right side. + +**Inputs:** +- `str` (`VARCHAR`) + +**Output:** +- `VARCHAR` + +**Example:** +```sql +SELECT RTRIM(' abc ') AS t; +-- Result: ' abc' +``` + ### Function: LENGTH / LEN **Description:** Character length. @@ -117,10 +149,11 @@ SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; ### Function: POSITION / STRPOS **Description:** -1-based index, 0 if not found. +1-based index, 0 if not found. +The first position of the `substr` in the `str`. **Inputs:** -- `substr, str` +- `substr` `IN` `str` **Output:** - `INT` diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 6dd17883..a6014c3f 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2454,6 +2454,18 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.trim() : null)" | } | }, + | "ltr": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"^\\\\s+\", \"\") : null)" + | } + | }, + | "rtr": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"\\\\s+$\", \"\") : null)" + | } + | }, | "con": { | "script": { | "lang": "painless", @@ -2488,7 +2500,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(";", "; ") .replaceAll("; if", ";if") .replaceAll("==", " == ") - .replaceAll("\\+", " + ") + .replaceAll("\\+(\\d)", " + $1") + .replaceAll("\\)\\+", ") + ") + .replaceAll("\\+String", " + String") .replaceAll("-", " - ") .replaceAll("\\*", " * ") .replaceAll("/", " / ") diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 1572497d..87de9d53 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2443,6 +2443,18 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.trim() : null)" | } | }, + | "ltr": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"^\\\\s+\", \"\") : null)" + | } + | }, + | "rtr": { + | "script": { + | "lang": "painless", + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"\\\\s+$\", \"\") : null)" + | } + | }, | "con": { | "script": { | "lang": "painless", @@ -2477,7 +2489,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(";", "; ") .replaceAll("; if", ";if") .replaceAll("==", " == ") - .replaceAll("\\+", " + ") + .replaceAll("\\+(\\d)", " + $1") + .replaceAll("\\)\\+", ") + ") + .replaceAll("\\+String", " + String") .replaceAll("-", " - ") .replaceAll("\\*", " * ") .replaceAll("/", " / ") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index b965eeca..5d36a54c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -22,14 +22,18 @@ package object string { override lazy val words: List[String] = List(sql, "UCASE") } case object Trim extends Expr("TRIM") with StringOp - //case object LTrim extends SQLExpr("LTRIM") with SQLStringOperator - //case object RTrim extends SQLExpr("RTRIM") with SQLStringOperator + case object Ltrim extends Expr("LTRIM") with StringOp { + override def painless: String = ".replaceAll(\"^\\\\s+\",\"\")" + } + case object Rtrim extends Expr("RTRIM") with StringOp { + override def painless: String = ".replaceAll(\"\\\\s+$\",\"\")" + } case object Substring extends Expr("SUBSTRING") with StringOp { override def painless: String = ".substring" override lazy val words: List[String] = List(sql, "SUBSTR") } case object To extends Expr("TO") with TokenRegex - case object LengthOp extends Expr("LENGTH") with StringOp { + case object Length extends Expr("LENGTH") with StringOp { override lazy val words: List[String] = List(sql, "LEN") } @@ -121,7 +125,7 @@ package object string { override def toSQL(base: String): String = sql } - case object Length extends StringFunction[SQLBigInt] { + case class Length() extends StringFunction[SQLBigInt] { override def outputType: SQLBigInt = SQLTypes.BigInt override def stringOp: StringOp = LengthOp override def args: List[PainlessScript] = List.empty diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index db519a4e..a997305c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -5,8 +5,9 @@ import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLVarchar} import app.softnetwork.elastic.sql.function.string.{ Concat, Length, - LengthOp, Lower, + Ltrim, + Rtrim, StringFunction, StringFunctionWithOp, Substring, @@ -38,8 +39,8 @@ package object string { } def length: PackratParser[StringFunction[SQLBigInt]] = - LengthOp.regex ^^ { _ => - Length + Length.regex ^^ { _ => + Length() } def lower: PackratParser[StringFunction[SQLVarchar]] = @@ -57,9 +58,19 @@ package object string { StringFunctionWithOp(Trim) } + def ltrim: PackratParser[StringFunction[SQLVarchar]] = + Ltrim.regex ^^ { _ => + StringFunctionWithOp(Ltrim) + } + + def rtrim: PackratParser[StringFunction[SQLVarchar]] = + Rtrim.regex ^^ { _ => + StringFunctionWithOp(Rtrim) + } + def string_functions: Parser[ StringFunction[_] - ] = /*concatFunction | substringFunction |*/ length | lower | upper | trim + ] = /*concatFunction | substringFunction |*/ length | lower | upper | trim | ltrim | rtrim } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index bf125f84..3237a9dd 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -163,7 +163,7 @@ object Queries { "SELECT identifier, (ABS(identifier) + 1.0) * 2, CEIL(identifier), FLOOR(identifier), SQRT(identifier), EXP(identifier), LOG(identifier), LOG10(identifier), POW(identifier, 3), ROUND(identifier), ROUND(identifier, 2), SIGN(identifier), COS(identifier), ACOS(identifier), SIN(identifier), ASIN(identifier), TAN(identifier), ATAN(identifier), ATAN2(identifier, 3.0) FROM Table WHERE SQRT(identifier) > 100.0" val string: String = - "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" + "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, LTRIM(identifier2) AS ltr, RTRIM(identifier2) AS rtr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" val topHits: String = "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary, ARRAY_AGG(name) OVER (PARTITION BY department ORDER BY hire_date ASC, salary DESC LIMIT 1000) AS employees FROM emp" From 6592285cb32f12d00ddbe32b95abb4e3561fc21d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 09:20:36 +0200 Subject: [PATCH 39/48] add support for LEFT, RIGHT, REPLACE, REVERSE, POSITION and REGEXP_LIKE string functions --- documentation/functions_string.md | 79 +++++++- .../elastic/sql/SQLQuerySpec.scala | 48 ++++- .../elastic/sql/SQLQuerySpec.scala | 48 ++++- .../elastic/sql/function/package.scala | 10 +- .../elastic/sql/function/string/package.scala | 180 +++++++++++++++++- .../sql/parser/function/string/package.scala | 60 ++++-- .../elastic/sql/SQLParserSpec.scala | 5 +- 7 files changed, 380 insertions(+), 50 deletions(-) diff --git a/documentation/functions_string.md b/documentation/functions_string.md index ea20e181..cf4d2a8a 100644 --- a/documentation/functions_string.md +++ b/documentation/functions_string.md @@ -105,7 +105,7 @@ SELECT LENGTH('abc') AS l; SQL 1-based substring. **Inputs:** -- `str` (`VARCHAR`), `start` (`INT` >=1), optional `length` (`INT`) +- `str` (`VARCHAR`) `,`|`FROM` `start` (`INT` >= 1) optional `,`|`FOR` `length` (`INT`) **Output:** - `VARCHAR` @@ -114,6 +114,47 @@ SQL 1-based substring. ```sql SELECT SUBSTRING('abcdef', 2, 3) AS s; -- Result: 'bcd' + +SELECT SUBSTRING('abcdef' FROM 2 FOR 3) AS s; +-- Result: 'bcd' + +SELECT SUBSTRING('abcdef' FROM 4) AS s; +-- Result: 'def' +``` + +### Function: LEFT +**Description:** +Leftmost characters. + +**Inputs:** +- `str` (`VARCHAR`) `,`|`FOR` `length` (`INT`) + +**Output:** +- `VARCHAR` + +**Example:** +```sql +SELECT LEFT('abcdef', 3) AS l; +-- Result: 'abc' +``` + +### Function: RIGHT +**Description:** +Rightmost characters. + +**Inputs:** +- `str` (`VARCHAR`) `,`|`FOR` `length` (`INT`) + +**Output:** +- `VARCHAR` + +**Example:** +```sql +SELECT RIGHT('abcdef', 3) AS r; +-- Result: 'def' + +SELECT RIGHT('abcdef' FOR 10) AS r; +-- Result: 'abcdef' ``` ### Function: CONCAT @@ -147,21 +188,43 @@ SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; -- Result: 'John' ``` -### Function: POSITION / STRPOS +### Function: REPLACE **Description:** -1-based index, 0 if not found. -The first position of the `substr` in the `str`. +Replace substring occurrences. **Inputs:** -- `substr` `IN` `str` +- `str, search, replace` -**Output:** -- `INT` +**Output:** +- `VARCHAR` **Example:** ```sql -SELECT POSITION('lo' IN 'hello') AS pos; +SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; +-- Result: 'John' +``` + +### Function: POSITION / STRPOS +**Description:** +1-based index, 0 if not found. +The first position of the `substr` in the `str`, starting at the optional `FROM` position (1-based). + +**Inputs:** +- `substr` `,` | `IN` `str` optional `,` | `FROM` `INT` + +**Output:** +- `BIGINT` + +**Example:** +```sql +SELECT POSITION('lo', 'hello') AS pos; -- Result: 4 + +SELECT POSITION('a' IN 'Elasticsearch' FROM 5) AS pos; +-- Result: 10 + +SELECT POSITION('z' IN 'Elasticsearch') AS pos; +-- Result: 0 ``` ### Function: REGEXP_LIKE / RLIKE diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index a6014c3f..2fdede4f 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2424,7 +2424,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "script_fields": { - | "l": { + | "len": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.length() : null)" @@ -2445,7 +2445,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "sub": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : ((1 - 1) < 0 || (1 - 1 + 3) > arg0.length()) ? null : arg0.substring((1 - 1), (1 - 1 + 3)))" + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.substring(1 - 1, Math.min(1 - 1 + 3, arg0.length())))" | } | }, | "tr": { @@ -2457,13 +2457,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ltr": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"^\\\\s+\", \"\") : null)" + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"^\\\\s+\",\"\") : null)" | } | }, | "rtr": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"\\\\s+$\", \"\") : null)" + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"\\\\s+$\",\"\") : null)" | } | }, | "con": { @@ -2471,6 +2471,42 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lang": "painless", | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : String.valueOf(arg0) + \"_test\" + String.valueOf(1))" | } + | }, + | "l": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.substring(0, Math.min(5, arg0.length())))" + | } + | }, + | "r": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : 3 == 0 ? \"\" : 3 > arg0.length() ? null : arg0.substring(arg0.length() - 3))" + | } + | }, + | "rep": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.replace(\"el\", \"le\"))" + | } + | }, + | "rev": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : new StringBuilder(arg0).reverse().toString())" + | } + | }, + | "pos": { + | "script": { + | "lang": "painless", + | "source": "(def arg1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg1 == null) ? null : arg1.indexOf(\"soft\", 1 - 1) + 1)" + | } + | }, + | "reg": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.matches(\"soft\"))" + | } | } | }, | "_source": { @@ -2513,6 +2549,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll(";\\s\\s", "; ") .replaceAll("false:", "false : ") + .replaceAll("(\\d),", "$1, ") + .replaceAll(":(\\d)", " : $1") + .replaceAll("new", "new ") + .replaceAll(""",\\"le""", """, \\"le""") } it should "handle top hits aggregation" in { diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 87de9d53..d9a799c0 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2413,7 +2413,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | "script_fields": { - | "l": { + | "len": { | "script": { | "lang": "painless", | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.length() : null)" @@ -2434,7 +2434,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "sub": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : ((1 - 1) < 0 || (1 - 1 + 3) > arg0.length()) ? null : arg0.substring((1 - 1), (1 - 1 + 3)))" + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.substring(1 - 1, Math.min(1 - 1 + 3, arg0.length())))" | } | }, | "tr": { @@ -2446,13 +2446,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ltr": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"^\\\\s+\", \"\") : null)" + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"^\\\\s+\",\"\") : null)" | } | }, | "rtr": { | "script": { | "lang": "painless", - | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"\\\\s+$\", \"\") : null)" + | "source": "(def e0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); e0 != null ? e0.replaceAll(\"\\\\s+$\",\"\") : null)" | } | }, | "con": { @@ -2460,6 +2460,42 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lang": "painless", | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : String.valueOf(arg0) + \"_test\" + String.valueOf(1))" | } + | }, + | "l": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.substring(0, Math.min(5, arg0.length())))" + | } + | }, + | "r": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : 3 == 0 ? \"\" : 3 > arg0.length() ? null : arg0.substring(arg0.length() - 3))" + | } + | }, + | "rep": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.replace(\"el\", \"le\"))" + | } + | }, + | "rev": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : new StringBuilder(arg0).reverse().toString())" + | } + | }, + | "pos": { + | "script": { + | "lang": "painless", + | "source": "(def arg1 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg1 == null) ? null : arg1.indexOf(\"soft\", 1 - 1) + 1)" + | } + | }, + | "reg": { + | "script": { + | "lang": "painless", + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.matches(\"soft\"))" + | } | } | }, | "_source": { @@ -2502,6 +2538,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll(";\\s\\s", "; ") .replaceAll("false:", "false : ") + .replaceAll("(\\d),", "$1, ") + .replaceAll(":(\\d)", " : $1") + .replaceAll("new", "new ") + .replaceAll(""",\\"le""", """, \\"le""") } it should "handle top hits aggregation" in { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 336cdd93..ac7d316d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -141,16 +141,14 @@ package object function { override def painless: String = { val nullCheck = - args - .filter(_.nullable) - .zipWithIndex + args.zipWithIndex + .filter(_._1.nullable) .map { case (_, i) => s"arg$i == null" } .mkString(" || ") val assignments = - args - .filter(_.nullable) - .zipWithIndex + args.zipWithIndex + .filter(_._1.nullable) .map { case (a, i) => s"def arg$i = ${SQLTypeUtils.coerce(a.painless, a.baseType, argTypes(i), nullable = false)};" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 5d36a54c..b5566430 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -1,7 +1,14 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{Expr, Identifier, IntValue, PainlessScript, TokenRegex} -import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLType, SQLTypeUtils, SQLTypes, SQLVarchar} +import app.softnetwork.elastic.sql.`type`.{ + SQLBigInt, + SQLBool, + SQLType, + SQLTypeUtils, + SQLTypes, + SQLVarchar +} package object string { @@ -32,10 +39,25 @@ package object string { override def painless: String = ".substring" override lazy val words: List[String] = List(sql, "SUBSTR") } - case object To extends Expr("TO") with TokenRegex + case object LeftOp extends Expr("LEFT") with StringOp + case object RightOp extends Expr("RIGHT") with StringOp + case object For extends Expr("FOR") with TokenRegex case object Length extends Expr("LENGTH") with StringOp { override lazy val words: List[String] = List(sql, "LEN") } + case object Replace extends Expr("REPLACE") with StringOp { + override lazy val words: List[String] = List(sql, "STR_REPLACE") + override def painless: String = ".replace" + } + case object Reverse extends Expr("REVERSE") with StringOp + case object Position extends Expr("POSITION") with StringOp { + override lazy val words: List[String] = List(sql, "STRPOS") + override def painless: String = ".indexOf" + } + case object RegexpLike extends Expr("REGEXP_LIKE") with StringOp { + override lazy val words: List[String] = List(sql, "REGEXP") + override def painless: String = ".matches" + } sealed trait StringFunction[Out <: SQLType] extends TransformFunction[SQLVarchar, Out] @@ -78,11 +100,11 @@ package object string { callArgs match { // SUBSTRING(expr, start, length) case List(arg0, arg1, arg2) => - s"(($arg1 - 1) < 0 || ($arg1 - 1 + $arg2) > $arg0.length()) ? null : $arg0.substring(($arg1 - 1), ($arg1 - 1 + $arg2))" + s"$arg0.substring($arg1 - 1, Math.min($arg1 - 1 + $arg2, $arg0.length()))" // SUBSTRING(expr, start) case List(arg0, arg1) => - s"(($arg1 - 1) < 0 || ($arg1 - 1) >= $arg0.length()) ? null : $arg0.substring(($arg1 - 1))" + s"$arg0.substring(Math.min($arg1 - 1, $arg0.length() - 1))" case _ => throw new IllegalArgumentException("SUBSTRING requires 2 or 3 arguments") } @@ -93,7 +115,8 @@ package object string { Left("SUBSTRING start position must be greater than or equal to 1 (SQL is 1-based)") else if (length.exists(_ < 0)) Left("SUBSTRING length must be non-negative") - else Right(()) + else + str.validate() override def toSQL(base: String): String = sql @@ -120,14 +143,157 @@ package object string { override def validate(): Either[String, Unit] = if (values.isEmpty) Left("CONCAT requires at least one argument") - else Right(()) + else + values.map(_.validate()).find(_.isLeft).getOrElse(Right(())) override def toSQL(base: String): String = sql } case class Length() extends StringFunction[SQLBigInt] { override def outputType: SQLBigInt = SQLTypes.BigInt - override def stringOp: StringOp = LengthOp + override def stringOp: StringOp = Length override def args: List[PainlessScript] = List.empty } + + case class LeftFunction(str: PainlessScript, length: Int) extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def stringOp: StringOp = LeftOp + + override def args: List[PainlessScript] = List(str, IntValue(length)) + + override def nullable: Boolean = str.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0, arg1) => + s"$arg0.substring(0, Math.min($arg1, $arg0.length()))" + case _ => throw new IllegalArgumentException("LEFT requires 2 arguments") + } + } + + override def validate(): Either[String, Unit] = + if (length < 0) + Left("LEFT length must be non-negative") + else + str.validate() + + override def toSQL(base: String): String = sql + } + + case class RightFunction(str: PainlessScript, length: Int) extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def stringOp: StringOp = RightOp + + override def args: List[PainlessScript] = List(str, IntValue(length)) + + override def nullable: Boolean = str.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0, arg1) => + s"""$arg1 == 0 ? "" : $arg1 > $arg0.length() ? null : $arg0.substring($arg0.length() - $arg1)""" + case _ => throw new IllegalArgumentException("RIGHT requires 2 arguments") + } + } + + override def validate(): Either[String, Unit] = + if (length < 0) + Left("RIGHT length must be non-negative") + else + str.validate() + + override def toSQL(base: String): String = sql + } + + case class Replace(str: PainlessScript, search: PainlessScript, replace: PainlessScript) + extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def stringOp: StringOp = Replace + + override def args: List[PainlessScript] = List(str, search, replace) + + override def nullable: Boolean = str.nullable || search.nullable || replace.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0, arg1, arg2) => + s"$arg0.replace($arg1, $arg2)" + case _ => throw new IllegalArgumentException("REPLACE requires 3 arguments") + } + } + + override def validate(): Either[String, Unit] = + args.map(_.validate()).find(_.isLeft).getOrElse(Right(())) + + override def toSQL(base: String): String = sql + } + + case class Reverse(str: PainlessScript) extends StringFunction[SQLVarchar] { + override def outputType: SQLVarchar = SQLTypes.Varchar + override def stringOp: StringOp = Reverse + + override def args: List[PainlessScript] = List(str) + + override def nullable: Boolean = str.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0) => s"new StringBuilder($arg0).reverse().toString()" + case _ => throw new IllegalArgumentException("REVERSE requires 1 argument") + } + } + + override def validate(): Either[String, Unit] = + str.validate() + + override def toSQL(base: String): String = sql + } + + case class Position(substr: PainlessScript, str: PainlessScript, start: Int) + extends StringFunction[SQLBigInt] { + override def outputType: SQLBigInt = SQLTypes.BigInt + override def stringOp: StringOp = Position + + override def args: List[PainlessScript] = List(substr, str, IntValue(start)) + + override def nullable: Boolean = substr.nullable || str.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0, arg1, arg2) => s"$arg1.indexOf($arg0, $arg2 - 1) + 1" + case _ => throw new IllegalArgumentException("POSITION requires 3 arguments") + } + } + + override def validate(): Either[String, Unit] = + if (start < 1) + Left("POSITION start must be greater than or equal to 1 (SQL is 1-based)") + else + str.validate().orElse(substr.validate()) + + override def toSQL(base: String): String = sql + } + + case class RegexpLike(str: PainlessScript, pattern: PainlessScript) + extends StringFunction[SQLBool] { + override def outputType: SQLBool = SQLTypes.Boolean + + override def stringOp: StringOp = RegexpLike + + override def args: List[PainlessScript] = List(str, pattern) + + override def nullable: Boolean = str.nullable || pattern.nullable + + override def toPainlessCall(callArgs: List[String]): String = { + callArgs match { + case List(arg0, arg1) => s"$arg0.matches($arg1)" + case _ => throw new IllegalArgumentException("REGEXP_LIKE requires 2 arguments") + } + } + + override def validate(): Either[String, Unit] = + args.map(_.validate()).find(_.isLeft).getOrElse(Right(())) + + override def toSQL(base: String): String = sql + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index a997305c..0dea7e81 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -1,20 +1,9 @@ package app.softnetwork.elastic.sql.parser.function import app.softnetwork.elastic.sql.Identifier -import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLVarchar} -import app.softnetwork.elastic.sql.function.string.{ - Concat, - Length, - Lower, - Ltrim, - Rtrim, - StringFunction, - StringFunctionWithOp, - Substring, - To, - Trim, - Upper -} +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLBool, SQLVarchar} +import app.softnetwork.elastic.sql.function.string._ +import app.softnetwork.elastic.sql.operator.IN import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.From @@ -22,19 +11,54 @@ package object string { trait StringParser { self: Parser => - def concatFunction: PackratParser[StringFunction[SQLVarchar]] = + def concat: PackratParser[StringFunction[SQLVarchar]] = Concat.regex ~ start ~ rep1sep(valueExpr, separator) ~ end ^^ { case _ ~ _ ~ vs ~ _ => Concat(vs) } - def substringFunction: PackratParser[StringFunction[SQLVarchar]] = - Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((To.regex | separator) ~ long).? ~ end ^^ { + def substr: PackratParser[StringFunction[SQLVarchar]] = + Substring.regex ~ start ~ valueExpr ~ (From.regex | separator) ~ long ~ ((For.regex | separator) ~ long).? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ eOpt ~ _ => Substring(v, s.value.toInt, eOpt.map { case _ ~ e => e.value.toInt }) } + def left: PackratParser[StringFunction[SQLVarchar]] = + LeftOp.regex ~ start ~ valueExpr ~ (For.regex | separator) ~ long ~ end ^^ { + case _ ~ _ ~ v ~ _ ~ l ~ _ => + LeftFunction(v, l.value.toInt) + } + + def right: PackratParser[StringFunction[SQLVarchar]] = + RightOp.regex ~ start ~ valueExpr ~ (For.regex | separator) ~ long ~ end ^^ { + case _ ~ _ ~ v ~ _ ~ l ~ _ => + RightFunction(v, l.value.toInt) + } + + def replace: PackratParser[StringFunction[SQLVarchar]] = + Replace.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ separator ~ valueExpr ~ end ^^ { + case _ ~ _ ~ v ~ _ ~ f ~ _ ~ r ~ _ => + Replace(v, f, r) + } + + def reverse: PackratParser[StringFunction[SQLVarchar]] = + Reverse.regex ~ start ~ valueExpr ~ end ^^ { case _ ~ _ ~ v ~ _ => + Reverse(v) + } + + def position: PackratParser[StringFunction[SQLBigInt]] = + Position.regex ~ start ~ valueExpr ~ (separator | IN.regex) ~ valueExpr ~ ((separator | From.regex) ~ long).? ~ end ^^ { + case _ ~ _ ~ sub ~ _ ~ str ~ from ~ _ => + Position(sub, str, from.map { case _ ~ f => f.value.toInt }.getOrElse(1)) + } + + def regexp: PackratParser[StringFunction[SQLBool]] = + RegexpLike.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ end ^^ { + case _ ~ _ ~ str ~ _ ~ pattern ~ _ => + RegexpLike(str, pattern) + } + def stringFunctionWithIdentifier: PackratParser[Identifier] = - (concatFunction | substringFunction) ^^ { sf => + (concat | substr | left | right | replace | reverse | position | regexp) ^^ { sf => sf.identifier } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 3237a9dd..2495d169 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -163,7 +163,7 @@ object Queries { "SELECT identifier, (ABS(identifier) + 1.0) * 2, CEIL(identifier), FLOOR(identifier), SQRT(identifier), EXP(identifier), LOG(identifier), LOG10(identifier), POW(identifier, 3), ROUND(identifier), ROUND(identifier, 2), SIGN(identifier), COS(identifier), ACOS(identifier), SIN(identifier), ASIN(identifier), TAN(identifier), ATAN(identifier), ATAN2(identifier, 3.0) FROM Table WHERE SQRT(identifier) > 100.0" val string: String = - "SELECT identifier, LENGTH(identifier2) AS l, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, LTRIM(identifier2) AS ltr, RTRIM(identifier2) AS rtr, CONCAT(identifier2, '_test', 1) AS con FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" + "SELECT identifier, LENGTH(identifier2) AS len, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, LTRIM(identifier2) AS ltr, RTRIM(identifier2) AS rtr, CONCAT(identifier2, '_test', 1) AS con, LEFT(identifier2, 5) AS l, RIGHT(identifier2, 3) AS r, REPLACE(identifier2, 'el', 'le') AS rep, REVERSE(identifier2) AS rev, POSITION('soft', identifier2, 1) AS pos, REGEXP_LIKE(identifier2, 'soft') AS reg FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" val topHits: String = "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary, ARRAY_AGG(name) OVER (PARTITION BY department ORDER BY hire_date ASC, salary DESC LIMIT 1000) AS employees FROM emp" @@ -805,8 +805,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { val result = Parser(string) result.toOption .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") - .equalsIgnoreCase(string) shouldBe true + .getOrElse("") shouldBe string } it should "parse top hits functions" in { From 7cc4dc49699758d440d524f5655824ae3c10e094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 10:49:41 +0200 Subject: [PATCH 40/48] update RIGHT and REGEXP_LIKE string functions --- documentation/functions_string.md | 43 +++++++++++++------ .../elastic/sql/SQLQuerySpec.scala | 7 ++- .../elastic/sql/SQLQuerySpec.scala | 7 ++- .../elastic/sql/function/string/package.scala | 37 +++++++++++++--- .../sql/parser/function/string/package.scala | 13 ++++-- .../elastic/sql/SQLParserSpec.scala | 2 +- 6 files changed, 81 insertions(+), 28 deletions(-) diff --git a/documentation/functions_string.md b/documentation/functions_string.md index cf4d2a8a..8a918bf4 100644 --- a/documentation/functions_string.md +++ b/documentation/functions_string.md @@ -124,7 +124,7 @@ SELECT SUBSTRING('abcdef' FROM 4) AS s; ### Function: LEFT **Description:** -Leftmost characters. +Returns the leftmost characters from a string. **Inputs:** - `str` (`VARCHAR`) `,`|`FOR` `length` (`INT`) @@ -140,7 +140,10 @@ SELECT LEFT('abcdef', 3) AS l; ### Function: RIGHT **Description:** -Rightmost characters. +Returns the rightmost characters from a string. +If `length` exceeds the string size, the implementation returns the full string. +If `length = 0`, an empty string is returned. +If `length < 0`, a validation error is raised. **Inputs:** - `str` (`VARCHAR`) `,`|`FOR` `length` (`INT`) @@ -174,7 +177,7 @@ SELECT CONCAT(firstName, ' ', lastName) AS full FROM users; ### Function: REPLACE **Description:** -Replace substring occurrences. +Replaces all occurrences of a substring with another substring. **Inputs:** - `str, search, replace` @@ -188,26 +191,27 @@ SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; -- Result: 'John' ``` -### Function: REPLACE +### Function: REVERSE **Description:** -Replace substring occurrences. +Reverses the characters in a string. **Inputs:** -- `str, search, replace` +- `str` (`VARCHAR`) **Output:** - `VARCHAR` **Example:** ```sql -SELECT REPLACE('Mr. John', 'Mr. ', '') AS r; --- Result: 'John' +SELECT REVERSE('abcdef') AS r; +-- Result: 'fedcba' ``` ### Function: POSITION / STRPOS **Description:** -1-based index, 0 if not found. -The first position of the `substr` in the `str`, starting at the optional `FROM` position (1-based). +Returns the 1-based position of the first occurrence of a substring in a string. +If the substring is not found, returns 0. +An optional FROM position (1-based) can be provided to start the search. **Inputs:** - `substr` `,` | `IN` `str` optional `,` | `FROM` `INT` @@ -229,17 +233,28 @@ SELECT POSITION('z' IN 'Elasticsearch') AS pos; ### Function: REGEXP_LIKE / RLIKE **Description:** -Regex match predicate. +`REGEXP_LIKE(string, pattern [, match_param])` + +Returns `TRUE` if the input string matches the regular expression `pattern`. +By default, the match is case-sensitive. **Inputs:** -- `str, pattern` +- `string`: The input string to test. +- `pattern`: A regular expression pattern. +- `match_param` *(optional)*: A string controlling the regex matching behavior. + - `'i'`: Case-insensitive match. + - `'c'`: Case-sensitive match (default). + - `'m'`: Multi-line mode. + - `'n'`: Allows the `.` to match newline characters. **Output:** - `BOOLEAN` -**Example:** +**Examples:** ```sql -SELECT REGEXP_LIKE(email, '.*@example\.com') AS ok FROM users; +SELECT REGEXP_LIKE('Hello', 'HEL'); -- false +SELECT REGEXP_LIKE('Hello', 'HEL', 'i'); -- true +SELECT REGEXP_LIKE('abc\nxyz', '^xyz', 'm') -- true ``` [Back to index](./README.md) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 2fdede4f..31bceb63 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2481,7 +2481,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "r": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : 3 == 0 ? \"\" : 3 > arg0.length() ? null : arg0.substring(arg0.length() - 3))" + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : 3 == 0 ? \"\" : arg0.substring(arg0.length() - Math.min(3, arg0.length())))" | } | }, | "rep": { @@ -2505,7 +2505,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "reg": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.matches(\"soft\"))" + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : java.util.regex.Pattern.compile(\"soft\", java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.MULTILINE).matcher(arg0).find())" | } | } | }, @@ -2553,6 +2553,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(":(\\d)", " : $1") .replaceAll("new", "new ") .replaceAll(""",\\"le""", """, \\"le""") + .replaceAll(":arg", " : arg") + .replaceAll(",java", ", java") + .replaceAll("\\|java", " | java") } it should "handle top hits aggregation" in { diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index d9a799c0..9d33ce4f 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2470,7 +2470,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "r": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : 3 == 0 ? \"\" : 3 > arg0.length() ? null : arg0.substring(arg0.length() - 3))" + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : 3 == 0 ? \"\" : arg0.substring(arg0.length() - Math.min(3, arg0.length())))" | } | }, | "rep": { @@ -2494,7 +2494,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "reg": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : arg0.matches(\"soft\"))" + | "source": "(def arg0 = (!doc.containsKey('identifier2') || doc['identifier2'].empty ? null : doc['identifier2'].value); (arg0 == null) ? null : java.util.regex.Pattern.compile(\"soft\", java.util.regex.Pattern.CASE_INSENSITIVE | java.util.regex.Pattern.MULTILINE).matcher(arg0).find())" | } | } | }, @@ -2542,6 +2542,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll(":(\\d)", " : $1") .replaceAll("new", "new ") .replaceAll(""",\\"le""", """, \\"le""") + .replaceAll(":arg", " : arg") + .replaceAll(",java", ", java") + .replaceAll("\\|java", " | java") } it should "handle top hits aggregation" in { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index b5566430..90c67664 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -54,11 +54,31 @@ package object string { override lazy val words: List[String] = List(sql, "STRPOS") override def painless: String = ".indexOf" } + case object RegexpLike extends Expr("REGEXP_LIKE") with StringOp { override lazy val words: List[String] = List(sql, "REGEXP") override def painless: String = ".matches" } + case class MatchFlags(flags: String) extends PainlessScript { + override def sql: String = s"'$flags'" + override def painless: String = flags.toCharArray + .map { + case 'i' => "java.util.regex.Pattern.CASE_INSENSITIVE" + case 'c' => "0" + case 'n' => "java.util.regex.Pattern.DOTALL" + case 'm' => "java.util.regex.Pattern.MULTILINE" + case _ => "" + } + .filter(_.nonEmpty) + .mkString(" | ") match { + case "" => "0" + case s => s + } + + override def nullable: Boolean = false + } + sealed trait StringFunction[Out <: SQLType] extends TransformFunction[SQLVarchar, Out] with FunctionWithIdentifier { @@ -191,7 +211,7 @@ package object string { override def toPainlessCall(callArgs: List[String]): String = { callArgs match { case List(arg0, arg1) => - s"""$arg1 == 0 ? "" : $arg1 > $arg0.length() ? null : $arg0.substring($arg0.length() - $arg1)""" + s"""$arg1 == 0 ? "" : $arg0.substring($arg0.length() - Math.min($arg1, $arg0.length()))""" case _ => throw new IllegalArgumentException("RIGHT requires 2 arguments") } } @@ -274,20 +294,25 @@ package object string { override def toSQL(base: String): String = sql } - case class RegexpLike(str: PainlessScript, pattern: PainlessScript) - extends StringFunction[SQLBool] { + case class RegexpLike( + str: PainlessScript, + pattern: PainlessScript, + matchFlags: Option[MatchFlags] = None + ) extends StringFunction[SQLBool] { override def outputType: SQLBool = SQLTypes.Boolean override def stringOp: StringOp = RegexpLike - override def args: List[PainlessScript] = List(str, pattern) + override def args: List[PainlessScript] = List(str, pattern) ++ matchFlags.toList override def nullable: Boolean = str.nullable || pattern.nullable override def toPainlessCall(callArgs: List[String]): String = { callArgs match { - case List(arg0, arg1) => s"$arg0.matches($arg1)" - case _ => throw new IllegalArgumentException("REGEXP_LIKE requires 2 arguments") + case List(arg0, arg1) => s"java.util.regex.Pattern.compile($arg1).matcher($arg0).find()" + case List(arg0, arg1, arg2) => + s"java.util.regex.Pattern.compile($arg1, $arg2).matcher($arg0).find()" + case _ => throw new IllegalArgumentException("REGEXP_LIKE requires 2 or 3 arguments") } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index 0dea7e81..3c6b57f2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -52,9 +52,16 @@ package object string { } def regexp: PackratParser[StringFunction[SQLBool]] = - RegexpLike.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ end ^^ { - case _ ~ _ ~ str ~ _ ~ pattern ~ _ => - RegexpLike(str, pattern) + RegexpLike.regex ~ start ~ valueExpr ~ separator ~ valueExpr ~ (separator ~ literal).? ~ end ^^ { + case _ ~ _ ~ str ~ _ ~ pattern ~ flags ~ _ => + RegexpLike( + str, + pattern, + flags match { + case Some(_ ~ f) => Some(MatchFlags(f.value)) + case _ => None + } + ) } def stringFunctionWithIdentifier: PackratParser[Identifier] = diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 2495d169..60acbea7 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -163,7 +163,7 @@ object Queries { "SELECT identifier, (ABS(identifier) + 1.0) * 2, CEIL(identifier), FLOOR(identifier), SQRT(identifier), EXP(identifier), LOG(identifier), LOG10(identifier), POW(identifier, 3), ROUND(identifier), ROUND(identifier, 2), SIGN(identifier), COS(identifier), ACOS(identifier), SIN(identifier), ASIN(identifier), TAN(identifier), ATAN(identifier), ATAN2(identifier, 3.0) FROM Table WHERE SQRT(identifier) > 100.0" val string: String = - "SELECT identifier, LENGTH(identifier2) AS len, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, LTRIM(identifier2) AS ltr, RTRIM(identifier2) AS rtr, CONCAT(identifier2, '_test', 1) AS con, LEFT(identifier2, 5) AS l, RIGHT(identifier2, 3) AS r, REPLACE(identifier2, 'el', 'le') AS rep, REVERSE(identifier2) AS rev, POSITION('soft', identifier2, 1) AS pos, REGEXP_LIKE(identifier2, 'soft') AS reg FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" + "SELECT identifier, LENGTH(identifier2) AS len, LOWER(identifier2) AS low, UPPER(identifier2) AS upp, SUBSTRING(identifier2, 1, 3) AS sub, TRIM(identifier2) AS tr, LTRIM(identifier2) AS ltr, RTRIM(identifier2) AS rtr, CONCAT(identifier2, '_test', 1) AS con, LEFT(identifier2, 5) AS l, RIGHT(identifier2, 3) AS r, REPLACE(identifier2, 'el', 'le') AS rep, REVERSE(identifier2) AS rev, POSITION('soft', identifier2, 1) AS pos, REGEXP_LIKE(identifier2, 'soft', 'im') AS reg FROM Table WHERE LENGTH(TRIM(identifier2)) > 10" val topHits: String = "SELECT department AS dept, firstName, CAST(hire_date AS DATE) AS hire_date, COUNT(DISTINCT salary) AS cnt, FIRST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS first_salary, LAST_VALUE(salary) OVER (PARTITION BY department ORDER BY hire_date ASC) AS last_salary, ARRAY_AGG(name) OVER (PARTITION BY department ORDER BY hire_date ASC, salary DESC LIMIT 1000) AS employees FROM emp" From fb9c0b14a38ed15b8d221ed6c73a9b33e17ecd86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 10:51:40 +0200 Subject: [PATCH 41/48] add link to SQL engine documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00dcbc4b..5a424715 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ This project provides a trait-based interface (`ElasticClientApi`) that aggregat By relying on these concrete implementations, developers can switch between versions with minimal changes to their business logic. **SQL to Elasticsearch Query Translation** -Elastic Client includes a parser capable of translating SQL `SELECT` queries into Elasticsearch queries. The parser produces an intermediate representation, which is then converted into [Elastic4s](https://github.com/sksamuel/elastic4s) DSL queries and ultimately into native Elasticsearch queries. This allows data engineers and analysts to express queries in familiar SQL syntax. +Elastic Client includes a parser capable of translating SQL `SELECT` queries into Elasticsearch queries. The parser produces an intermediate representation, which is then converted into [Elastic4s](https://github.com/sksamuel/elastic4s) DSL queries and ultimately into native Elasticsearch queries. This allows data engineers and analysts to express queries in familiar [SQL](documentation/README.md) syntax. **Dynamic Mapping Migration** Elastic Client provides tools to analyze and compare existing mappings with new ones. If differences are detected, it can automatically perform safe migrations. This includes creating temporary indices, reindexing, and renaming — all while preserving data integrity. This eliminates the need for manual mapping migrations and reduces downtime. From 4f22fa779e1d28001651f445610d734e53fa4ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 11:53:16 +0200 Subject: [PATCH 42/48] update keywords --- documentation/keywords.md | 152 ++++++++++++++++-- .../elastic/sql/function/math/package.scala | 4 +- .../elastic/sql/time/package.scala | 1 + 3 files changed, 144 insertions(+), 13 deletions(-) diff --git a/documentation/keywords.md b/documentation/keywords.md index eff9a6ac..b25cc15b 100644 --- a/documentation/keywords.md +++ b/documentation/keywords.md @@ -4,17 +4,145 @@ A list of reserved words recognized by the parser for this engine. -``` -SELECT, FROM, WHERE, GROUP BY, HAVING, ORDER BY, UNNEST, AS, CAST, -COUNT, SUM, AVG, MIN, MAX, FIRST_VALUE, LAST_VALUE, ARRAY_AGG, -ROW_NUMBER, RANK, LAG, LEAD, DAY, MONTH, YEAR, HOUR, MINUTE, SECOND, -LAST_DAY, DAYOFWEEK, DAYOFYEAR, WEEK, QUARTER, NANOSECOND, MICROSECOND, -MILLISECOND, EPOCHDAY, OFFSET, UPPER, LOWER, TRIM, LENGTH, SUBSTRING, -CONCAT, POSITION, REGEXP_LIKE, REPLACE, ABS, ROUND, FLOOR, CEIL, POWER, -SQRT, LOG, LOG10, EXP, SIGN, COS, ACOS, SIN, ASIN, TAN, ATAN, ATAN2, -CASE, WHEN, THEN, ELSE, END, COALESCE, ISNULL, ISNOTNULL, NULLIF, -CURRENT_DATE, CURDATE, TODAY, NOW, CURRENT_TIME, CURTIME, CURRENT_TIMESTAMP, -LIKE, RLIKE, IN, NOT IN, BETWEEN, IS NULL, IS NOT NULL, AND, OR, NOT, INTERVAL -``` +## Main clauses +SELECT +FROM +UNNEST +WHERE +GROUP BY +HAVING +ORDER BY +OFFSET + +## Aliases and type conversion +AS +CAST +TRY_CAST +CONVERT +:: + +## Aggregates +COUNT +DISTINCT +SUM +AVG +MIN +MAX +OVER +FIRST_VALUE +LAST_VALUE +ARRAY_AGG + +## String functions +UPPER +UCASE +LOWER +LCASE +TRIM +LTRIM +RTRIM +LENGTH +SUBSTRING +SUBSTR +CONCAT +POSITION +REGEXP_LIKE +REGEXP +REPLACE +REVERSE + +## Math functions +ABS +ROUND +FLOOR +CEIL +POWER +POW +SQRT +LOG +LOG10 +EXP +SIGN +COS +ACOS +SIN +ASIN +TAN +ATAN +ATAN2 + +## Conditional functions +CASE +WHEN +THEN +ELSE +END +COALESCE +ISNULL +ISNOTNULL +NULLIF + +## Date/Time/Datetime/Timestamp functions +YEAR +QUARTER +MONTH +WEEK +DAY +HOUR +MINUTE +SECOND +MILLISECOND +MICROSECOND +NANOSECOND +EPOCHDAY +OFFSET_SECONDS +LAST_DAY +WEEKDAY +YEARDAY +INTERVAL +CURRENT_DATE +CURDATE +TODAY +NOW +CURRENT_TIME +CURTIME +CURRENT_DATETIME +CURRENT_TIMESTAMP +DATE_ADD +DATEADD +DATE_SUB +DATESUB +DATETIME_ADD +DATETIMEADD +DATETIME_SUB +DATETIMESUB +DATE_DIFF +DATEDIFF +DATE_FORMAT +DATE_PARSE +DATETIME_FORMAT +DATETIME_PARSE +DATE_TRUNC +EXTRACT + +## Geo functions +POINT +ST_DISTANCE +DISTANCE + +## Conditional operators +LIKE +RLIKE +IN +NOT IN +BETWEEN +NOT BETWEEN +IS NULL +IS NOT NULL + +## Logical operators +AND +OR +NOT [Back to index](./README.md) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index d8ef52c9..77678461 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -23,7 +23,9 @@ package object math { case object Exp extends Expr("EXP") with MathOp case object Log extends Expr("LOG") with MathOp case object Log10 extends Expr("LOG10") with MathOp - case object Pow extends Expr("POW") with MathOp + case object Pow extends Expr("POW") with MathOp { + override def words: List[String] = List("POWER", sql) + } case object Sqrt extends Expr("SQRT") with MathOp case object Sign extends Expr("SIGN") with MathOp { override def baseType: SQLNumeric = SQLTypes.TinyInt diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index 24c26015..0505a5ce 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -55,6 +55,7 @@ package object time { override val timeField: String = "EPOCH_DAY" } case object OFFSET_SECONDS extends Expr("OFFSET") with TimeField { + override lazy val words: List[String] = List("OFFSET_SECONDS", sql) override val timeField: String = "OFFSET_SECONDS" } } From 0f88cc07cad82a989ce4b50bce36ef4dcc274267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 12:53:18 +0200 Subject: [PATCH 43/48] update documentation --- documentation/functions_aggregate.md | 2 +- documentation/functions_date_time.md | 92 +++++++++++++------ documentation/functions_geo.md | 2 +- documentation/functions_math.md | 8 +- documentation/functions_string.md | 12 +-- documentation/functions_type_conversion.md | 4 +- documentation/keywords.md | 18 ++-- documentation/type_conversion.md | 28 +----- .../elastic/sql/function/math/package.scala | 1 + .../sql/parser/function/time/package.scala | 14 ++- .../elastic/sql/time/package.scala | 3 +- .../elastic/sql/SQLParserSpec.scala | 4 +- 12 files changed, 106 insertions(+), 82 deletions(-) diff --git a/documentation/functions_aggregate.md b/documentation/functions_aggregate.md index 2c8b458b..86b06d69 100644 --- a/documentation/functions_aggregate.md +++ b/documentation/functions_aggregate.md @@ -8,7 +8,7 @@ This page documents aggregate functions. --- -### Function: COUNT (Aliases: COUNT(*)) +### Function: COUNT **Description:** Count rows or non-null expressions. With `DISTINCT` counts distinct values. diff --git a/documentation/functions_date_time.md b/documentation/functions_date_time.md index 40ee21cb..07988f2c 100644 --- a/documentation/functions_date_time.md +++ b/documentation/functions_date_time.md @@ -8,7 +8,7 @@ This page documents TEMPORAL functions. --- -### Function: CURRENT_TIMESTAMP (Aliases: NOW, CURRENT_DATETIME) +### Function: CURRENT_TIMESTAMP (Alias: NOW, CURRENT_DATETIME) **Description:** Returns current datetime (ZonedDateTime) in UTC. @@ -26,7 +26,7 @@ SELECT CURRENT_TIMESTAMP AS now; --- -### Function: CURRENT_DATE (Aliases: CURDATE, TODAY) +### Function: CURRENT_DATE (Alias: CURDATE, TODAY) **Description:** Returns current date as `DATE`. @@ -44,7 +44,7 @@ SELECT CURRENT_DATE AS today; --- -### Function: CURRENT_TIME (Aliases: CURTIME) +### Function: CURRENT_TIME (Alias: CURTIME) **Description:** Returns current time-of-day. @@ -62,12 +62,31 @@ SELECT CURRENT_TIME AS t; --- -### Function: DATE_ADD / DATEADD (Aliases: DATEADD) +### Function: INTERVAL +**Description:** +Literal syntax for time intervals. + +**Inputs:** +- n (`INT`) +- `UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`|`MILLISECOND`|`MICROSECOND`|`NANOSECOND`) + +**Output:** +- `INTERVAL` +- Note: `INTERVAL` is not a standalone type, it can only be used as part of date/datetime arithmetic functions. + +**Example:** +```sql +SELECT DATE_ADD('2025-01-10'::DATE, INTERVAL 1 MONTH); +-- Result: 2025-02-10 +``` + +### Function: DATE_ADD (Alias: DATEADD) **Description:** Adds interval to `DATE`. **Inputs:** -- `date_expr` (`DATE`), `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`) +- `date_expr` (`DATE`) +- `INTERVAL` n (`INT`) `UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`) **Output:** - `DATE` @@ -80,12 +99,13 @@ SELECT DATE_ADD('2025-01-10'::DATE, INTERVAL 1 MONTH) AS next_month; --- -### Function: DATE_SUB / DATESUB +### Function: DATE_SUB (Alias: DATESUB) **Description:** Subtract interval from `DATE`. **Inputs:** -- `date_expr` (`DATE`), `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`) +- `date_expr` (`DATE`) +- `INTERVAL` n (`INT`) `UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`) **Output:** - `DATE` @@ -98,12 +118,13 @@ SELECT DATE_SUB('2025-01-10'::DATE, INTERVAL 7 DAY) AS week_before; --- -### Function: DATETIME_ADD / DATETIMEADD +### Function: DATETIME_ADD (Alias: DATETIMEADD) **Description:** Adds interval to `DATETIME` / `TIMESTAMP` **Inputs:** -- `datetime_expr` (`DATETIME`), `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) +- `datetime_expr` (`DATETIME`) +- `INTERVAL` n (`INT`) `UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) **Output:** - `DATETIME` @@ -116,12 +137,13 @@ SELECT DATETIME_ADD('2025-01-10T12:00:00Z'::TIMESTAMP, INTERVAL 1 DAY) AS tomorr --- -### Function: DATETIME_SUB / DATETIMESUB +### Function: DATETIME_SUB (Alias: DATETIMESUB) **Description:** Subtract interval from `DATETIME` / `TIMESTAMP`. **Inputs:** -- `datetime_expr`, `INTERVAL n UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) +- `datetime_expr` +- `INTERVAL` n (`INT`) `UNIT` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) **Output:** - `DATETIME` @@ -134,12 +156,14 @@ SELECT DATETIME_SUB('2025-01-10T12:00:00Z'::TIMESTAMP, INTERVAL 2 HOUR) AS earli --- -### Function: DATEDIFF / DATE_DIFF +### Function: DATEDIFF (Alias: DATE_DIFF) **Description:** Difference between 2 dates (date1 - date2) in the specified time unit. **Inputs:** -- `date1` (`DATE` or `DATETIME`), `date2` (`DATE` or `DATETIME`), `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) +- `date1` (`DATE` or `DATETIME`) +- `date2` (`DATE` or `DATETIME`), +- optional `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`), `DAY` by default **Output:** - `BIGINT` @@ -154,10 +178,12 @@ SELECT DATEDIFF('2025-01-10'::DATE, '2025-01-01'::DATE) AS diff; ### Function: DATE_FORMAT **Description:** -Format `DATE` / `DATETIME` to `VARCHAR`. +Format `DATE` to `VARCHAR`. **Inputs:** -- `date_expr`, `pattern` +- `date_expr` (`DATE`) +- `pattern` (`VARCHAR`) +- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). **Output:** - `VARCHAR` @@ -175,7 +201,9 @@ SELECT DATE_FORMAT('2025-01-10'::DATE, 'yyyy-MM-dd') AS fmt; Parse `VARCHAR` into `DATE`. **Inputs:** -- `VARCHAR`, `pattern` +- `VARCHAR` +- `pattern` (`VARCHAR`) +- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). **Output:** - `DATE` @@ -193,7 +221,9 @@ SELECT DATE_PARSE('2025-01-10','yyyy-MM-dd') AS d; Parse `VARCHAR` into `DATETIME` / `TIMESTAMP`. **Inputs:** -- `VARCHAR`, `pattern` +- `VARCHAR` +- `pattern` (`VARCHAR`) +- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). **Output:** - `DATETIME` @@ -210,7 +240,10 @@ SELECT DATETIME_PARSE('2025-01-10T12:00:00Z','yyyy-MM-dd''T''HH:mm:ssZ') AS dt; **Description:** Format `DATETIME` / `TIMESTAMP` to `VARCHAR` with pattern. -**Inputs:** `datetime_expr`, `pattern` +**Inputs:** +- `datetime_expr` (`DATETIME` or `TIMESTAMP`) +- `pattern` (`VARCHAR`) +- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). **Output:** - `VARCHAR` @@ -224,10 +257,11 @@ SELECT DATETIME_FORMAT('2025-01-10T12:00:00Z'::TIMESTAMP,'yyyy-MM-dd HH:mm:ss') ### Function: DATE_TRUNC **Description:** -Truncate date/datetime to a `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`). +Truncate date/datetime to a `unit`. **Inputs:** -- `date_or_datetime_expr`, `unit` +- `date_or_datetime_expr` (`DATE` or `DATETIME`) +- `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) **Output:** - `DATE` or `DATETIME` @@ -245,7 +279,7 @@ SELECT DATE_TRUNC('2025-01-15'::DATE, MONTH) AS start_month; Extract field from date or datetime. **Inputs:** -- `unit FROM date_expr` +- `unit` (`YEAR`|`QUARTER`|`MONTH`|`WEEK`|`DAY`|`HOUR`|`MINUTE`|`SECOND`) `FROM` `date_expr` (`DATE` or `DATETIME`) **Output:** - `INT` / `BIGINT` @@ -263,7 +297,7 @@ SELECT EXTRACT(YEAR FROM '2025-01-10T12:00:00Z'::TIMESTAMP) AS y; Last day of month for a date. **Inputs:** -- `date_expr` +- `date_expr` (`DATE`) **Output:** - `DATE` @@ -281,7 +315,7 @@ SELECT LAST_DAY('2025-02-15'::DATE) AS ld; ISO week number (1..53) **Inputs:** -- `date_expr` +- `date_expr` (`DATE`) **Output:** - `INT` @@ -299,7 +333,7 @@ SELECT WEEK('2025-01-01'::DATE) AS w; Quarter number (1..4) **Inputs:** -- `date_expr` +- `date_expr` (`DATE`) **Output:** - `INT` @@ -317,7 +351,7 @@ SELECT QUARTER('2025-05-10'::DATE) AS q; Sub-second extraction. **Inputs:** -- `datetime_expr` +- `datetime_expr` (`DATETIME` or `TIMESTAMP`) **Output:** - `INT` @@ -335,7 +369,7 @@ SELECT MILLISECOND('2025-01-01T12:00:00.123Z'::TIMESTAMP) AS ms; Days since epoch. **Inputs:** -- `date_expr` +- `date_expr` (`DATE`) **Output:** - `BIGINT` @@ -348,19 +382,19 @@ SELECT EPOCHDAY('1970-01-02'::DATE) AS d; --- -### Function: OFFSET / OFFSET_SECONDS +### Function: OFFSET_SECONDS **Description:** Timezone offset in seconds. **Inputs:** -- `timestamp_expr` +- `timestamp_expr` (`TIMESTAMP` with timezone) **Output:** - `INT` **Example:** ```sql -SELECT OFFSET('2025-01-01T12:00:00+02:00'::TIMESTAMP) AS off; +SELECT OFFSET_SECONDS('2025-01-01T12:00:00+02:00'::TIMESTAMP) AS off; -- Result: 7200 ``` diff --git a/documentation/functions_geo.md b/documentation/functions_geo.md index 5a107021..dc62f499 100644 --- a/documentation/functions_geo.md +++ b/documentation/functions_geo.md @@ -4,7 +4,7 @@ --- -### Function: ST_DISTANCE (Aliases: DISTANCE) +### Function: ST_DISTANCE (Alias: DISTANCE) **Description:** Computes the geodesic distance (great-circle distance) in meters between two points. diff --git a/documentation/functions_math.md b/documentation/functions_math.md index 102238c4..d5748d83 100644 --- a/documentation/functions_math.md +++ b/documentation/functions_math.md @@ -53,7 +53,7 @@ SELECT FLOOR(3.9) AS f; -- Result: 3 ``` -### Function: CEIL / CEILING +### Function: CEIL (Alias: CEILING) **Description:** Smallest `BIGINT` ≥ x. @@ -69,7 +69,7 @@ SELECT CEIL(3.1) AS c; -- Result: 4 ``` -### Function: POWER / POW +### Function: POWER (Alias: POW) **Description:** x^y. @@ -101,7 +101,7 @@ SELECT SQRT(16) AS s; -- Result: 4 ``` -### Function: LOG / LN +### Function: LOG (Alias: LN) **Description:** Natural logarithm. @@ -149,7 +149,7 @@ SELECT EXP(1) AS e; -- Result: 2.71828... ``` -### Function: SIGN / SGN +### Function: SIGN (Alias SGN) **Description:** Returns -1, 0, or 1 according to sign. diff --git a/documentation/functions_string.md b/documentation/functions_string.md index 8a918bf4..18be5214 100644 --- a/documentation/functions_string.md +++ b/documentation/functions_string.md @@ -4,7 +4,7 @@ --- -### Function: UPPER / UCASE +### Function: UPPER (Alias: UCASE) **Description:** Convert string to upper case. @@ -20,7 +20,7 @@ SELECT UPPER('hello') AS up; -- Result: 'HELLO' ``` -### Function: LOWER / LCASE +### Function: LOWER (Alias: LCASE) **Description:** Convert string to lower case. @@ -84,7 +84,7 @@ SELECT RTRIM(' abc ') AS t; -- Result: ' abc' ``` -### Function: LENGTH / LEN +### Function: LENGTH (Alias: LEN) **Description:** Character length. @@ -100,7 +100,7 @@ SELECT LENGTH('abc') AS l; -- Result: 3 ``` -### Function: SUBSTRING / SUBSTR +### Function: SUBSTRING (Alias: SUBSTR) **Description:** SQL 1-based substring. @@ -207,7 +207,7 @@ SELECT REVERSE('abcdef') AS r; -- Result: 'fedcba' ``` -### Function: POSITION / STRPOS +### Function: POSITION (Alias: STRPOS) **Description:** Returns the 1-based position of the first occurrence of a substring in a string. If the substring is not found, returns 0. @@ -231,7 +231,7 @@ SELECT POSITION('z' IN 'Elasticsearch') AS pos; -- Result: 0 ``` -### Function: REGEXP_LIKE / RLIKE +### Function: REGEXP_LIKE (Alias: REGEXP) **Description:** `REGEXP_LIKE(string, pattern [, match_param])` diff --git a/documentation/functions_type_conversion.md b/documentation/functions_type_conversion.md index a17e9da2..7040d495 100644 --- a/documentation/functions_type_conversion.md +++ b/documentation/functions_type_conversion.md @@ -4,7 +4,7 @@ --- -### Function: CAST (Aliases: CONVERT) +### Function: CAST (Alias: CONVERT) **Description:** Cast expression to a target SQL type. @@ -22,7 +22,7 @@ SELECT CAST(salary AS DOUBLE) AS s FROM emp; -- Result: 12345.0 ``` -### Function: TRY_CAST (Aliases: none) +### Function: TRY_CAST (Alias: SAFE_CAST) **Description:** Attempt a cast and return NULL on failure (safer alternative). diff --git a/documentation/keywords.md b/documentation/keywords.md index b25cc15b..9e853c00 100644 --- a/documentation/keywords.md +++ b/documentation/keywords.md @@ -15,10 +15,11 @@ ORDER BY OFFSET ## Aliases and type conversion -AS +AS CAST -TRY_CAST CONVERT +TRY_CAST +SAFE_CAST :: ## Aggregates @@ -56,7 +57,8 @@ ABS ROUND FLOOR CEIL -POWER +CEILING +POWER POW SQRT LOG @@ -95,10 +97,10 @@ MILLISECOND MICROSECOND NANOSECOND EPOCHDAY -OFFSET_SECONDS +OFFSET_SECONDS LAST_DAY WEEKDAY -YEARDAY +YEARDAY INTERVAL CURRENT_DATE CURDATE @@ -122,7 +124,7 @@ DATE_FORMAT DATE_PARSE DATETIME_FORMAT DATETIME_PARSE -DATE_TRUNC +DATE_TRUNC EXTRACT ## Geo functions @@ -134,9 +136,9 @@ DISTANCE LIKE RLIKE IN -NOT IN BETWEEN -NOT BETWEEN +NOT IN +NOT BETWEEN IS NULL IS NOT NULL diff --git a/documentation/type_conversion.md b/documentation/type_conversion.md index dbd5da68..ead0e61f 100644 --- a/documentation/type_conversion.md +++ b/documentation/type_conversion.md @@ -2,7 +2,7 @@ # Type Conversion Functions and Operators -## Function: CAST (Alias: NONE) +## Function: CAST (Alias: CONVERT) **Description:** Converts a value to a specified SQL type. Fails if the conversion is invalid. @@ -22,7 +22,7 @@ SELECT CAST('2025-09-11' AS DATE) AS d; --- -## Function: TRY_CAST (Aliases: SAFE_CAST) +## Function: TRY_CAST (Alias: SAFE_CAST) **Description:** Attempts to convert a value to a specified SQL type. Returns `NULL` if the conversion fails instead of raising an error. @@ -42,26 +42,6 @@ SELECT TRY_CAST('invalid-date' AS DATE) AS d; --- -## Function: CONVERT (Alias: NONE) - -**Description:** -Converts a value to a specified SQL type. Equivalent to `CAST`, but uses function syntax instead of `CAST ... AS ...`. - -**Inputs:** -- `value` (ANY type) -- `targetType` (SQL type) - -**Output:** -- `targetType` - -**Example:** -```sql -SELECT CONVERT('125', BIGINT) AS b; --- Result: 125 -``` - ---- - ## Operator: `::` (Cast Operator) **Description:** @@ -84,9 +64,9 @@ SELECT '2025-09-11'::DATE AS d, '125'::BIGINT AS b; ## Behavior Notes -- `CAST` and `CONVERT` will raise errors on invalid conversions. +- `CAST` (`CONVERT`) will raise errors on invalid conversions. - `TRY_CAST` (`SAFE_CAST`) returns `NULL` instead of failing. - `::` is syntactic sugar, easier to read in queries. -- Type inference relies on `baseType`, and explicit `CAST`/`CONVERT`/`::` updates the type context for following functions. +- Type inference relies on `baseType`, and explicit `CAST`/`TRY_CAST`/`::` updates the type context for following functions. [Back to index](./README.md) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 77678461..11b27be6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -14,6 +14,7 @@ package object math { case object Abs extends Expr("ABS") with MathOp case object Ceil extends Expr("CEIL") with MathOp { + override def words: List[String] = List("CEILING", sql) override def baseType: SQLNumeric = SQLTypes.BigInt } case object Floor extends Expr("FLOOR") with MathOp { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index ba7e7e31..ccceea4a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -10,7 +10,7 @@ import app.softnetwork.elastic.sql.function.{ import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.parser.time.TimeParser import app.softnetwork.elastic.sql.parser.{Delimiter, Parser} -import app.softnetwork.elastic.sql.time.{IsoField, TimeField} +import app.softnetwork.elastic.sql.time.{IsoField, TimeField, TimeUnit} package object time { @@ -154,8 +154,16 @@ package object time { self: Parser => def date_diff: PackratParser[BinaryFunction[_, _, _]] = - DateDiff.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ time_unit ~ end ^^ { - case _ ~ _ ~ d1 ~ _ ~ d2 ~ _ ~ u ~ _ => DateDiff(d1, d2, u) + DateDiff.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ separator ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ (separator ~ time_unit).? ~ end ^^ { + case _ ~ _ ~ d1 ~ _ ~ d2 ~ u ~ _ => + DateDiff( + d1, + d2, + u match { + case Some(_ ~ unit) => unit + case None => TimeUnit.DAYS + } + ) } def date_diff_identifier: PackratParser[Identifier] = date_diff ^^ { dd => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index 0505a5ce..f8264441 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -54,8 +54,7 @@ package object time { case object EPOCH_DAY extends Expr("EPOCHDAY") with TimeField { override val timeField: String = "EPOCH_DAY" } - case object OFFSET_SECONDS extends Expr("OFFSET") with TimeField { - override lazy val words: List[String] = List("OFFSET_SECONDS", sql) + case object OFFSET_SECONDS extends Expr("OFFSET_SECONDS") with TimeField { override val timeField: String = "OFFSET_SECONDS" } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 60acbea7..63d28719 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -154,7 +154,7 @@ object Queries { "SELECT CASE CURRENT_DATE - INTERVAL 7 DAY WHEN CAST(lastUpdated AS date) - INTERVAL 3 DAY THEN lastUpdated WHEN lastSeen THEN lastSeen + INTERVAL 2 DAY ELSE createdAt END AS c, identifier FROM Table" val extract: String = - "SELECT EXTRACT(DAY FROM createdAt) AS dom, EXTRACT(WEEKDAY FROM createdAt) AS dow, EXTRACT(YEARDAY FROM createdAt) AS doy, EXTRACT(MONTH FROM createdAt) AS m, EXTRACT(YEAR FROM createdAt) AS y, EXTRACT(HOUR FROM createdAt) AS h, EXTRACT(MINUTE FROM createdAt) AS minutes, EXTRACT(SECOND FROM createdAt) AS s, EXTRACT(NANOSECOND FROM createdAt) AS nano, EXTRACT(MICROSECOND FROM createdAt) AS micro, EXTRACT(MILLISECOND FROM createdAt) AS milli, EXTRACT(EPOCHDAY FROM createdAt) AS epoch, EXTRACT(OFFSET FROM createdAt) AS off, EXTRACT(WEEK FROM createdAt) AS w, EXTRACT(QUARTER FROM createdAt) AS q FROM Table" + "SELECT EXTRACT(DAY FROM createdAt) AS dom, EXTRACT(WEEKDAY FROM createdAt) AS dow, EXTRACT(YEARDAY FROM createdAt) AS doy, EXTRACT(MONTH FROM createdAt) AS m, EXTRACT(YEAR FROM createdAt) AS y, EXTRACT(HOUR FROM createdAt) AS h, EXTRACT(MINUTE FROM createdAt) AS minutes, EXTRACT(SECOND FROM createdAt) AS s, EXTRACT(NANOSECOND FROM createdAt) AS nano, EXTRACT(MICROSECOND FROM createdAt) AS micro, EXTRACT(MILLISECOND FROM createdAt) AS milli, EXTRACT(EPOCHDAY FROM createdAt) AS epoch, EXTRACT(OFFSET_SECONDS FROM createdAt) AS off, EXTRACT(WEEK FROM createdAt) AS w, EXTRACT(QUARTER FROM createdAt) AS q FROM Table" val arithmetic: String = "SELECT identifier, identifier + 1 AS add, identifier - 1 AS sub, identifier * 2 AS mul, identifier / 2 AS div, identifier % 2 AS mod, (identifier * identifier2) - 10 FROM Table WHERE identifier * (EXTRACT(year FROM CURRENT_DATE) - 10) > 10000" @@ -172,7 +172,7 @@ object Queries { "SELECT LAST_DAY(CAST(createdAt AS DATE)) AS ld, identifier FROM Table WHERE EXTRACT(DAY FROM LAST_DAY(CURRENT_TIMESTAMP)) > 28" val extractors: String = - "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" + "SELECT YEAR(createdAt) AS y, MONTH(createdAt) AS m, WEEKDAY(createdAt) AS wd, YEARDAY(createdAt) AS yd, DAY(createdAt) AS d, HOUR(createdAt) AS h, MINUTE(createdAt) AS minutes, SECOND(createdAt) AS s, NANOSECOND(createdAt) AS nano, MICROSECOND(createdAt) AS micro, MILLISECOND(createdAt) AS milli, EPOCHDAY(createdAt) AS epoch, OFFSET_SECONDS(createdAt) AS off, WEEK(createdAt) AS w, QUARTER(createdAt) AS q FROM Table" val geoDistance = "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(-70.0, 40.0)) < 1000 km" From fd596d482966e05891bb60157201a7c8ac36b11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 15:09:44 +0200 Subject: [PATCH 44/48] update date time patterns to use MySQL-style --- documentation/functions_date_time.md | 67 ++++++++++++++++--- .../elastic/sql/SQLQuerySpec.scala | 8 ++- .../elastic/sql/SQLQuerySpec.scala | 8 ++- .../elastic/sql/function/time/package.scala | 65 ++++++++++++++---- .../elastic/sql/type/SQLTypeUtils.scala | 8 ++- .../elastic/sql/SQLParserSpec.scala | 14 ++-- 6 files changed, 135 insertions(+), 35 deletions(-) diff --git a/documentation/functions_date_time.md b/documentation/functions_date_time.md index 07988f2c..b441cafa 100644 --- a/documentation/functions_date_time.md +++ b/documentation/functions_date_time.md @@ -183,15 +183,20 @@ Format `DATE` to `VARCHAR`. **Inputs:** - `date_expr` (`DATE`) - `pattern` (`VARCHAR`) -- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). +- Note: pattern follows [MySQL-style](#supported-mysql-style-datetime-patterns). **Output:** - `VARCHAR` **Example:** ```sql -SELECT DATE_FORMAT('2025-01-10'::DATE, 'yyyy-MM-dd') AS fmt; +-- Simple date formatting +SELECT DATE_FORMAT('2025-01-10'::DATE, '%Y-%m-%d') AS fmt; -- Result: '2025-01-10' + +-- Day of the week (%W) +SELECT DATE_FORMAT('2025-01-10'::DATE, '%W') AS weekday; +-- Result: 'Friday' ``` --- @@ -203,14 +208,19 @@ Parse `VARCHAR` into `DATE`. **Inputs:** - `VARCHAR` - `pattern` (`VARCHAR`) -- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). +- Note: pattern follows [MySQL-style](#supported-mysql-style-datetime-patterns). **Output:** - `DATE` **Example:** ```sql -SELECT DATE_PARSE('2025-01-10','yyyy-MM-dd') AS d; +-- Parse ISO-style date +SELECT DATE_PARSE('2025-01-10','%Y-%m-%d') AS d; +-- Result: 2025-01-10 + +-- Parse with day of week (%W) +SELECT DATE_PARSE('Friday 2025-01-10','%W %Y-%m-%d') AS d; -- Result: 2025-01-10 ``` @@ -223,15 +233,20 @@ Parse `VARCHAR` into `DATETIME` / `TIMESTAMP`. **Inputs:** - `VARCHAR` - `pattern` (`VARCHAR`) -- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). +- Note: pattern follows [MySQL-style](#supported-mysql-style-datetime-patterns). **Output:** - `DATETIME` **Example:** ```sql -SELECT DATETIME_PARSE('2025-01-10T12:00:00Z','yyyy-MM-dd''T''HH:mm:ssZ') AS dt; --- Result: 2025-01-10T12:00:00Z +-- Parse full datetime with microseconds (%f) +SELECT DATETIME_PARSE('2025-01-10 12:00:00.123456','%Y-%m-%d %H:%i:%s.%f') AS dt; +-- Result: 2025-01-10T12:00:00.123456Z + +-- Parse 12-hour clock with AM/PM (%p) +SELECT DATETIME_PARSE('2025-01-10 01:45:30 PM','%Y-%m-%d %h:%i:%s %p') AS dt; +-- Result: 2025-01-10T13:45:30Z ``` --- @@ -243,14 +258,24 @@ Format `DATETIME` / `TIMESTAMP` to `VARCHAR` with pattern. **Inputs:** - `datetime_expr` (`DATETIME` or `TIMESTAMP`) - `pattern` (`VARCHAR`) -- Note: Patterns follow Java DateTimeFormatter syntax (not MySQL-style). +- Note: pattern follows [MySQL-style](#supported-mysql-style-datetime-patterns). **Output:** - `VARCHAR` **Example:** ```sql -SELECT DATETIME_FORMAT('2025-01-10T12:00:00Z'::TIMESTAMP,'yyyy-MM-dd HH:mm:ss') AS s; +-- Format with seconds and microseconds +SELECT DATETIME_FORMAT('2025-01-10T12:00:00.123456Z'::TIMESTAMP,'%Y-%m-%d %H:%i:%s.%f') AS s; +-- Result: '2025-01-10 12:00:00.123456' + +-- Format 12-hour clock with AM/PM +SELECT DATETIME_FORMAT('2025-01-10T13:45:30Z'::TIMESTAMP,'%Y-%m-%d %h:%i:%s %p') AS s; +-- Result: '2025-01-10 01:45:30 PM' + +-- Format with full weekday name +SELECT DATETIME_FORMAT('2025-01-10T13:45:30Z'::TIMESTAMP,'%W, %Y-%m-%d') AS s; +-- Result: 'Friday, 2025-01-10' ``` --- @@ -398,4 +423,28 @@ SELECT OFFSET_SECONDS('2025-01-01T12:00:00+02:00'::TIMESTAMP) AS off; -- Result: 7200 ``` +--- + +### Supported MySQL-style Date/Time Patterns + +| Pattern | Description | Example Output | +|---------|------------------------------|----------------| +| `%Y` | Year (4 digits) | `2025` | +| `%y` | Year (2 digits) | `25` | +| `%m` | Month (2 digits) | `01` | +| `%c` | Month (1–12) | `1` | +| `%M` | Month name (full) | `January` | +| `%b` | Month name (abbrev) | `Jan` | +| `%d` | Day of month (2 digits) | `10` | +| `%e` | Day of month (1–31) | `9` | +| `%W` | Weekday name (full) | `Friday` | +| `%a` | Weekday name (abbrev) | `Fri` | +| `%H` | Hour (00–23) | `13` | +| `%h` | Hour (01–12) | `01` | +| `%I` | Hour (01–12, synonym for %h) | `01` | +| `%i` | Minutes (00–59) | `45` | +| `%s` | Seconds (00–59) | `30` | +| `%f` | Microseconds (000–999) | `123` | +| `%p` | AM/PM marker | `AM` / `PM` | + [Back to index](./README.md) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 31bceb63..e730dce4 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1293,7 +1293,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoField.YEAR) : null)" + | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss.SSS XXX').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoField.YEAR) : null)" | } | } | } @@ -1317,6 +1317,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll(">", " > ") .replaceAll(",ZonedDateTime", ", ZonedDateTime") + .replaceAll("SSSXXX", "SSS XXX") + .replaceAll("ddHH", "dd HH") } it should "handle date_diff function as script field" in { @@ -1383,7 +1385,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def arg1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); (arg0 == null || arg1 == null) ? null : ChronoUnit.DAYS.between(arg0, arg1))" + | "source": "(def arg0 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def arg1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss.SSS XXX').parse(e0, ZonedDateTime::from) : null); (arg0 == null || arg1 == null) ? null : ChronoUnit.DAYS.between(arg0, arg1))" | } | } | } @@ -1408,6 +1410,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("&&", " && ") .replaceAll("\\|\\|", " || ") .replaceAll("ZonedDateTime", " ZonedDateTime") + .replaceAll("SSSXXX", "SSS XXX") + .replaceAll("ddHH", "dd HH") } it should "handle date_add function as script field" in { diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 9d33ce4f..a613cf1b 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1288,7 +1288,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "field": "createdAt", | "script": { | "lang": "painless", - | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoField.YEAR) : null)" + | "source": "(def e2 = (def e1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss.SSS XXX').parse(e0, ZonedDateTime::from) : null); e1 != null ? e1.truncatedTo(ChronoUnit.MINUTES) : null); e2 != null ? e2.get(ChronoField.YEAR) : null)" | } | } | } @@ -1312,6 +1312,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("\\|\\|", " || ") .replaceAll(">", " > ") .replaceAll(",ZonedDateTime", ", ZonedDateTime") + .replaceAll("SSSXXX", "SSS XXX") + .replaceAll("ddHH", "dd HH") } it should "handle date_diff function as script field" in { @@ -1378,7 +1380,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def arg1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-ddTHH:mm:ssZ').parse(e0, ZonedDateTime::from) : null); (arg0 == null || arg1 == null) ? null : ChronoUnit.DAYS.between(arg0, arg1))" + | "source": "(def arg0 = (!doc.containsKey('updatedAt') || doc['updatedAt'].empty ? null : doc['updatedAt'].value); def arg1 = (def e0 = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); e0 != null ? DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss.SSS XXX').parse(e0, ZonedDateTime::from) : null); (arg0 == null || arg1 == null) ? null : ChronoUnit.DAYS.between(arg0, arg1))" | } | } | } @@ -1403,6 +1405,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("&&", " && ") .replaceAll("\\|\\|", " || ") .replaceAll("ZonedDateTime", " ZonedDateTime") + .replaceAll("SSSXXX", "SSS XXX") + .replaceAll("ddHH", "dd HH") } it should "handle date_add function as script field" in { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index d7588d59..deb5691a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -334,6 +334,43 @@ package object time { } } + sealed trait FunctionWithDateTimeFormat { + def format: String + + val sqlToJava: Map[String, String] = Map( + "%Y" -> "yyyy", + "%y" -> "yy", + "%m" -> "MM", + "%c" -> "M", + "%d" -> "dd", + "%e" -> "d", + "%H" -> "HH", + "%h" -> "hh", + "%I" -> "hh", + "%i" -> "mm", + "%s" -> "ss", + "%f" -> "SSS", // microseconds + "%p" -> "a", + "%W" -> "EEEE", + "%a" -> "EEE", + "%M" -> "MMMM", + "%b" -> "MMM" + ) + + def convert(includeTimeZone: Boolean = false): String = { + val basePattern = sqlToJava.foldLeft(format) { case (pattern, (sql, java)) => + pattern.replace(sql, java) + } + + val patternWithTZ = + if (basePattern.contains("Z")) basePattern.replace("Z", "X") + else if (includeTimeZone) s"$basePattern XXX" + else basePattern + + patternWithTZ + } + } + case object DateParse extends Expr("DATE_PARSE") with TokenRegex with PainlessScript { override def painless: String = ".parse" } @@ -341,7 +378,8 @@ package object time { case class DateParse(identifier: Identifier, format: String) extends DateFunction with TransformFunction[SQLVarchar, SQLDate] - with FunctionWithIdentifier { + with FunctionWithIdentifier + with FunctionWithDateTimeFormat { override def fun: Option[PainlessScript] = Some(DateParse) override def args: List[PainlessScript] = List.empty @@ -357,9 +395,9 @@ package object time { override def painless: String = throw new NotImplementedError("Use toPainless instead") override def toPainless(base: String, idx: Int): String = if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').parse(e$idx, LocalDate::from) : null)" + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('${convert()}').parse(e$idx, LocalDate::from) : null)" else - s"DateTimeFormatter.ofPattern('$format').parse($base, LocalDate::from)" + s"DateTimeFormatter.ofPattern('${convert()}').parse($base, LocalDate::from)" } case object DateFormat extends Expr("DATE_FORMAT") with TokenRegex with PainlessScript { @@ -369,7 +407,8 @@ package object time { case class DateFormat(identifier: Identifier, format: String) extends DateFunction with TransformFunction[SQLDate, SQLVarchar] - with FunctionWithIdentifier { + with FunctionWithIdentifier + with FunctionWithDateTimeFormat { override def fun: Option[PainlessScript] = Some(DateFormat) override def args: List[PainlessScript] = List.empty @@ -385,9 +424,9 @@ package object time { override def painless: String = throw new NotImplementedError("Use toPainless instead") override def toPainless(base: String, idx: Int): String = if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').format(e$idx) : null)" + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('${convert()}').format(e$idx) : null)" else - s"DateTimeFormatter.ofPattern('$format').format($base)" + s"DateTimeFormatter.ofPattern('${convert()}').format($base)" } case object DateTimeAdd extends Expr("DATETIME_ADD") with TokenRegex { @@ -431,7 +470,8 @@ package object time { case class DateTimeParse(identifier: Identifier, format: String) extends DateTimeFunction with TransformFunction[SQLVarchar, SQLDateTime] - with FunctionWithIdentifier { + with FunctionWithIdentifier + with FunctionWithDateTimeFormat { override def fun: Option[PainlessScript] = Some(DateTimeParse) override def args: List[PainlessScript] = List.empty @@ -447,9 +487,9 @@ package object time { override def painless: String = throw new NotImplementedError("Use toPainless instead") override def toPainless(base: String, idx: Int): String = if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').parse(e$idx, ZonedDateTime::from) : null)" + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('${convert(includeTimeZone = true)}').parse(e$idx, ZonedDateTime::from) : null)" else - s"DateTimeFormatter.ofPattern('$format').parse($base, ZonedDateTime::from)" + s"DateTimeFormatter.ofPattern('${convert(includeTimeZone = true)}').parse($base, ZonedDateTime::from)" } case object DateTimeFormat extends Expr("DATETIME_FORMAT") with TokenRegex with PainlessScript { @@ -459,7 +499,8 @@ package object time { case class DateTimeFormat(identifier: Identifier, format: String) extends DateTimeFunction with TransformFunction[SQLDateTime, SQLVarchar] - with FunctionWithIdentifier { + with FunctionWithIdentifier + with FunctionWithDateTimeFormat { override def fun: Option[PainlessScript] = Some(DateTimeFormat) override def args: List[PainlessScript] = List.empty @@ -475,9 +516,9 @@ package object time { override def painless: String = throw new NotImplementedError("Use toPainless instead") override def toPainless(base: String, idx: Int): String = if (nullable) - s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('$format').format(e$idx) : null)" + s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('${convert(includeTimeZone = true)}').format(e$idx) : null)" else - s"DateTimeFormatter.ofPattern('$format').format($base)" + s"DateTimeFormatter.ofPattern('${convert(includeTimeZone = true)}').format($base)" } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index b0967826..57159f9e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -133,7 +133,7 @@ object SQLTypeUtils { case (SQLTypes.Boolean, SQLTypes.TinyInt) => s"(byte)($expr ? 1 : 0)" - // ---- VARCHAR -> TEMPORAL ---- + // ---- VARCHAR -> NUMERIC ---- case (SQLTypes.Varchar, SQLTypes.Int) => s"Integer.parseInt($expr).intValue()" case (SQLTypes.Varchar, SQLTypes.BigInt) => @@ -147,12 +147,14 @@ object SQLTypeUtils { case (SQLTypes.Varchar, SQLTypes.TinyInt) => s"Byte.parseByte($expr).byteValue()" - // ---- VARCHAR -> DATE ---- + // ---- VARCHAR -> TEMPORAL ---- case (SQLTypes.Varchar, SQLTypes.Date) => s"LocalDate.parse($expr, DateTimeFormatter.ofPattern('yyyy-MM-dd'))" case (SQLTypes.Varchar, SQLTypes.Time) => s"LocalTime.parse($expr, DateTimeFormatter.ofPattern('HH:mm:ss'))" - case (SQLTypes.Varchar, SQLTypes.DateTime | SQLTypes.Timestamp) => + case (SQLTypes.Varchar, SQLTypes.DateTime) => + s"ZonedDateTime.parse($expr, DateTimeFormatter.ISO_DATE_TIME)" + case (SQLTypes.Varchar, SQLTypes.Timestamp) => s"ZonedDateTime.parse($expr, DateTimeFormatter.ISO_ZONED_DATE_TIME)" // ---- IDENTITY ---- diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 63d28719..461545ab 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -100,7 +100,7 @@ object Queries { |ORDER BY Country ASC""".stripMargin .replaceAll("\n", " ") val dateParse = - "SELECT identifier, COUNT(identifier2) AS ct, MAX(DATE_PARSE(createdAt, 'yyyy-MM-dd')) AS lastSeen FROM Table WHERE identifier2 is NOT null GROUP BY identifier ORDER BY COUNT(identifier2) DESC" + "SELECT identifier, COUNT(identifier2) AS ct, MAX(DATE_PARSE(createdAt, '%Y-%m-%d')) AS lastSeen FROM Table WHERE identifier2 is NOT null GROUP BY identifier ORDER BY COUNT(identifier2) DESC" val dateTimeParse: String = """SELECT identifier, COUNT(identifier2) AS ct, |MAX( @@ -108,7 +108,7 @@ object Queries { |date_trunc( |datetime_parse( |createdAt, - |'yyyy-MM-ddTHH:mm:ssZ' + |'%Y-%m-%d %H:%i:%s.%f' |), MINUTE))) AS lastSeen |FROM Table |WHERE identifier2 is NOT null @@ -121,12 +121,12 @@ object Queries { val dateDiff = "SELECT date_diff(createdAt, updatedAt, DAY) AS diff, identifier FROM Table" val aggregationWithDateDiff = - "SELECT MAX(date_diff(datetime_parse(createdAt, 'yyyy-MM-ddTHH:mm:ssZ'), updatedAt, DAY)) AS max_diff FROM Table GROUP BY identifier" + "SELECT MAX(date_diff(datetime_parse(createdAt, '%Y-%m-%d %H:%i:%s.%f'), updatedAt, DAY)) AS max_diff FROM Table GROUP BY identifier" val dateFormat = - "SELECT identifier, date_format(date_trunc(lastUpdated, MONTH), 'yyyy-MM-dd') AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, date_format(date_trunc(lastUpdated, MONTH), '%Y-%m-%d') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateTimeFormat = - "SELECT identifier, datetime_format(date_trunc(lastUpdated, MONTH), 'yyyy-MM-ddThh:mm:ssZ') AS lastSeen FROM Table WHERE identifier2 is NOT null" + "SELECT identifier, datetime_format(date_trunc(lastUpdated, MONTH), '%Y-%m-%d %H:%i:%s') AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateAdd = "SELECT identifier, date_add(lastUpdated, INTERVAL 10 DAY) AS lastSeen FROM Table WHERE identifier2 is NOT null" val dateSub = @@ -143,9 +143,9 @@ object Queries { val coalesce: String = "SELECT COALESCE(createdAt - INTERVAL 35 MINUTE, CURRENT_DATE) AS c, identifier FROM Table" val nullif: String = - "SELECT COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd') - INTERVAL 2 DAY), CURRENT_DATE) AS c, identifier FROM Table" + "SELECT COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', '%Y-%m-%d') - INTERVAL 2 DAY), CURRENT_DATE) AS c, identifier FROM Table" val conversion: String = - "SELECT TRY_CAST(COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', 'yyyy-MM-dd')), CURRENT_DATE - INTERVAL 2 HOUR) BIGINT) AS c, CONVERT(CURRENT_TIMESTAMP, BIGINT) AS c2, CURRENT_TIMESTAMP::DATE AS c3, '125'::BIGINT AS c4, '2025-09-11'::DATE AS c5, identifier FROM Table" + "SELECT TRY_CAST(COALESCE(NULLIF(createdAt, DATE_PARSE('2025-09-11', '%Y-%m-%d')), CURRENT_DATE - INTERVAL 2 HOUR) BIGINT) AS c, CONVERT(CURRENT_TIMESTAMP, BIGINT) AS c2, CURRENT_TIMESTAMP::DATE AS c3, '125'::BIGINT AS c4, '2025-09-11'::DATE AS c5, identifier FROM Table" val allCasts = "SELECT CAST(identifier AS int) AS c1, CAST(identifier AS bigint) AS c2, CAST(identifier AS double) AS c3, CAST(identifier AS real) AS c4, CAST(identifier AS boolean) AS c5, CAST(identifier AS char) AS c6, CAST(identifier AS varchar) AS c7, CAST(createdAt AS date) AS c8, CAST(createdAt AS time) AS c9, CAST(createdAt AS datetime) AS c10, CAST(createdAt AS timestamp) AS c11, CAST(identifier AS smallint) AS c12, CAST(identifier AS tinyint) AS c13 FROM Table" val caseWhen: String = From 072c619ea01c230312ecb4984ad14803c4d54d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 30 Sep 2025 15:14:53 +0200 Subject: [PATCH 45/48] add date time MySQL-style patterns to convert --- .../elastic/sql/function/time/package.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index deb5691a..b6cba1f8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -345,16 +345,24 @@ package object time { "%d" -> "dd", "%e" -> "d", "%H" -> "HH", + "%k" -> "H", "%h" -> "hh", "%I" -> "hh", + "%l" -> "h", "%i" -> "mm", "%s" -> "ss", + "%S" -> "ss", "%f" -> "SSS", // microseconds "%p" -> "a", "%W" -> "EEEE", "%a" -> "EEE", "%M" -> "MMMM", - "%b" -> "MMM" + "%b" -> "MMM", + "%T" -> "HH:mm:ss", + "%r" -> "hh:mm:ss a", + "%j" -> "DDD", + "%x" -> "YY", + "%X" -> "YYYY" ) def convert(includeTimeZone: Boolean = false): String = { From d5c8d7010b2aaae6d227282498a414d5f5e5bee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 1 Oct 2025 06:29:47 +0200 Subject: [PATCH 46/48] rename MathScript to DateMathScript, add SQLNull type --- .../elastic/sql/bridge/package.scala | 13 +++++++++--- .../elastic/sql/bridge/package.scala | 21 +++++++++++++------ .../elastic/sql/function/package.scala | 6 ++++-- .../elastic/sql/function/time/package.scala | 12 +++++++---- .../elastic/sql/operator/time/package.scala | 4 ++-- .../app/softnetwork/elastic/sql/package.scala | 4 +++- .../elastic/sql/time/package.scala | 6 +++--- .../elastic/sql/type/SQLType.scala | 2 ++ .../elastic/sql/type/SQLTypes.scala | 2 +- 9 files changed, 48 insertions(+), 22 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 073158ae..8ba83420 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble, SQLTemporal, SQLVarchar} import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ @@ -468,17 +468,24 @@ package object bridge { case _ => } val r = - out match { + fromTo.out match { case _: SQLDouble => rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Double] lte fromTo.to.value .asInstanceOf[Double] case _: SQLBigInt => rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Long] lte fromTo.to.value .asInstanceOf[Long] - case _ => + case _: SQLVarchar => rangeQuery(identifier.name) gte String.valueOf(fromTo.from.value) lte String.valueOf( fromTo.to.value ) + case _: SQLTemporal => + // TODO + throw new IllegalArgumentException( + "Range queries on temporal values are not supported yet" + ) + case other => + throw new IllegalArgumentException(s"Unsupported type for range query: $other") } maybeNot match { case Some(_) => not(r) diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 628224c4..ee211a2a 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble} +import app.softnetwork.elastic.sql.`type`.{SQLBigInt, SQLDouble, SQLTemporal, SQLVarchar} import app.softnetwork.elastic.sql.function.aggregate.COUNT import app.softnetwork.elastic.sql.function.geo.{Distance, Meters} import app.softnetwork.elastic.sql.operator._ @@ -468,13 +468,22 @@ package object bridge { case _ => } val r = - out match { + fromTo.out match { case _: SQLDouble => - rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Double] lte fromTo.to.value.asInstanceOf[Double] + rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Double] lte fromTo.to.value + .asInstanceOf[Double] case _: SQLBigInt => - rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Long] lte fromTo.to.value.asInstanceOf[Long] - case _ => - rangeQuery(identifier.name) gte String.valueOf(fromTo.from.value) lte String.valueOf(fromTo.to.value) + rangeQuery(identifier.name) gte fromTo.from.value.asInstanceOf[Long] lte fromTo.to.value + .asInstanceOf[Long] + case _: SQLVarchar => + rangeQuery(identifier.name) gte String.valueOf(fromTo.from.value) lte String.valueOf( + fromTo.to.value + ) + case _: SQLTemporal => + // TODO + throw new IllegalArgumentException("Range queries on temporal values are not supported yet") + case other => + throw new IllegalArgumentException(s"Unsupported type for range query: $other") } maybeNot match { case Some(_) => not(r) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index ac7d316d..5b6ff639 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -63,8 +63,8 @@ package object function { def toScript: Option[String] = { val orderedFunctions = FunctionUtils.transformFunctions(this).reverse orderedFunctions.foldLeft(Option("")) { - case (expr, f: MathScript) if expr.isDefined => Option(s"${expr.get}${f.script}") - case (_, _) => None // ignore non math scripts + case (expr, f: DateMathScript) if expr.isDefined => Option(s"${expr.get}${f.script}") + case (_, _) => None // ignore non math scripts } match { case Some(s) if s.nonEmpty => out match { @@ -75,6 +75,8 @@ package object function { } } + override def dateMathScript: Boolean = toScript.isDefined + override def system: Boolean = functions.lastOption.exists(_.system) def applyTo(expr: Token): Unit = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index b6cba1f8..bc438c02 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Expr, Identifier, MathScript, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{DateMathScript, Expr, Identifier, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.operator.time._ import app.softnetwork.elastic.sql.`type`.{ SQLDate, @@ -18,7 +18,7 @@ package object time { sealed trait IntervalFunction[IO <: SQLTemporal] extends TransformFunction[IO, IO] - with MathScript { + with DateMathScript { def operator: IntervalOperator override def fun: Option[IntervalOperator] = Some(operator) @@ -93,13 +93,13 @@ package object time { sealed trait CurrentDateTimeFunction extends DateTimeFunction with CurrentFunction - with MathScript { + with DateMathScript { override def painless: String = SQLTypeUtils.coerce(now, this.baseType, this.out, nullable = false) override def script: String = "now" } - sealed trait CurrentDateFunction extends DateFunction with CurrentFunction with MathScript { + sealed trait CurrentDateFunction extends DateFunction with CurrentFunction with DateMathScript { override def painless: String = SQLTypeUtils.coerce(s"$now.toLocalDate()", this.baseType, this.out, nullable = false) override def script: String = "now" @@ -315,6 +315,7 @@ package object time { override def toSQL(base: String): String = { s"$sql($base, ${interval.sql})" } + override def dateMathScript: Boolean = identifier.dateMathScript } case object DateSub extends Expr("DATE_SUB") with TokenRegex { @@ -332,6 +333,7 @@ package object time { override def toSQL(base: String): String = { s"$sql($base, ${interval.sql})" } + override def dateMathScript: Boolean = identifier.dateMathScript } sealed trait FunctionWithDateTimeFormat { @@ -452,6 +454,7 @@ package object time { override def toSQL(base: String): String = { s"$sql($base, ${interval.sql})" } + override def dateMathScript: Boolean = identifier.dateMathScript } case object DateTimeSub extends Expr("DATETIME_SUB") with TokenRegex { @@ -469,6 +472,7 @@ package object time { override def toSQL(base: String): String = { s"$sql($base, ${interval.sql})" } + override def dateMathScript: Boolean = identifier.dateMathScript } case object DateTimeParse extends Expr("DATETIME_PARSE") with TokenRegex with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala index c51b5b93..e7ed25b6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala @@ -1,10 +1,10 @@ package app.softnetwork.elastic.sql.operator -import app.softnetwork.elastic.sql.{Expr, MathScript} +import app.softnetwork.elastic.sql.{DateMathScript, Expr} package object time { - sealed trait IntervalOperator extends Operator with BinaryOperator with MathScript { + sealed trait IntervalOperator extends Operator with BinaryOperator with DateMathScript { override def script: String = sql override def toString: String = s" $sql " override def painless: String = this match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 94f90791..f4ff2cdf 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -42,6 +42,7 @@ package object sql { } def system: Boolean = false def nullable: Boolean = !system + def dateMathScript: Boolean = false } trait PainlessScript extends Token { @@ -53,8 +54,9 @@ package object sql { def params: Map[String, Any] } - trait MathScript extends Token { + trait DateMathScript extends Token { def script: String + override def dateMathScript: Boolean = true } trait Updateable extends Token { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index f8264441..232dfeaa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -77,7 +77,7 @@ package object time { } - sealed trait TimeUnit extends PainlessScript with MathScript { + sealed trait TimeUnit extends PainlessScript with DateMathScript { lazy val regex: Regex = s"\\b(?i)$sql(s)?\\b".r def timeUnit: String = sql.toUpperCase() + "S" @@ -124,7 +124,7 @@ package object time { case object Interval extends Expr("INTERVAL") with TokenRegex - sealed trait TimeInterval extends PainlessScript with MathScript { + sealed trait TimeInterval extends PainlessScript with DateMathScript { def value: Int def unit: TimeUnit override def sql: String = s"$Interval $value ${unit.sql}" @@ -148,7 +148,7 @@ package object time { case _ => Left(s"Invalid interval unit $unit for TIME") } case SQLTypes.DateTime => - Right(SQLTypes.Timestamp) + Right(SQLTypes.DateTime) case SQLTypes.Timestamp => Right(SQLTypes.Timestamp) case SQLTypes.Temporal => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index a8a7db8f..444ea830 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -7,6 +7,8 @@ sealed trait SQLType { trait SQLAny extends SQLType +trait SQLNull extends SQLType + trait SQLTemporal extends SQLType trait SQLDate extends SQLTemporal diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 057448dc..0959ba29 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -3,7 +3,7 @@ package app.softnetwork.elastic.sql.`type` object SQLTypes { case object Any extends SQLAny { val typeId = "ANY" } - case object Null extends SQLAny { val typeId = "NULL" } + case object Null extends SQLNull { val typeId = "NULL" } case object Temporal extends SQLTemporal { val typeId = "TEMPORAL" } From ccd8b6a6ffb9f51998a07bc3cc8b78b1575516ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 1 Oct 2025 20:42:50 +0200 Subject: [PATCH 47/48] add support for between with identifiers, support for rounding date math script, add generic support for date math script with identifier, add TokenValue and review FromTo trait, add support for applying multiple intervals --- .../elastic/sql/bridge/ElasticQuery.scala | 2 +- .../elastic/sql/bridge/package.scala | 42 +++++-- .../elastic/sql/SQLQuerySpec.scala | 51 ++++++-- .../elastic/sql/bridge/ElasticQuery.scala | 2 +- .../elastic/sql/bridge/package.scala | 44 +++++-- .../elastic/sql/SQLQuerySpec.scala | 14 +-- .../sql/function/convert/package.scala | 8 +- .../elastic/sql/function/package.scala | 21 +--- .../elastic/sql/function/time/package.scala | 69 +++++++++-- .../elastic/sql/operator/time/package.scala | 2 +- .../app/softnetwork/elastic/sql/package.scala | 115 ++++++++++++++++-- .../elastic/sql/parser/OrderByParser.scala | 2 +- .../elastic/sql/parser/Parser.scala | 32 ++--- .../elastic/sql/parser/WhereParser.scala | 10 +- .../parser/function/aggregate/package.scala | 4 +- .../sql/parser/function/cond/package.scala | 10 +- .../sql/parser/function/convert/package.scala | 43 +++---- .../sql/parser/function/geo/package.scala | 2 + .../sql/parser/function/math/package.scala | 22 ++-- .../sql/parser/function/string/package.scala | 2 +- .../sql/parser/function/time/package.scala | 57 +++++---- .../sql/parser/operator/math/package.scala | 2 +- .../elastic/sql/parser/time/package.scala | 7 +- .../elastic/sql/parser/type/package.scala | 6 + .../elastic/sql/query/Select.scala | 6 +- .../softnetwork/elastic/sql/query/Where.scala | 6 +- .../elastic/sql/time/package.scala | 37 ++++-- .../elastic/sql/type/SQLType.scala | 2 +- .../elastic/sql/SQLParserSpec.scala | 17 +++ 29 files changed, 449 insertions(+), 188 deletions(-) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index bfba52f6..cfeb311f 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -66,7 +66,7 @@ case class ElasticQuery(filter: ElasticFilter) { case isNull: IsNullExpr => isNull case isNotNull: IsNotNullExpr => isNotNull case in: InExpr[_, _] => in - case between: BetweenExpr[_] => between + case between: BetweenExpr => between // case geoDistance: DistanceCriteria => geoDistance case matchExpression: ElasticMatch => matchExpression case isNull: IsNullCriteria => isNull diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 8ba83420..4d3c79df 100644 --- a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -366,7 +366,7 @@ package object bridge { case i: Identifier => operator match { case op: ComparisonOperator => - i.toScript match { + i.script match { case Some(script) => val o = if (maybeNot.isDefined) op.not else op o match { @@ -435,7 +435,7 @@ package object bridge { } implicit def betweenToQuery( - between: BetweenExpr[_] + between: BetweenExpr ): Query = { import between._ // Geo distance special case @@ -480,12 +480,38 @@ package object bridge { fromTo.to.value ) case _: SQLTemporal => - // TODO - throw new IllegalArgumentException( - "Range queries on temporal values are not supported yet" - ) - case other => - throw new IllegalArgumentException(s"Unsupported type for range query: $other") + fromTo match { + case ft: IdentifierFromTo => + (ft.from.script, ft.to.script) match { + case (Some(from), Some(to)) => + rangeQuery(identifier.name) gte from lte to + case (Some(from), None) => + val fq = rangeQuery(identifier.name) gte from + val tq = GenericExpression(identifier, LE, ft.to, None) + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + case (None, Some(to)) => + val fq = GenericExpression(identifier, GE, ft.from, None) + val tq = rangeQuery(identifier.name) lte to + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + case _ => + val fq = GenericExpression(identifier, GE, ft.from, None) + val tq = GenericExpression(identifier, LE, ft.to, None) + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + } + case other => + throw new IllegalArgumentException(s"Unsupported type for range query: $other") + } + case _ => + throw new IllegalArgumentException(s"Unsupported out type for range query: ${fromTo.out}") } maybeNot match { case Some(_) => not(r) diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index e730dce4..0bde0b93 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -989,18 +989,16 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "bool": { | "filter": [ | { - | "script": { - | "script": { - | "lang": "painless", - | "source": "def left = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); left == null ? false : left < ZonedDateTime.now(ZoneId.of('Z')).toLocalTime()" + | "range": { + | "createdAt": { + | "lt": "now/s" | } | } | }, | { - | "script": { - | "script": { - | "lang": "painless", - | "source": "def left = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); left == null ? false : left >= ZonedDateTime.now(ZoneId.of('Z')).toLocalTime().minus(10, ChronoUnit.MINUTES)" + | "range": { + | "createdAt": { + | "gte": "now-10m/s" | } | } | } @@ -3011,4 +3009,41 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") } + it should "handle between with temporal" in { + val select: ElasticSearchRequest = + SQLQuery(betweenTemporal) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "bool": { + | "filter": [ + | { + | "range": { + | "createdAt": { + | "gte": "now-1M/d", + | "lte": "now/d" + | } + | } + | }, + | { + | "range": { + | "lastUpdated": { + | "gte": "2025-09-11||/d", + | "lte": "now/d" + | } + | } + | } + | ] + | } + | }, + | "_source": { + | "includes": [ + | "*" + | ] + | } + |}""".stripMargin + .replaceAll("\\s+", "") + } } diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala index 7ac0a6e3..04c558f2 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -66,7 +66,7 @@ case class ElasticQuery(filter: ElasticFilter) { case isNull: IsNullExpr => isNull case isNotNull: IsNotNullExpr => isNotNull case in: InExpr[_, _] => in - case between: BetweenExpr[_] => between + case between: BetweenExpr => between // case geoDistance: DistanceCriteria => geoDistance case matchExpression: ElasticMatch => matchExpression case isNull: IsNullCriteria => isNull diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index ee211a2a..a93097d8 100644 --- a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -111,7 +111,9 @@ package object bridge { _search scriptfields scriptFields.map { field => scriptField( field.scriptName, - Script(script = field.painless).lang("painless").scriptType("source") + Script(script = field.painless) + .lang("painless") + .scriptType("source") .params(field.identifier.functions.headOption match { case Some(f: PainlessParams) => f.params case _ => Map.empty[String, Any] @@ -366,7 +368,7 @@ package object bridge { case i: Identifier => operator match { case op: ComparisonOperator => - i.toScript match { + i.script match { case Some(script) => val o = if (maybeNot.isDefined) op.not else op o match { @@ -435,7 +437,7 @@ package object bridge { } implicit def betweenToQuery( - between: BetweenExpr[_] + between: BetweenExpr ): Query = { import between._ // Geo distance special case @@ -480,10 +482,38 @@ package object bridge { fromTo.to.value ) case _: SQLTemporal => - // TODO - throw new IllegalArgumentException("Range queries on temporal values are not supported yet") - case other => - throw new IllegalArgumentException(s"Unsupported type for range query: $other") + fromTo match { + case ft: IdentifierFromTo => + (ft.from.script, ft.to.script) match { + case (Some(from), Some(to)) => + rangeQuery(identifier.name) gte from lte to + case (Some(from), None) => + val fq = rangeQuery(identifier.name) gte from + val tq = GenericExpression(identifier, LE, ft.to, None) + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + case (None, Some(to)) => + val fq = GenericExpression(identifier, GE, ft.from, None) + val tq = rangeQuery(identifier.name) lte to + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + case _ => + val fq = GenericExpression(identifier, GE, ft.from, None) + val tq = GenericExpression(identifier, LE, ft.to, None) + maybeNot match { + case Some(_) => return not(fq, tq) + case _ => return must(fq, tq) + } + } + case other => + throw new IllegalArgumentException(s"Unsupported type for range query: $other") + } + case _ => + throw new IllegalArgumentException(s"Unsupported out type for range query: ${fromTo.out}") } maybeNot match { case Some(_) => not(r) diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index a613cf1b..9e8aa2b8 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -986,18 +986,16 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "bool": { | "filter": [ | { - | "script": { - | "script": { - | "lang": "painless", - | "source": "def left = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); left == null ? false : left < ZonedDateTime.now(ZoneId.of('Z')).toLocalTime()" + | "range": { + | "createdAt": { + | "lt": "now/s" | } | } | }, | { - | "script": { - | "script": { - | "lang": "painless", - | "source": "def left = (!doc.containsKey('createdAt') || doc['createdAt'].empty ? null : doc['createdAt'].value); left == null ? false : left >= ZonedDateTime.now(ZoneId.of('Z')).toLocalTime().minus(10, ChronoUnit.MINUTES)" + | "range": { + | "createdAt": { + | "gte": "now-10m/s" | } | } | } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 65bdd5e6..a6be2a65 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -1,11 +1,11 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{Alias, Expr, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{Alias, DateMathRounding, Expr, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} package object convert { - sealed trait Conversion extends TransformFunction[SQLType, SQLType] { + sealed trait Conversion extends TransformFunction[SQLType, SQLType] with DateMathRounding { override def toSQL(base: String): String = sql def value: PainlessScript @@ -28,6 +28,10 @@ package object convert { if (safe) s"try $retWithBrackets catch (Exception e) { return null; }" else ret } + + override def roundingScript: Option[String] = DateMathRounding(targetType) + + override def dateMathScript: Boolean = isTemporal } case object Cast extends Expr("CAST") with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 5b6ff639..172eb749 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator @@ -22,7 +22,7 @@ package object function { def identifier: Identifier } - trait FunctionWithValue[+T] extends Function { + trait FunctionWithValue[+T] extends Function with TokenValue { def value: T } @@ -60,23 +60,6 @@ package object function { fun.toSQL(expr) }) - def toScript: Option[String] = { - val orderedFunctions = FunctionUtils.transformFunctions(this).reverse - orderedFunctions.foldLeft(Option("")) { - case (expr, f: DateMathScript) if expr.isDefined => Option(s"${expr.get}${f.script}") - case (_, _) => None // ignore non math scripts - } match { - case Some(s) if s.nonEmpty => - out match { - case SQLTypes.Date => Some(s"$s/d") - case _ => Some(s) - } - case _ => None - } - } - - override def dateMathScript: Boolean = toScript.isDefined - override def system: Boolean = functions.lastOption.exists(_.system) def applyTo(expr: Token): Unit = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index bc438c02..4a65aecd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -1,6 +1,14 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{DateMathScript, Expr, Identifier, PainlessScript, TokenRegex} +import app.softnetwork.elastic.sql.{ + DateMathRounding, + DateMathScript, + Expr, + Identifier, + PainlessScript, + StringValue, + TokenRegex +} import app.softnetwork.elastic.sql.operator.time._ import app.softnetwork.elastic.sql.`type`.{ SQLDate, @@ -30,7 +38,10 @@ package object time { override def argsSeparator: String = " " override def sql: String = s"$operator${args.map(_.sql).mkString(argsSeparator)}" - override def script: String = s"${operator.script}${interval.script}" + override def script: Option[String] = (operator.script, interval.script) match { + case (Some(op), Some(iv)) => Some(s"$op$iv") + case _ => None + } private[this] var _out: SQLType = outputType @@ -88,21 +99,18 @@ package object time { override def system: Boolean = true } - sealed trait CurrentFunction extends SystemFunction with PainlessScript + sealed trait CurrentFunction extends SystemFunction with PainlessScript with DateMathScript { + override def script: Option[String] = Some("now") + } - sealed trait CurrentDateTimeFunction - extends DateTimeFunction - with CurrentFunction - with DateMathScript { + sealed trait CurrentDateTimeFunction extends DateTimeFunction with CurrentFunction { override def painless: String = SQLTypeUtils.coerce(now, this.baseType, this.out, nullable = false) - override def script: String = "now" } - sealed trait CurrentDateFunction extends DateFunction with CurrentFunction with DateMathScript { + sealed trait CurrentDateFunction extends DateFunction with CurrentFunction { override def painless: String = SQLTypeUtils.coerce(s"$now.toLocalDate()", this.baseType, this.out, nullable = false) - override def script: String = "now" } sealed trait CurrentTimeFunction extends TimeFunction with CurrentFunction { @@ -160,7 +168,8 @@ package object time { case class DateTrunc(identifier: Identifier, unit: TimeUnit) extends DateTimeFunction with TransformFunction[SQLTemporal, SQLTemporal] - with FunctionWithIdentifier { + with FunctionWithIdentifier + with DateMathRounding { override def fun: Option[PainlessScript] = Some(DateTrunc) override def args: List[PainlessScript] = List(unit) @@ -172,6 +181,8 @@ package object time { override def toSQL(base: String): String = { s"$sql($base, ${unit.sql})" } + + override def roundingScript: Option[String] = unit.roundingScript } case object Extract extends Expr("EXTRACT") with TokenRegex with PainlessScript { @@ -389,7 +400,8 @@ package object time { extends DateFunction with TransformFunction[SQLVarchar, SQLDate] with FunctionWithIdentifier - with FunctionWithDateTimeFormat { + with FunctionWithDateTimeFormat + with DateMathScript { override def fun: Option[PainlessScript] = Some(DateParse) override def args: List[PainlessScript] = List.empty @@ -408,6 +420,21 @@ package object time { s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('${convert()}').parse(e$idx, LocalDate::from) : null)" else s"DateTimeFormatter.ofPattern('${convert()}').parse($base, LocalDate::from)" + + override def script: Option[String] = { + val base: String = FunctionUtils + .transformFunctions(identifier) + .reverse + .collectFirst { case s: StringValue => s.value } + .getOrElse(identifier.name) + if (base.nonEmpty) { + Some(s"$base||") + } else { + None + } + } + + override def formatScript: Option[String] = Some(format) } case object DateFormat extends Expr("DATE_FORMAT") with TokenRegex with PainlessScript { @@ -483,7 +510,8 @@ package object time { extends DateTimeFunction with TransformFunction[SQLVarchar, SQLDateTime] with FunctionWithIdentifier - with FunctionWithDateTimeFormat { + with FunctionWithDateTimeFormat + with DateMathScript { override def fun: Option[PainlessScript] = Some(DateTimeParse) override def args: List[PainlessScript] = List.empty @@ -502,6 +530,21 @@ package object time { s"(def e$idx = $base; e$idx != null ? DateTimeFormatter.ofPattern('${convert(includeTimeZone = true)}').parse(e$idx, ZonedDateTime::from) : null)" else s"DateTimeFormatter.ofPattern('${convert(includeTimeZone = true)}').parse($base, ZonedDateTime::from)" + + override def script: Option[String] = { + val base: String = FunctionUtils + .transformFunctions(identifier) + .reverse + .collectFirst { case s: StringValue => s.value } + .getOrElse(identifier.name) + if (base.nonEmpty) { + Some(s"$base||") + } else { + None + } + } + + override def formatScript: Option[String] = Some(format) } case object DateTimeFormat extends Expr("DATETIME_FORMAT") with TokenRegex with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala index e7ed25b6..8a347d43 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/time/package.scala @@ -5,7 +5,7 @@ import app.softnetwork.elastic.sql.{DateMathScript, Expr} package object time { sealed trait IntervalOperator extends Operator with BinaryOperator with DateMathScript { - override def script: String = sql + override def script: Option[String] = Some(sql) override def toString: String = s" $sql " override def painless: String = this match { case PLUS => ".plus" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index f4ff2cdf..cd8b06bb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -2,6 +2,11 @@ package app.softnetwork.elastic import app.softnetwork.elastic.sql.function.aggregate.{MAX, MIN} import app.softnetwork.elastic.sql.function.geo.DistanceUnit +import app.softnetwork.elastic.sql.function.time.{ + CurrentDateFunction, + CurrentDateTimeFunction, + CurrentFunction +} import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.parser.{Validation, Validator} import app.softnetwork.elastic.sql.query._ @@ -43,6 +48,11 @@ package object sql { def system: Boolean = false def nullable: Boolean = !system def dateMathScript: Boolean = false + def isTemporal: Boolean = out.isInstanceOf[SQLTemporal] + } + + trait TokenValue extends Token { + def value: Any } trait PainlessScript extends Token { @@ -55,8 +65,27 @@ package object sql { } trait DateMathScript extends Token { - def script: String + def script: Option[String] + def hasScript: Boolean = script.isDefined override def dateMathScript: Boolean = true + def formatScript: Option[String] = None + def hasFormat: Boolean = formatScript.isDefined + } + + object DateMathRounding { + def apply(out: SQLType): Option[String] = + out match { + case _: SQLDate => Some("/d") + /*case _: SQLDateTime => Some("/s") + case _: SQLTimestamp => Some("/s")*/ + case _: SQLTime => Some("/s") + case _ => None + } + } + + trait DateMathRounding { + def roundingScript: Option[String] = None + def hasRounding: Boolean = roundingScript.isDefined } trait Updateable extends Token { @@ -230,18 +259,23 @@ package object sql { override def painless: String = s"$value" } - sealed abstract class FromTo[+T](val from: Value[T], val to: Value[T]) extends Token { + sealed abstract class FromTo(val from: TokenValue, val to: TokenValue) extends Token { override def sql = s"${from.sql} AND ${to.sql}" override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(List(from.baseType, to.baseType)) - override def validate(): Either[String, Unit] = - Validator.validateTypesMatching(from.out, to.out) + override def validate(): Either[String, Unit] = { + for { + _ <- from.validate() + _ <- to.validate() + _ <- Validator.validateTypesMatching(from.out, to.out) + } yield () + } } case class LiteralFromTo(override val from: StringValue, override val to: StringValue) - extends FromTo[String](from, to) { + extends FromTo(from, to) { def between: Seq[String] => Boolean = { _.exists { s => s >= from.value && s <= to.value } } @@ -251,7 +285,7 @@ package object sql { } case class LongFromTo(override val from: LongValue, override val to: LongValue) - extends FromTo[Long](from, to) { + extends FromTo(from, to) { def between: Seq[Long] => Boolean = { _.exists { n => n >= from.value && n <= to.value } } @@ -261,7 +295,7 @@ package object sql { } case class DoubleFromTo(override val from: DoubleValue, override val to: DoubleValue) - extends FromTo[Double](from, to) { + extends FromTo(from, to) { def between: Seq[Double] => Boolean = { _.exists { n => n >= from.value && n <= to.value } } @@ -271,7 +305,7 @@ package object sql { } case class GeoDistanceFromTo(override val from: GeoDistance, override val to: GeoDistance) - extends FromTo[Double](from, to) { + extends FromTo(from, to) { def between: Seq[Double] => Boolean = { _.exists { n => n >= from.value && n <= to.value } } @@ -280,6 +314,9 @@ package object sql { } } + case class IdentifierFromTo(override val from: Identifier, override val to: Identifier) + extends FromTo(from, to) + sealed abstract class Values[+R: TypeTag, +T <: Value[R]](val values: Seq[T]) extends Token with PainlessScript { @@ -411,7 +448,12 @@ package object sql { def update(request: SQLSearchRequest): Source } - sealed trait Identifier extends Token with Source with FunctionChain with PainlessScript { + sealed trait Identifier + extends TokenValue + with Source + with FunctionChain + with PainlessScript + with DateMathScript { def name: String def withFunctions(functions: List[Function]): Identifier @@ -474,6 +516,56 @@ package object sql { expr } + def script: Option[String] = + if (isTemporal) { + var orderedFunctions = FunctionUtils.transformFunctions(this).reverse + + val baseOpt: Option[String] = orderedFunctions.headOption match { + case Some(head) => + head match { + case s: StringValue if s.value.nonEmpty => + orderedFunctions = orderedFunctions.tail + Some(s.value + "||") + case current: CurrentFunction => + orderedFunctions = orderedFunctions.tail + current.script + case _ => Option(name).filter(_.nonEmpty).map(_ + "||") + } + case _ => Option(name).filter(_.nonEmpty).map(_ + "||") + } + + val roundingOpt: Option[String] = + orderedFunctions + .collectFirst { + case r: DateMathRounding if r.hasRounding => r.roundingScript.get + } + .orElse(DateMathRounding(out)) + + orderedFunctions.foldLeft(baseOpt) { + case (expr, f: Function) if expr.isDefined && f.dateMathScript => + f match { + case s: DateMathScript => + s.script match { + case Some(script) if script.nonEmpty => + Some(s"${expr.get}$script") + case _ => expr + } + case _ => expr + } + case (_, _) => None // ignore non math scripts + } match { + case Some(s) if s.nonEmpty => + roundingOpt match { + case Some(r) if r.nonEmpty => Some(s"$s$r") + case _ => Some(s) + } + case _ => None + } + } else + None + + override def dateMathScript: Boolean = isTemporal + def checkNotNull: String = if (name.isEmpty) "" else @@ -495,6 +587,11 @@ package object sql { override def nullable: Boolean = _nullable + override def value: String = + script match { + case Some(s) => s + case _ => painless + } } object Identifier { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala index bdfb2889..bbf56622 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/OrderByParser.scala @@ -14,7 +14,7 @@ trait OrderByParser { """\b(?!(?i)limit\b)[a-zA-Z_][a-zA-Z0-9_]*""".r ^^ (f => f) def fieldWithFunction: PackratParser[(String, List[Function])] = - rep1sep(sql_functions, start) ~ start.? ~ fieldName ~ rep1(end) ^^ { case f ~ _ ~ n ~ _ => + rep1sep(sql_function, start) ~ start.? ~ fieldName ~ rep1(end) ^^ { case f ~ _ ~ n ~ _ => (n, f) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 3b552f40..126de8bc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -95,21 +95,18 @@ trait Parser identifierWithTransformation | // transformations applied to an identifier identifierWithIntervalFunction | identifierWithFunction | // fonctions applied to an identifier - literal | // 'string' - pi | - double | - long | - boolean | + identifierWithValue | identifier implicit def functionAsIdentifier(mf: Function): Identifier = mf match { - case id: Identifier => id - case fid: FunctionWithIdentifier => fid.identifier - case _ => Identifier(mf) + case id: Identifier => id + case fid: FunctionWithIdentifier => + fid.identifier //.withFunctions(fid +: fid.identifier.functions) + case _ => Identifier(mf) } - def sql_functions: PackratParser[Function] = - aggregates | date_diff | date_trunc | extractors | date_functions | datetime_functions | conditional_functions | string_functions + def sql_function: PackratParser[Function] = + aggregate_function | time_function | conditional_function | string_function private val reservedKeywords = Seq( "select", @@ -241,24 +238,19 @@ trait Parser None, d.isDefined ) - }) >> castOperator + }) >> cast def identifierWithTransformation: PackratParser[Identifier] = (mathematicalFunctionWithIdentifier | conversionFunctionWithIdentifier | conditionalFunctionWithIdentifier | - systemFunctionWithIdentifier | - dateFunctionWithIdentifier | - dateTimeFunctionWithIdentifier | + timeFunctionWithIdentifier | stringFunctionWithIdentifier | - date_diff_identifier | - extract_identifier | - case_when_identifier | - distance_identifier) >> castOperator + geoFunctionWithIdentifier) >> cast def identifierWithFunction: PackratParser[Identifier] = (rep1sep( - sql_functions, + sql_function, start ) ~ start.? ~ (identifierWithTransformation | identifierWithIntervalFunction | identifier).? ~ rep1( end @@ -272,7 +264,7 @@ trait Parser } case Some(id) => id.withFunctions(id.functions ++ f) } - }) >> castOperator + }) >> cast private val regexAlias = s"""\\b(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[a-zA-Z0-9_]*""".stripMargin diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala index c79c5668..56e76708 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/WhereParser.scala @@ -7,6 +7,7 @@ import app.softnetwork.elastic.sql.{ GeoDistance, GeoDistanceFromTo, Identifier, + IdentifierFromTo, LiteralFromTo, LongFromTo, LongValue, @@ -76,11 +77,12 @@ trait WhereParser { private def diff: PackratParser[ComparisonOperator] = DIFF.sql ^^ (_ => DIFF) private def any_identifier: PackratParser[Identifier] = + identifierWithArithmeticExpression | identifierWithTransformation | identifierWithAggregation | identifierWithIntervalFunction | - identifierWithArithmeticExpression | identifierWithFunction | + identifierWithValue | identifier private def equality: PackratParser[GenericExpression] = @@ -162,6 +164,11 @@ trait WhereParser { case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, DoubleFromTo(from, to), n) } + def betweenIdentifiers: PackratParser[Criteria] = + any_identifier ~ not.? ~ BETWEEN.regex ~ any_identifier ~ and ~ any_identifier ^^ { + case i ~ n ~ _ ~ from ~ _ ~ to => BetweenExpr(i, IdentifierFromTo(from, to), n) + } + def betweenDistances: PackratParser[Criteria] = distance_identifier ~ not.? ~ BETWEEN.regex ~ (geo_distance | long) ~ and ~ (geo_distance | long) ^^ { case i ~ n ~ _ ~ from ~ _ ~ to => @@ -217,6 +224,7 @@ trait WhereParser { betweenDistances | betweenLongs | betweenDoubles | + betweenIdentifiers | isNotNull | isNull | /*coalesce | nullif | distanceCriteria | */ matchCriteria | diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala index da496a5f..2dbe1b06 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/aggregate/package.scala @@ -19,10 +19,10 @@ package object aggregate { def sum: PackratParser[AggregateFunction] = SUM.regex ^^ (_ => SUM) - def aggregates: PackratParser[AggregateFunction] = count | min | max | avg | sum + def aggregate_function: PackratParser[AggregateFunction] = count | min | max | avg | sum def identifierWithAggregation: PackratParser[Identifier] = - aggregates ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { + aggregate_function ~ start ~ (identifierWithFunction | identifierWithIntervalFunction | identifier) ~ end ^^ { case a ~ _ ~ i ~ _ => i.withFunctions(a +: i.functions) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala index b2a7e2a8..04b3a7f4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/cond/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.parser.function -import app.softnetwork.elastic.sql.function.TransformFunction +import app.softnetwork.elastic.sql.function.{FunctionWithIdentifier, TransformFunction} import app.softnetwork.elastic.sql.function.cond.{ Case, Coalesce, @@ -83,13 +83,13 @@ package object cond { Identifier(cw) } - def conditional_functions: PackratParser[TransformFunction[_, _]] = - is_null | is_notnull | coalesce | nullif | case_when + def conditional_function: PackratParser[FunctionWithIdentifier] = + is_null | is_notnull | coalesce | nullif def conditionalFunctionWithIdentifier: PackratParser[Identifier] = - (is_null | is_notnull | coalesce | nullif) ^^ { t => + conditional_function ^^ { t => t.identifier.withFunctions(t +: t.identifier.functions) - } + } | case_when_identifier } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala index e195b5f7..6fce1809 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/convert/package.scala @@ -8,44 +8,45 @@ package object convert { trait ConvertParser { self: Parser => - def castFunctionWithIdentifier: PackratParser[Identifier] = + def cast_identifier: PackratParser[Identifier] = Cast.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | - identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { - case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => - i.withFunctions(a.toList ++ (Cast(i, targetType = t, as = as.isDefined) +: i.functions)) + identifier) ~ Alias.regex.? ~ sql_type ~ end ^^ { case _ ~ _ ~ i ~ as ~ t ~ _ => + i.withFunctions(Cast(i, targetType = t, as = as.isDefined) +: i.functions) } - def tryCastFunctionWithIdentifier: PackratParser[Identifier] = + def try_cast_identifier: PackratParser[Identifier] = TryCast.regex ~ start ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | - identifier) ~ Alias.regex.? ~ sql_type ~ end ~ intervalFunction.? ^^ { - case _ ~ _ ~ i ~ as ~ t ~ _ ~ a => - i.withFunctions( - a.toList ++ (Cast(i, targetType = t, as = as.isDefined, safe = true) +: i.functions) - ) + identifier) ~ Alias.regex.? ~ sql_type ~ end ^^ { case _ ~ _ ~ i ~ as ~ t ~ _ => + i.withFunctions( + Cast(i, targetType = t, as = as.isDefined, safe = true) +: i.functions + ) } - def castOperator: Identifier => PackratParser[Identifier] = i => + def convert_identifier: PackratParser[Identifier] = + Convert.regex ~ start ~ (identifierWithTransformation | + identifierWithIntervalFunction | + identifierWithFunction | + identifier) ~ separator ~ sql_type ~ end ^^ { case _ ~ _ ~ i ~ _ ~ t ~ _ => + i.withFunctions(Convert(i, targetType = t) +: i.functions) + } + + def cast: Identifier => PackratParser[Identifier] = i => (CastOperator.regex ~ sql_type).? ^^ { case None => i case Some(_ ~ t) => i.withFunctions(CastOperator(i, targetType = t) +: i.functions) } - def convertFunctionWithIdentifier: PackratParser[Identifier] = - Convert.regex ~ start ~ (identifierWithTransformation | - identifierWithIntervalFunction | - identifierWithFunction | - identifier) ~ separator ~ sql_type ~ end ~ intervalFunction.? ^^ { - case _ ~ _ ~ i ~ _ ~ t ~ _ ~ a => - i.withFunctions(a.toList ++ (Convert(i, targetType = t) +: i.functions)) - } - def conversionFunctionWithIdentifier: PackratParser[Identifier] = - castFunctionWithIdentifier | tryCastFunctionWithIdentifier | convertFunctionWithIdentifier + (cast_identifier | try_cast_identifier | convert_identifier) ~ rep( + intervalFunction + ) ^^ { case id ~ funcs => + id.withFunctions(funcs ++ id.functions) + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala index 7736fc4c..421dc0f6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/geo/package.scala @@ -41,5 +41,7 @@ package object geo { long ~ distance_unit ^^ { case value ~ unit => GeoDistance(value, unit) } def distance_identifier: PackratParser[Identifier] = distance ^^ functionAsIdentifier + + def geoFunctionWithIdentifier: PackratParser[Identifier] = distance_identifier } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala index b6383e8b..6893fe43 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/math/package.scala @@ -44,7 +44,7 @@ package object math { private[this] def log10: PackratParser[MathOp] = Log10.regex ^^ (_ => Log10) - def arithmeticFunction: PackratParser[MathematicalFunction] = + def arithmetic_function: PackratParser[MathematicalFunction] = (abs | ceil | exp | floor | log | log10 | sqrt) ~ start ~ valueExpr ~ end ^^ { case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) } @@ -63,42 +63,40 @@ package object math { private[this] def atan2: PackratParser[Trigonometric] = Atan2.regex ^^ (_ => Atan2) - def atan2Function: PackratParser[MathematicalFunction] = + def atan2_function: PackratParser[MathematicalFunction] = atan2 ~ start ~ (double | valueExpr) ~ separator ~ (double | valueExpr) ~ end ^^ { case _ ~ _ ~ y ~ _ ~ x ~ _ => Atan2(y, x) } - def trigonometricFunction: PackratParser[MathematicalFunction] = - atan2Function | ((sin | asin | cos | acos | tan | atan) ~ start ~ valueExpr ~ end ^^ { + def trigonometric_function: PackratParser[MathematicalFunction] = + atan2_function | ((sin | asin | cos | acos | tan | atan) ~ start ~ valueExpr ~ end ^^ { case op ~ _ ~ v ~ _ => MathematicalFunctionWithOp(op, v) }) private[this] def round: PackratParser[MathOp] = Round.regex ^^ (_ => Round) - def roundFunction: PackratParser[MathematicalFunction] = + def round_function: PackratParser[MathematicalFunction] = round ~ start ~ valueExpr ~ separator.? ~ long.? ~ end ^^ { case _ ~ _ ~ v ~ _ ~ s ~ _ => Round(v, s.map(_.value.toInt)) } private[this] def pow: PackratParser[MathOp] = Pow.regex ^^ (_ => Pow) - def powFunction: PackratParser[MathematicalFunction] = + def pow_function: PackratParser[MathematicalFunction] = pow ~ start ~ valueExpr ~ separator ~ long ~ end ^^ { case _ ~ _ ~ v1 ~ _ ~ e ~ _ => Pow(v1, e.value.toInt) } private[this] def sign: PackratParser[MathOp] = Sign.regex ^^ (_ => Sign) - def signFunction: PackratParser[MathematicalFunction] = + def sign_function: PackratParser[MathematicalFunction] = sign ~ start ~ valueExpr ~ end ^^ { case _ ~ _ ~ v ~ _ => Sign(v) } - def mathematicalFunction: PackratParser[MathematicalFunction] = - arithmeticFunction | trigonometricFunction | roundFunction | powFunction | signFunction + def mathematical_function: PackratParser[MathematicalFunction] = + arithmetic_function | trigonometric_function | round_function | pow_function | sign_function def mathematicalFunctionWithIdentifier: PackratParser[Identifier] = - mathematicalFunction ^^ { mf => - mf.identifier - } + mathematical_function ^^ functionAsIdentifier } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index 3c6b57f2..a265a5ff 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -99,7 +99,7 @@ package object string { StringFunctionWithOp(Rtrim) } - def string_functions: Parser[ + def string_function: Parser[ StringFunction[_] ] = /*concatFunction | substringFunction |*/ length | lower | upper | trim | ltrim | rtrim diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index ccceea4a..e4079ec5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.parser.function -import app.softnetwork.elastic.sql.{Identifier, StringValue} +import app.softnetwork.elastic.sql.{function, Identifier, StringValue} import app.softnetwork.elastic.sql.`type`.{SQLLiteral, SQLNumeric, SQLTemporal} import app.softnetwork.elastic.sql.function.{ BinaryFunction, @@ -14,7 +14,7 @@ import app.softnetwork.elastic.sql.time.{IsoField, TimeField, TimeUnit} package object time { - trait SystemParser { self: Parser with TimeParser => + trait CurrentParser { self: Parser with TimeParser => def parens: PackratParser[List[Delimiter]] = start ~ end ^^ { case s ~ e => s :: e :: Nil } @@ -42,16 +42,11 @@ package object time { Today(p.isDefined) } - def systemFunctions: PackratParser[CurrentFunction] = + private[this] def current_function: PackratParser[CurrentFunction] = current_date | current_time | current_timestamp | now | today - def systemFunctionWithIdentifier: PackratParser[Identifier] = - systemFunctions ~ intervalFunction.? ^^ { case f1 ~ f2 => - f2 match { - case Some(f) => Identifier(List(f, f1)) - case None => Identifier(f1) - } - } + def currentFunctionWithIdentifier: PackratParser[Identifier] = + current_function ^^ functionAsIdentifier } @@ -91,17 +86,13 @@ package object time { case _ ~ _ ~ i ~ _ => LastDayOfMonth(i) } - def date_functions: PackratParser[DateFunction] = + def date_function: PackratParser[DateFunction] = date_add | date_sub | date_parse | date_format | last_day def dateFunctionWithIdentifier: PackratParser[Identifier] = - (date_parse | date_format | date_add | date_sub | last_day) ~ intervalFunction.? ^^ { - case t ~ af => - af match { - case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) - case None => t.identifier.withFunctions(t +: t.identifier.functions) - } - } + (date_parse | date_format | date_add | date_sub | last_day) ^^ (t => + t.identifier.withFunctions(t +: t.identifier.functions) + ) } @@ -136,21 +127,17 @@ package object time { DateTimeFormat(i, f.value) } - def datetime_functions: PackratParser[DateTimeFunction] = + def datetime_function: PackratParser[DateTimeFunction] = datetime_add | datetime_sub | datetime_parse | datetime_format def dateTimeFunctionWithIdentifier: PackratParser[Identifier] = - (date_trunc | datetime_parse | datetime_format | datetime_add | datetime_sub) ~ intervalFunction.? ^^ { - case t ~ af => - af match { - case Some(f) => t.identifier.withFunctions(f +: t +: t.identifier.functions) - case None => t.identifier.withFunctions(t +: t.identifier.functions) - } + (datetime_parse | datetime_format | datetime_add | datetime_sub) ^^ { t => + t.identifier.withFunctions(t +: t.identifier.functions) } } - trait TemporalParser extends SystemParser with TimeParser with DateParser with DateTimeParser { + trait TemporalParser extends CurrentParser with TimeParser with DateParser with DateTimeParser { self: Parser => def date_diff: PackratParser[BinaryFunction[_, _, _]] = @@ -176,6 +163,10 @@ package object time { DateTrunc(i, u) } + def date_trunc_identifier: PackratParser[Identifier] = date_trunc ^^ { dt => + dt.identifier.withFunctions(dt +: dt.identifier.functions) + } + def extract_identifier: PackratParser[Identifier] = Extract.regex ~ start ~ time_field ~ "(?i)from".r ~ (identifierWithTransformation | identifierWithIntervalFunction | identifierWithFunction | identifier) ~ end ^^ { case _ ~ _ ~ u ~ _ ~ i ~ _ => @@ -217,7 +208,7 @@ package object time { def week_of_week_based_year_tr: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = IsoField.WEEK_OF_WEEK_BASED_YEAR.regex ^^ (_ => new WeekOfWeekBasedYear) - def extractors: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = + def extractor_function: PackratParser[TransformFunction[SQLTemporal, SQLNumeric]] = year_tr | month_of_year_tr | day_of_month_tr | @@ -234,6 +225,18 @@ package object time { quarter_of_year_tr | week_of_week_based_year_tr + def time_function: Parser[function.Function] = + date_function | datetime_function | date_diff | date_trunc | extractor_function + + def timeFunctionWithIdentifier: Parser[Identifier] = + (currentFunctionWithIdentifier | + dateFunctionWithIdentifier | + dateTimeFunctionWithIdentifier | + date_diff_identifier | + date_trunc_identifier | + extract_identifier) ~ rep(intervalFunction) ^^ { case i ~ f => + i.withFunctions(f ++ i.functions) + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala index 7a211d3b..991ec3ba 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/operator/math/package.scala @@ -55,7 +55,7 @@ package object math { case f: FunctionWithIdentifier => f.identifier case f: Function => Identifier(f) case other => throw new Exception(s"Unexpected expression $other") - }) >> castOperator + }) >> cast } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala index 5d6f0c8a..b3ec69f8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/time/package.scala @@ -95,9 +95,10 @@ package object time { def identifierWithIntervalFunction: PackratParser[Identifier] = ((identifierWithTransformation | identifierWithFunction | - identifier) ~ intervalFunction ^^ { case i ~ f => - i.withFunctions(f +: i.functions) - }) >> castOperator + identifierWithValue | + identifier) ~ rep(intervalFunction) ^^ { case i ~ f => + i.withFunctions(f ++ i.functions) + }) >> cast } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index c9dc2063..f3195889 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -3,6 +3,7 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.{ BooleanValue, DoubleValue, + Identifier, LongValue, PiValue, StringValue, @@ -31,6 +32,11 @@ package object `type` { def boolean: PackratParser[BooleanValue] = """(?i)(true|false)\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) + def value: PackratParser[Value[_]] = + literal | pi | double | long | boolean + + def identifierWithValue: Parser[Identifier] = (value ^^ functionAsIdentifier) >> cast + def char_type: PackratParser[SQLTypes.Char.type] = "(?i)char".r ^^ (_ => SQLTypes.Char) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index d0be9421..1da93cec 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -6,6 +6,7 @@ import app.softnetwork.elastic.sql.{ asString, Alias, AliasUtils, + DateMathScript, Expr, Identifier, PainlessScript, @@ -20,7 +21,8 @@ case class Field( fieldAlias: Option[Alias] = None ) extends Updateable with FunctionChain - with PainlessScript { + with PainlessScript + with DateMathScript { def isScriptField: Boolean = functions.nonEmpty && !aggregation && identifier.bucket.isEmpty override def sql: String = s"$identifier${asString(fieldAlias)}" lazy val sourceField: String = { @@ -64,6 +66,8 @@ case class Field( def painless: String = identifier.painless + def script: Option[String] = identifier.script + lazy val scriptName: String = fieldAlias.map(_.alias).getOrElse(sourceField) override def validate(): Either[String, Unit] = identifier.validate() diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 97722750..e6ba336c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -408,9 +408,9 @@ case class InExpr[R, +T <: Value[R]]( override def painless: String = s"$painlessNot${identifier.painless}$painlessOp($painlessValue)" } -case class BetweenExpr[+T]( +case class BetweenExpr( identifier: Identifier, - fromTo: FromTo[T], + fromTo: FromTo, maybeNot: Option[NOT.type] ) extends Expression { override def sql = s"$identifier $notAsString$operator $fromTo" @@ -431,7 +431,7 @@ case class BetweenExpr[+T]( for { _ <- identifier.validate() _ <- fromTo.validate() - _ <- Validator.validateTypesMatching(identifier.out, fromTo.from.out) + _ <- Validator.validateTypesMatching(identifier.out, fromTo.out) } yield () } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index 232dfeaa..e51e3d9e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -77,7 +77,7 @@ package object time { } - sealed trait TimeUnit extends PainlessScript with DateMathScript { + sealed trait TimeUnit extends PainlessScript with DateMathScript with DateMathRounding { lazy val regex: Regex = s"\\b(?i)$sql(s)?\\b".r def timeUnit: String = sql.toUpperCase() + "S" @@ -85,6 +85,11 @@ package object time { override def painless: String = s"ChronoUnit.$timeUnit" override def nullable: Boolean = false + + override def roundingScript: Option[String] = script match { + case Some(s) if s.nonEmpty => Some(s"/$s") + case _ => None + } } sealed trait CalendarUnit extends TimeUnit @@ -92,32 +97,32 @@ package object time { object TimeUnit { case object YEARS extends Expr("YEAR") with CalendarUnit { - override def script: String = "y" + override def script: Option[String] = Some("y") } case object MONTHS extends Expr("MONTH") with CalendarUnit { - override def script: String = "M" + override def script: Option[String] = Some("M") } case object QUARTERS extends Expr("QUARTER") with CalendarUnit { - override def script: String = throw new IllegalArgumentException( + override def script: Option[String] = throw new IllegalArgumentException( "Quarter must be converted to months (value * 3) before creating date-math" ) } case object WEEKS extends Expr("WEEK") with CalendarUnit { - override def script: String = "w" + override def script: Option[String] = Some("w") } case object DAYS extends Expr("DAY") with CalendarUnit with FixedUnit { - override def script: String = "d" + override def script: Option[String] = Some("d") } case object HOURS extends Expr("HOUR") with FixedUnit { - override def script: String = "H" + override def script: Option[String] = Some("H") } case object MINUTES extends Expr("MINUTE") with FixedUnit { - override def script: String = "m" + override def script: Option[String] = Some("m") } case object SECONDS extends Expr("SECOND") with FixedUnit { - override def script: String = "s" + override def script: Option[String] = Some("s") } } @@ -131,7 +136,7 @@ package object time { override def painless: String = s"$value, ${unit.painless}" - override def script: String = TimeInterval.script(this) + override def script: Option[String] = Some(TimeInterval.script(this)) def checkType(in: SQLType): Either[String, SQLType] = { import TimeUnit._ @@ -173,8 +178,16 @@ package object time { } def script(interval: TimeInterval): String = interval match { case CalendarInterval(v, QUARTERS) => s"${v * 3}M" - case CalendarInterval(v, u) => s"$v${u.script}" - case FixedInterval(v, u) => s"$v${u.script}" + case CalendarInterval(v, u) => + u.script match { + case Some(s) if s.nonEmpty => s"$v$s" + case _ => throw new IllegalArgumentException(s"Invalid calendar unit $u") + } + case FixedInterval(v, u) => + u.script match { + case Some(s) if s.nonEmpty => s"$v$s" + case _ => throw new IllegalArgumentException(s"Invalid fixed unit $u") + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index 444ea830..b5b28491 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -7,7 +7,7 @@ sealed trait SQLType { trait SQLAny extends SQLType -trait SQLNull extends SQLType +trait SQLNull extends SQLAny trait SQLTemporal extends SQLType diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 461545ab..1b8c0b0a 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -177,6 +177,8 @@ object Queries { val geoDistance = "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(-70.0, 40.0)) < 1000 km" + val betweenTemporal = + "SELECT * FROM Table WHERE createdAt BETWEEN CURRENT_DATE - INTERVAL 1 MONTH AND CURRENT_DATE AND lastUpdated BETWEEN '2025-09-11'::DATE AND TODAY" //DATE_TRUNC(CURRENT_TIMESTAMP, DATE)" } /** Created by smanciot on 15/02/17. @@ -193,6 +195,13 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalEq) shouldBe true } + it should "parse BETWEEN with temporal fields" in { + val result = Parser(betweenTemporal) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") shouldBe betweenTemporal + } + it should "parse numerical NE" in { val result = Parser(numericalNe) result.toOption @@ -835,4 +844,12 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .flatMap(_.left.toOption.map(_.sql)) .getOrElse("") shouldBe geoDistance } + + /*it should "parse BETWEEN with temporal fields" in { + val result = Parser(betweenTemporal) + result.toOption + .flatMap(_.left.toOption.map(_.sql)) + .getOrElse("") shouldBe betweenTemporal + }*/ + } From 450307c1a6545f84009d1f4a9d0bc7fbd1da24f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 2 Oct 2025 07:30:57 +0200 Subject: [PATCH 48/48] finalize support for temporal between --- documentation/operators.md | 9 ++ .../elastic/sql/SQLQuerySpec.scala | 57 +++++++++++-- .../elastic/sql/SQLQuerySpec.scala | 84 +++++++++++++++++++ .../elastic/sql/function/time/package.scala | 2 + .../sql/parser/function/time/package.scala | 10 +-- .../elastic/sql/SQLParserSpec.scala | 13 +-- 6 files changed, 154 insertions(+), 21 deletions(-) diff --git a/documentation/operators.md b/documentation/operators.md index a2ef6134..52aa9fc6 100644 --- a/documentation/operators.md +++ b/documentation/operators.md @@ -160,6 +160,15 @@ FROM users WHERE age BETWEEN 18 AND 30; ``` +- Temporal BETWEEN +```sql +SELECT * +FROM users +WHERE createdAt BETWEEN CURRENT_DATE - INTERVAL 1 MONTH AND CURRENT_DATE +AND +lastUpdated BETWEEN LAST_DAY('2025-09-11'::DATE) AND DATE_TRUNC(CURRENT_TIMESTAMP, DAY) +``` + - Distance BETWEEN (using meters) ```sql diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 0bde0b93..2d305239 100644 --- a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -3028,11 +3028,24 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } | }, | { - | "range": { - | "lastUpdated": { - | "gte": "2025-09-11||/d", - | "lte": "now/d" - | } + | "bool": { + | "must": [ + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "def left = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); left == null ? false : left >= (def e2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern('yyyy-MM-dd')); e2.withDayOfMonth(e2.lengthOfMonth()))" + | } + | } + | }, + | { + | "range": { + | "lastUpdated": { + | "lte": "now/d" + | } + | } + | } + | ] | } | } | ] @@ -3045,5 +3058,39 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | } |}""".stripMargin .replaceAll("\\s+", "") + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll(">=", " >= ") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll(">(\\d)", " > $1") + .replaceAll("=(\\d)", "= $1") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",params", ", params") + .replaceAll("GeoPoint", " GeoPoint") + .replaceAll("lat,arg", "lat, arg") + .replaceAll("false:", "false : ") + .replaceAll("DateTimeFormatter", " DateTimeFormatter") } } diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 9e8aa2b8..cd2431f9 100644 --- a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2998,4 +2998,88 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") } + it should "handle between with temporal" in { + val select: ElasticSearchRequest = + SQLQuery(betweenTemporal) + val query = select.query + println(query) + query shouldBe + """{ + | "query": { + | "bool": { + | "filter": [ + | { + | "range": { + | "createdAt": { + | "gte": "now-1M/d", + | "lte": "now/d" + | } + | } + | }, + | { + | "bool": { + | "must": [ + | { + | "script": { + | "script": { + | "lang": "painless", + | "source": "def left = (!doc.containsKey('lastUpdated') || doc['lastUpdated'].empty ? null : doc['lastUpdated'].value); left == null ? false : left >= (def e2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern('yyyy-MM-dd')); e2.withDayOfMonth(e2.lengthOfMonth()))" + | } + | } + | }, + | { + | "range": { + | "lastUpdated": { + | "lte": "now/d" + | } + | } + | } + | ] + | } + | } + | ] + | } + | }, + | "_source": { + | "includes": [ + | "*" + | ] + | } + |}""".stripMargin + .replaceAll("\\s+", "") + .replaceAll("\\s+", "") + .replaceAll("defv", " def v") + .replaceAll("defa", "def a") + .replaceAll("defe", "def e") + .replaceAll("defl", "def l") + .replaceAll("def_", "def _") + .replaceAll("=_", " = _") + .replaceAll(",_", ", _") + .replaceAll(",\\(", ", (") + .replaceAll("if\\(", "if (") + .replaceAll(">=", " >= ") + .replaceAll("=\\(", " = (") + .replaceAll(":\\(", " : (") + .replaceAll(",(\\d)", ", $1") + .replaceAll("\\?", " ? ") + .replaceAll(":null", " : null") + .replaceAll("null:", "null : ") + .replaceAll("return", " return ") + .replaceAll(";", "; ") + .replaceAll("; if", ";if") + .replaceAll("==", " == ") + .replaceAll("\\+", " + ") + .replaceAll(">(\\d)", " > $1") + .replaceAll("=(\\d)", "= $1") + .replaceAll("<", " < ") + .replaceAll("!=", " != ") + .replaceAll("&&", " && ") + .replaceAll("\\|\\|", " || ") + .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",params", ", params") + .replaceAll("GeoPoint", " GeoPoint") + .replaceAll("lat,arg", "lat, arg") + .replaceAll("false:", "false : ") + .replaceAll("DateTimeFormatter", " DateTimeFormatter") + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 4a65aecd..77e69089 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -183,6 +183,8 @@ package object time { } override def roundingScript: Option[String] = unit.roundingScript + + override def dateMathScript: Boolean = identifier.dateMathScript } case object Extract extends Expr("EXTRACT") with TokenRegex with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index e4079ec5..f0061cec 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -86,13 +86,11 @@ package object time { case _ ~ _ ~ i ~ _ => LastDayOfMonth(i) } - def date_function: PackratParser[DateFunction] = + def date_function: PackratParser[DateFunction with FunctionWithIdentifier] = date_add | date_sub | date_parse | date_format | last_day def dateFunctionWithIdentifier: PackratParser[Identifier] = - (date_parse | date_format | date_add | date_sub | last_day) ^^ (t => - t.identifier.withFunctions(t +: t.identifier.functions) - ) + date_function ^^ (t => t.identifier.withFunctions(t +: t.identifier.functions)) } @@ -127,11 +125,11 @@ package object time { DateTimeFormat(i, f.value) } - def datetime_function: PackratParser[DateTimeFunction] = + def datetime_function: PackratParser[DateTimeFunction with FunctionWithIdentifier] = datetime_add | datetime_sub | datetime_parse | datetime_format def dateTimeFunctionWithIdentifier: PackratParser[Identifier] = - (datetime_parse | datetime_format | datetime_add | datetime_sub) ^^ { t => + datetime_function ^^ { t => t.identifier.withFunctions(t +: t.identifier.functions) } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala index 1b8c0b0a..919819f3 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala @@ -178,7 +178,7 @@ object Queries { "SELECT ST_DISTANCE(POINT(-70.0, 40.0), toLocation) AS d1, ST_DISTANCE(fromLocation, POINT(-70.0, 40.0)) AS d2, ST_DISTANCE(POINT(-70.0, 40.0), POINT(0.0, 0.0)) AS d3 FROM Table WHERE ST_DISTANCE(POINT(-70.0, 40.0), toLocation) BETWEEN 4000 km AND 5000 km AND ST_DISTANCE(fromLocation, toLocation) < 2000 km AND ST_DISTANCE(POINT(-70.0, 40.0), POINT(-70.0, 40.0)) < 1000 km" val betweenTemporal = - "SELECT * FROM Table WHERE createdAt BETWEEN CURRENT_DATE - INTERVAL 1 MONTH AND CURRENT_DATE AND lastUpdated BETWEEN '2025-09-11'::DATE AND TODAY" //DATE_TRUNC(CURRENT_TIMESTAMP, DATE)" + "SELECT * FROM Table WHERE createdAt BETWEEN CURRENT_DATE - INTERVAL 1 MONTH AND CURRENT_DATE AND lastUpdated BETWEEN LAST_DAY('2025-09-11'::DATE) AND DATE_TRUNC(CURRENT_TIMESTAMP, DAY)" } /** Created by smanciot on 15/02/17. @@ -195,13 +195,6 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .equalsIgnoreCase(numericalEq) shouldBe true } - it should "parse BETWEEN with temporal fields" in { - val result = Parser(betweenTemporal) - result.toOption - .flatMap(_.left.toOption.map(_.sql)) - .getOrElse("") shouldBe betweenTemporal - } - it should "parse numerical NE" in { val result = Parser(numericalNe) result.toOption @@ -845,11 +838,11 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { .getOrElse("") shouldBe geoDistance } - /*it should "parse BETWEEN with temporal fields" in { + it should "parse BETWEEN with temporal fields" in { val result = Parser(betweenTemporal) result.toOption .flatMap(_.left.toOption.map(_.sql)) .getOrElse("") shouldBe betweenTemporal - }*/ + } }