diff --git a/kafka-gateway/pom.xml b/kafka-gateway/pom.xml index a76652b07..5f04c175a 100644 --- a/kafka-gateway/pom.xml +++ b/kafka-gateway/pom.xml @@ -46,14 +46,28 @@ za.co.absa.spline - producer-model-v1.1 + producer-model-mapper ${project.version} + + + za.co.absa.spline - producer-model-mapper + producer-model-v1.2 ${project.version} + + + + + za.co.absa.spline + producer-model-v1.1 + 0.7.5 + + + + org.springframework.kafka spring-kafka diff --git a/producer-model-mapper/pom.xml b/producer-model-mapper/pom.xml index 693a4e711..3ca04f7a6 100644 --- a/producer-model-mapper/pom.xml +++ b/producer-model-mapper/pom.xml @@ -31,20 +31,35 @@ za.co.absa.spline - producer-model-v1.1 + commons ${project.version} + + + za.co.absa.spline - commons + producer-model-v1.2 ${project.version} + + + + + za.co.absa.spline + producer-model-v1.1 + 0.7.5 + true + za.co.absa.spline producer-model 0.5.1 + true + + io.bfil automapper_${scala.compat.version} diff --git a/producer-model/pom.xml b/producer-model/pom.xml index 7461ab7a9..4c2a6ad15 100644 --- a/producer-model/pom.xml +++ b/producer-model/pom.xml @@ -25,7 +25,7 @@ 1.0.0-SNAPSHOT - producer-model-v1.1 + producer-model-v1.2 diff --git a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/AttrOrExprRef.scala b/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/AttrOrExprRef.scala deleted file mode 100644 index 300dbc24f..000000000 --- a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/AttrOrExprRef.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.producer.model.v1_1 - -/** - * A wrapper class that holds either Attribute ID or Expression ID in one of the corresponding optional properties - * (the other property is respectively `None`). - * - * The class is designed to support (de-)serialization as well as data structures where - * Expression ID and Attribute ID are mixed with other potentially arbitrary data type, so that - * from the context of that data structure it's hard to tell if a given value represents one of - * those references (and which one if so) or any other value with different semantics. - * - * It can be seen as an alternative to a `_typeHint` property, but instead of a separate discriminator property - * holding an enum value, here the value property name serves as a discriminator. - * - * The assumption is that there is no object with a single property named as `__attrId` or `__exprId` - * that is not a representation of this class. - * - * E.g. JSON object `{ __attrId: 42 }` represents a reference to an attribute with ID 42, while - * `{ _attrId: 42 }` or `{ __attrId: 42, foo: 1 }` represent arbitrary key-value pairs. - * - * @param __attrId attribute ID - * @param __exprId expression ID - */ - -case class AttrOrExprRef( - __attrId: Option[Attribute.Id], - __exprId: Option[ExpressionLike.Id]) { - - require( - __attrId.isDefined ^ __exprId.isDefined, - s"Either `__attrId` or `__exprId` should be defined. Was: ${__attrId}, ${__exprId}") -} - -object AttrOrExprRef { - - implicit class AttrOrExprOps(val ref: AttrOrExprRef) extends AnyVal { - def refId: ExpressionLike.Id = (ref.__attrId orElse ref.__exprId).get - - def isAttribute: Boolean = ref.__attrId.isDefined - - def isExpression: Boolean = ref.__exprId.isDefined - } - - def attrRef(attrId: Attribute.Id): AttrOrExprRef = AttrOrExprRef(Option(attrId), None) - - def exprRef(exprId: ExpressionLike.Id): AttrOrExprRef = AttrOrExprRef(None, Option(exprId)) - - def fromMap(obj: Map[String, Any]): Option[AttrOrExprRef] = { - if (obj.size != 1) None - else obj.head match { - case ("__attrId", attrId: Attribute.Id) => Some(attrRef(attrId)) - case ("__exprId", exprId: ExpressionLike.Id) => Some(exprRef(exprId)) - case _ => None - } - } -} diff --git a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/Attribute.scala b/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/Attribute.scala deleted file mode 100644 index 5284522f5..000000000 --- a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/Attribute.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2020 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.producer.model.v1_1 - -case class Attribute( - id: Attribute.Id, - dataType: Option[Any] = None, - childRefs: Seq[Attribute.ChildRef] = Nil, - extra: Map[String, Any] = Map.empty, - name: String, -) - -object Attribute { - type Id = ExpressionLike.Id - type ChildRef = AttrOrExprRef -} diff --git a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/ExecutionEvent.scala b/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/ExecutionEvent.scala deleted file mode 100644 index 8bc0d81d1..000000000 --- a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/ExecutionEvent.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.producer.model.v1_1 - -import za.co.absa.spline.producer.model.v1_1.ExecutionEvent._ - -import scala.language.implicitConversions - -case class ExecutionEvent( - planId: ExecutionPlan.Id, - timestamp: Long, - durationNs: Option[DurationNs], - discriminator: Option[ExecutionPlan.Discriminator] = None, - error: Option[Any] = None, - extra: Map[String, Any] = Map.empty -) - -object ExecutionEvent { - type DurationNs = java.lang.Long - - implicit def optJavaLong2OptScalaLong(opt: Option[java.lang.Long]): Option[Long] = opt.map(identity(_)) -} diff --git a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/executionPlan.scala b/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/executionPlan.scala deleted file mode 100644 index 2f6494fdc..000000000 --- a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/executionPlan.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2020 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.producer.model.v1_1 - -import za.co.absa.spline.common.validation.{Constraint, ValidationUtils} - -import java.util.UUID - -case class ExecutionPlan( - id: ExecutionPlan.Id = UUID.randomUUID(), - name: Option[ExecutionPlan.Name], - discriminator: Option[ExecutionPlan.Discriminator] = None, - - operations: Operations, - attributes: Seq[Attribute] = Nil, - expressions: Option[Expressions] = None, - - systemInfo: NameAndVersion, - agentInfo: Option[NameAndVersion] = None, - extraInfo: Map[String, Any] = Map.empty -) { - ValidationUtils.validate(Constraint.unique(attributes) by (_.id) as "attribute ID") - - def dataSources: Set[ExecutionPlan.DataSourceUri] = { - val readSources = operations.reads.flatMap(_.inputSources).toSet - val writeSource = operations.write.outputSource - readSources + writeSource - } -} - -object ExecutionPlan { - type Id = UUID - type Name = String - type DataSourceUri = String - type Discriminator = String -} - -case class Operations( - write: WriteOperation, - reads: Seq[ReadOperation] = Nil, - other: Seq[DataOperation] = Nil -) { - ValidationUtils.validate(Constraint.unique(all) by (_.id) as "operation ID") - - def all: Seq[OperationLike] = reads ++ other :+ write -} - -case class Expressions( - functions: Seq[FunctionalExpression] = Nil, - constants: Seq[Literal] = Nil -) { - ValidationUtils.validate(Constraint.unique(all) by (_.id) as "expression ID") - - def all: Seq[ExpressionLike] = functions ++ constants -} - -case class NameAndVersion(name: String, version: String) diff --git a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/expressions.scala b/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/expressions.scala deleted file mode 100644 index c85f5e9f7..000000000 --- a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/expressions.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.producer.model.v1_1 - -import com.fasterxml.jackson.databind.annotation.JsonDeserialize -import za.co.absa.spline.common.webmvc.jackson.NullAcceptingDeserializer - -sealed trait ExpressionLike { - def id: ExpressionLike.Id - - def dataType: Option[Any] - - def childRefs: Seq[ExpressionLike.ChildRef] - def extra: Map[String, Any] -} - -object ExpressionLike { - type Id = String - type ChildRef = AttrOrExprRef - type Params = Map[String, Any] - type Extras = Map[String, Any] -} - -/** - * Represents a functional expression that computes a value based on the given input. - * - * @param id expression ID - * @param name expression name - * @param dataType output data type - * @param childRefs input expression (or attribute) IDs - * @param params optional static expression parameters (don't confuse with input parameters) - * @param extra optional metadata - */ -case class FunctionalExpression( - override val id: ExpressionLike.Id, - override val dataType: Option[Any] = None, - override val childRefs: Seq[ExpressionLike.ChildRef] = Nil, - override val extra: Map[String, Any] = Map.empty, - name: String, - params: Map[String, Any] = Map.empty, -) extends ExpressionLike - -/** - * Literal expression - * - * @param id expression ID - * @param value literal value - * @param dataType value data type - * @param extra optional metadata - */ -case class Literal( - override val id: ExpressionLike.Id, - override val dataType: Option[Any] = None, - override val extra: Map[String, Any] = Map.empty, - @JsonDeserialize(using = classOf[NullAcceptingDeserializer]) - value: Any, -) extends ExpressionLike { - override def childRefs: Seq[ExpressionLike.ChildRef] = Nil -} diff --git a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/operations.scala b/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/operations.scala deleted file mode 100644 index 26bc0ef3f..000000000 --- a/producer-model/src/main/scala/za/co/absa/spline/producer/model/v1_1/operations.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2020 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.spline.producer.model.v1_1 - -import za.co.absa.commons.graph.GraphImplicits.DAGNodeIdMapping - -sealed trait OperationLike { - def id: OperationLike.Id - def name: Option[OperationLike.Name] - def childIds: Seq[OperationLike.Id] - def output: Option[OperationLike.Schema] - def params: Map[String, Any] - def extra: Map[String, Any] -} - -object OperationLike { - type Id = String - type Name = String - type Schema = Seq[Attribute.Id] - - implicit object OpNav extends DAGNodeIdMapping[OperationLike, OperationLike.Id] { - - override def selfId(op: OperationLike): Id = op.id - - override def refIds(op: OperationLike): Seq[Id] = op.childIds - } - -} - - -case class DataOperation( - override val id: OperationLike.Id, - override val name: Option[OperationLike.Name] = None, - override val childIds: Seq[OperationLike.Id] = Nil, - override val output: Option[OperationLike.Schema] = None, - override val params: Map[String, Any] = Map.empty, - override val extra: Map[String, Any] = Map.empty -) extends OperationLike - -case class ReadOperation( - inputSources: Seq[String], - override val id: OperationLike.Id, - override val name: Option[OperationLike.Name] = None, - override val output: Option[OperationLike.Schema] = None, - override val params: Map[String, Any] = Map.empty, - override val extra: Map[String, Any] = Map.empty -) extends OperationLike { - override def childIds: Seq[OperationLike.Id] = Nil -} - -case class WriteOperation( - outputSource: String, - append: Boolean, - override val id: OperationLike.Id, - override val name: Option[OperationLike.Name] = None, - override val childIds: Seq[OperationLike.Id], - override val params: Map[String, Any] = Map.empty, - override val extra: Map[String, Any] = Map.empty -) extends OperationLike { - override def output: Option[OperationLike.Schema] = None -} diff --git a/producer-rest-core/pom.xml b/producer-rest-core/pom.xml index bbe65d251..b9f2069dc 100644 --- a/producer-rest-core/pom.xml +++ b/producer-rest-core/pom.xml @@ -45,11 +45,30 @@ producer-model-mapper ${project.version} + + + + + za.co.absa.spline + producer-model-v1.2 + ${project.version} + + + + + + za.co.absa.spline + producer-model-v1.1 + 0.7.5 + za.co.absa.spline producer-model 0.5.1 + + + javax.servlet javax.servlet-api diff --git a/producer-services/pom.xml b/producer-services/pom.xml index c6f81b192..4e78bdbf8 100644 --- a/producer-services/pom.xml +++ b/producer-services/pom.xml @@ -40,7 +40,7 @@ za.co.absa.spline - producer-model-v1.1 + producer-model-v1.2 ${project.version}