From f5e679574fd9f7b2199b217c501012a8aeeb1bc0 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 14 Jun 2024 08:35:19 +0200 Subject: [PATCH 01/11] Addid circe dependency and using it --- project/Dependencies.scala | 13 +++ .../CreateOrUpdateAdditionalData.scala | 8 +- .../CreatePartitioningIfNotExists.scala | 14 +-- .../runs/functions/WriteCheckpoint.scala | 2 +- .../absa/atum/server/api/http/Endpoints.scala | 2 +- .../server/model/CirceJsonImplicits.scala | 73 ++++++++++++++ .../atum/server/model/ErrorResponse.scala | 50 ++++++++-- .../atum/server/model/PartitioningForDB.scala | 34 +++++-- .../atum/server/model/PlayJsonImplicits.scala | 96 ------------------- ...ateCheckpointEndpointIntegrationTest.scala | 2 +- ...PartitioningEndpointIntegrationTests.scala | 2 +- 11 files changed, 172 insertions(+), 124 deletions(-) create mode 100644 server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4bb377157..9dadfcbf0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -74,6 +74,7 @@ object Dependencies { val http4sBlazeBackend = "0.23.15" val http4sPrometheus = "0.23.6" val playJson = "3.0.1" + val circeJson = "0.14.7" val sttpPlayJson = "3.9.3" val awssdk = "2.23.15" @@ -115,6 +116,8 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided + lazy val sttp = "com.softwaremill.sttp.client3" %% "core" % Versions.sttp + Seq( jacksonModuleScala, json4sExt, @@ -161,6 +164,12 @@ object Dependencies { lazy val playJson = playOrg %% "play-json" % Versions.playJson lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test + lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson + lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson + lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson + lazy val circeGenericExtras = "io.circe" %% "circe-generic-extras" % Versions.circeJson + lazy val circeSttp = "com.softwaremill.sttp.client3" %% "circe" % Versions.sttp + // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb @@ -191,7 +200,11 @@ object Dependencies { tapirPlayJson, tapirPrometheus, tapirStubServer, + circeGenericExtras, playJson, + circeCore, + circeGeneric, + circeParser, sttpPlayJson, awsSecretsManagerSdk, zioTest, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 563d8ace3..0b0df1b8b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -16,10 +16,12 @@ package za.co.absa.atum.server.api.database.runs.functions + import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read -import play.api.libs.json.Json +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.atum.model.dto.AdditionalDataSubmitDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs @@ -34,12 +36,12 @@ import zio.interop.catz._ import doobie.postgres.implicits._ class CreateOrUpdateAdditionalData(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[AdditionalDataSubmitDTO, Unit, Task] with StandardStatusHandling { override def sql(values: AdditionalDataSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = Json.toJson(partitioning).toString + val partitioningJsonString = partitioning.asJson.noSpaces // implicits from Doobie can't handle Map[String, Option[String]] -> HStore, so we converted None to null basically val additionalDataNormalized = values.additionalData.map{ case (k, v) => (k, v.orNull)} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 8c6135476..82aa9d056 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -16,10 +16,12 @@ package za.co.absa.atum.server.api.database.runs.functions + import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read -import play.api.libs.json.Json +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.atum.model.dto.PartitioningSubmitDTO import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.fadb.DBSchema @@ -32,16 +34,16 @@ import zio._ import zio.interop.catz._ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) - extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] + extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] with StandardStatusHandling { override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = Json.toJson(partitioning).toString + val partitioningJsonString = partitioning.asJson.noSpaces val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - Json.toJson(parentPartitioningForDB).toString + parentPartitioningForDB.asJson.noSpaces } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( @@ -49,8 +51,8 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningJsonString }, - ${values.authorIfNew}, - ${ + ${values.authorIfNew}, + ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString parentPartitioningJsonString } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 35b615dad..25e45c9c4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -30,7 +30,7 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import play.api.libs.json.Json -import za.co.absa.atum.server.model.PlayJsonImplicits.writesMeasurementDTO +import za.co.absa.atum.server.model.CirceJsonImplicit._ import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 7babe13dc..25d38baa0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -24,7 +24,7 @@ import sttp.tapir.ztapir._ import za.co.absa.atum.model.dto.{AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO, AdditionalDataSubmitDTO} import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse -import za.co.absa.atum.server.model.PlayJsonImplicits._ +import za.co.absa.atum.server.model.CirceJsonImplicits._ trait Endpoints extends BaseEndpoints { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala new file mode 100644 index 000000000..450e2e56c --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.model + +import io.circe._, io.circe.generic.semiauto._ +import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import za.co.absa.atum.model.dto._ + +object CirceJsonImplicits { + + implicit val decodeOptionString: Decoder[Option[String]] = Decoder.decodeOption[String] + implicit val encodeOptionString: Encoder[Option[String]] = Encoder.encodeOption[String] + + implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { + case "String" => Right(ResultValueType.String) + case "Long" => Right(ResultValueType.Long) + case "BigDecimal" => Right(ResultValueType.BigDecimal) + case "Double" => Right(ResultValueType.Double) + case _ => Left("Invalid ResultValueType") + } + + implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap[ResultValueType] { + case ResultValueType.String => "String" + case ResultValueType.Long => "Long" + case ResultValueType.BigDecimal => "BigDecimal" + case ResultValueType.Double => "Double" + } + + implicit val decodeTypedValue: Decoder[MeasureResultDTO.TypedValue] = deriveDecoder + implicit val encodeTypedValue: Encoder[MeasureResultDTO.TypedValue] = deriveEncoder + + implicit val decodeMeasureResultDTO: Decoder[MeasureResultDTO] = deriveDecoder + implicit val encodeMeasureResultDTO: Encoder[MeasureResultDTO] = deriveEncoder + + implicit val decodeMeasureDTO: Decoder[MeasureDTO] = deriveDecoder + implicit val encodeMeasureDTO: Encoder[MeasureDTO] = deriveEncoder + + implicit val decodeMeasurementDTO: Decoder[MeasurementDTO] = deriveDecoder + implicit val encodeMeasurementDTO: Encoder[MeasurementDTO] = deriveEncoder + + implicit val decodePartitionDTO: Decoder[PartitionDTO] = deriveDecoder + implicit val encodePartitionDTO: Encoder[PartitionDTO] = deriveEncoder + + implicit val decodeCheckpointDTO: Decoder[CheckpointDTO] = deriveDecoder + implicit val encodeCheckpointDTO: Encoder[CheckpointDTO] = deriveEncoder + + implicit val decodePartitioningSubmitDTO: Decoder[PartitioningSubmitDTO] = deriveDecoder + implicit val encodePartitioningSubmitDTO: Encoder[PartitioningSubmitDTO] = deriveEncoder + + implicit val decodeStringMap: Decoder[Map[String, Option[String]]] = Decoder.decodeMap[String, Option[String]] + implicit val encodeStringMap: Encoder[Map[String, Option[String]]] = Encoder.encodeMap[String, Option[String]] + + implicit val decodeAdditionalDataSubmitDTO: Decoder[AdditionalDataSubmitDTO] = deriveDecoder + implicit val encodeAdditionalDataSubmitDTO: Encoder[AdditionalDataSubmitDTO] = deriveEncoder + + implicit val decodeAtumContextDTO: Decoder[AtumContextDTO] = deriveDecoder + implicit val encodeAtumContextDTO: Encoder[AtumContextDTO] = deriveEncoder + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index 2fbc0c697..c1af90ad1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -16,34 +16,66 @@ package za.co.absa.atum.server.model -import play.api.libs.json.{Json, Reads, Writes} +//import play.api.libs.json.{Json, Reads, Writes} +// +//sealed trait ErrorResponse { +// def message: String +//} +// +//object ErrorResponse { +// implicit val reads: Reads[ErrorResponse] = Json.reads[ErrorResponse] +// implicit val writes: Writes[ErrorResponse] = Json.writes[ErrorResponse] +//} +// +//final case class BadRequestResponse(message: String) extends ErrorResponse +// +//object BadRequestResponse { +// implicit val reads: Reads[BadRequestResponse] = Json.reads[BadRequestResponse] +// implicit val writes: Writes[BadRequestResponse] = Json.writes[BadRequestResponse] +//} +// +//final case class GeneralErrorResponse(message: String) extends ErrorResponse +// +//object GeneralErrorResponse { +// implicit val reads: Reads[GeneralErrorResponse] = Json.reads[GeneralErrorResponse] +// implicit val writes: Writes[GeneralErrorResponse] = Json.writes[GeneralErrorResponse] +//} +// +//final case class InternalServerErrorResponse(message: String) extends ErrorResponse +// +//object InternalServerErrorResponse { +// implicit val reads: Reads[InternalServerErrorResponse] = Json.reads[InternalServerErrorResponse] +// implicit val writes: Writes[InternalServerErrorResponse] = Json.writes[InternalServerErrorResponse] +//} + +import io.circe._, io.circe.generic.semiauto._ sealed trait ErrorResponse { def message: String } object ErrorResponse { - implicit val reads: Reads[ErrorResponse] = Json.reads[ErrorResponse] - implicit val writes: Writes[ErrorResponse] = Json.writes[ErrorResponse] + implicit val decodeErrorResponse: Decoder[ErrorResponse] = deriveDecoder + implicit val encodeErrorResponse: Encoder[ErrorResponse] = deriveEncoder } final case class BadRequestResponse(message: String) extends ErrorResponse object BadRequestResponse { - implicit val reads: Reads[BadRequestResponse] = Json.reads[BadRequestResponse] - implicit val writes: Writes[BadRequestResponse] = Json.writes[BadRequestResponse] + implicit val decodeBadRequestResponse: Decoder[BadRequestResponse] = deriveDecoder + implicit val encodeBadRequestResponse: Encoder[BadRequestResponse] = deriveEncoder } final case class GeneralErrorResponse(message: String) extends ErrorResponse object GeneralErrorResponse { - implicit val reads: Reads[GeneralErrorResponse] = Json.reads[GeneralErrorResponse] - implicit val writes: Writes[GeneralErrorResponse] = Json.writes[GeneralErrorResponse] + implicit val decodeGeneralErrorResponse: Decoder[GeneralErrorResponse] = deriveDecoder + implicit val encodeGeneralErrorResponse: Encoder[GeneralErrorResponse] = deriveEncoder } final case class InternalServerErrorResponse(message: String) extends ErrorResponse object InternalServerErrorResponse { - implicit val reads: Reads[InternalServerErrorResponse] = Json.reads[InternalServerErrorResponse] - implicit val writes: Writes[InternalServerErrorResponse] = Json.writes[InternalServerErrorResponse] + implicit val decodeInternalServerErrorResponse: Decoder[InternalServerErrorResponse] = deriveDecoder + implicit val encodeInternalServerErrorResponse: Encoder[InternalServerErrorResponse] = deriveEncoder } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index 55a349923..bb5a14006 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,14 +16,36 @@ package za.co.absa.atum.server.model -import play.api.libs.json.{Json, Writes} +//import play.api.libs.json.{Json, Writes} +//import za.co.absa.atum.model.dto.PartitioningDTO +// +//private[server] case class PartitioningForDB private ( +// version: Int = 1, +// keys: Seq[String], +// keysToValues: Map[String, String] +//) +// +//object PartitioningForDB { +// +// def fromSeqPartitionDTO(partitioning: PartitioningDTO): PartitioningForDB = { +// val allKeys = partitioning.map(_.key) +// val mapOfKeysAndValues = partitioning.map(p => p.key -> p.value).toMap[String, String] +// +// PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) +// } +// +// implicit val writes: Writes[PartitioningForDB] = Json.writes +// +//} + +import io.circe._, io.circe.generic.semiauto._ import za.co.absa.atum.model.dto.PartitioningDTO private[server] case class PartitioningForDB private ( - version: Int = 1, - keys: Seq[String], - keysToValues: Map[String, String] -) + version: Int = 1, + keys: Seq[String], + keysToValues: Map[String, String] + ) object PartitioningForDB { @@ -34,6 +56,6 @@ object PartitioningForDB { PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) } - implicit val writes: Writes[PartitioningForDB] = Json.writes + implicit val encodePartitioningForDB: Encoder[PartitioningForDB] = deriveEncoder } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala deleted file mode 100644 index 03a3776df..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import play.api.libs.functional.syntax.toFunctionalBuilderOps -import play.api.libs.json._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} -import za.co.absa.atum.model.dto._ - -object PlayJsonImplicits { - - implicit val optionStringReads: Reads[Option[String]] = new Reads[Option[String]] { - def reads(json: JsValue): JsResult[Option[String]] = json match { - case JsNull => JsSuccess(None) - case JsString(s) => JsSuccess(Some(s)) - case _ => JsError("Expected JsString or JsNull") - } - } - - implicit val optionStringWrites: Writes[Option[String]] = new Writes[Option[String]] { - def writes(opt: Option[String]): JsValue = opt match { - case Some(s) => JsString(s) - case None => JsNull - } - } - - implicit val resultValueTypeReads: Reads[ResultValueType] = new Reads[ResultValueType] { - override def reads(json: JsValue): JsResult[ResultValueType] = json match { - case JsString("String") => JsSuccess(ResultValueType.String) - case JsString("Long") => JsSuccess(ResultValueType.Long) - case JsString("BigDecimal") => JsSuccess(ResultValueType.BigDecimal) - case JsString("Double") => JsSuccess(ResultValueType.Double) - case _ => JsError("Invalid ResultValueType") - } - } - - implicit val resultValueTypeWrites: Writes[ResultValueType] = new Writes[ResultValueType] { - def writes(resultValueType: ResultValueType): JsValue = resultValueType match { - case ResultValueType.String => Json.toJson("String") - case ResultValueType.Long => Json.toJson("Long") - case ResultValueType.BigDecimal => Json.toJson("BigDecimal") - case ResultValueType.Double => Json.toJson("Double") - } - } - - implicit val readsTypedValue: Reads[MeasureResultDTO.TypedValue] = Json.reads[MeasureResultDTO.TypedValue] - implicit val writesTypedValue: Writes[MeasureResultDTO.TypedValue] = Json.writes[MeasureResultDTO.TypedValue] - - implicit val readsMeasureResultDTO: Reads[MeasureResultDTO] = { - ((__ \ "mainValue").read[MeasureResultDTO.TypedValue] and - (__ \ "supportValues").readNullable[Map[String, TypedValue]].map(_.getOrElse(Map.empty)) - )(MeasureResultDTO.apply _) - } - - implicit val writesMeasureResultDTO: Writes[MeasureResultDTO] = Json.writes[MeasureResultDTO] - - implicit val readsMeasureDTO: Reads[MeasureDTO] = Json.reads[MeasureDTO] - implicit val writesMeasureDTO: Writes[MeasureDTO] = Json.writes[MeasureDTO] - - implicit val readsMeasurementDTO: Reads[MeasurementDTO] = Json.reads[MeasurementDTO] - implicit val writesMeasurementDTO: Writes[MeasurementDTO] = Json.writes[MeasurementDTO] - - implicit val readsPartitionDTO: Reads[PartitionDTO] = Json.reads[PartitionDTO] - implicit val writesPartitionDTO: Writes[PartitionDTO] = Json.writes[PartitionDTO] - - implicit val readsCheckpointDTO: Reads[CheckpointDTO] = Json.reads[CheckpointDTO] - implicit val writesCheckpointDTO: Writes[CheckpointDTO] = Json.writes[CheckpointDTO] - - implicit val readsPartitioningSubmitDTO: Reads[PartitioningSubmitDTO] = Json.reads[PartitioningSubmitDTO] - implicit val writesPartitioningSubmitDTO: Writes[PartitioningSubmitDTO] = Json.writes[PartitioningSubmitDTO] - - implicit val readsStringMap: Reads[Map[String, Option[String]]] = Reads.mapReads[Option[String]] - implicit val writesStringMap: OWrites[MapWrites.Map[String, Option[String]]] = - Writes.genericMapWrites[Option[String], MapWrites.Map] - - implicit val readsAdditionalDataSubmitDTO: Reads[AdditionalDataSubmitDTO] = Json.reads[AdditionalDataSubmitDTO] - implicit val writesAdditionalDataSubmitDTO: Writes[AdditionalDataSubmitDTO] = Json.writes[AdditionalDataSubmitDTO] - - implicit val readsAtumContextDTO: Reads[AtumContextDTO] = Json.reads[AtumContextDTO] - implicit val writesAtumContextDTO: Writes[AtumContextDTO] = Json.writes[AtumContextDTO] - -} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala index c53a2db82..5cf59d286 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointIntegrationTest.scala @@ -29,7 +29,7 @@ import za.co.absa.atum.server.api.controller.CheckpointController import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} import zio.test._ import zio._ -import za.co.absa.atum.server.model.PlayJsonImplicits.{readsCheckpointDTO, writesCheckpointDTO} +import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeCheckpointDTO, encodeCheckpointDTO} import zio.test.Assertion.equalTo object CreateCheckpointEndpointIntegrationTests extends ZIOSpecDefault with Endpoints with TestData { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala index b7a3de799..923c3284c 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala @@ -30,7 +30,7 @@ import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorRe import zio.test.Assertion.equalTo import zio._ import zio.test._ -import za.co.absa.atum.server.model.PlayJsonImplicits.{readsAtumContextDTO, writesPartitioningSubmitDTO} +import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} object CreatePartitioningEndpointIntegrationTests extends ZIOSpecDefault with Endpoints with TestData { From ac908df2adf085e6b3ecffb296a5a4799367ff27 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 17 Jun 2024 13:18:07 +0200 Subject: [PATCH 02/11] modifying dependencies --- project/Dependencies.scala | 5 +++ .../absa/atum/server/api/http/Endpoints.scala | 7 ++-- ...PartitioningEndpointIntegrationTests.scala | 34 ++++++++++++++----- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6f952da64..d53aad28c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -76,6 +76,7 @@ object Dependencies { val http4sPrometheus = "0.23.6" val playJson = "3.0.1" val sttpPlayJson = "3.9.3" + val sttpCirceJson = "4.0.0-M16" val awssdk = "2.23.15" @@ -140,6 +141,7 @@ object Dependencies { val logbackOrg = "ch.qos.logback" val awsSdkOrg = "software.amazon.awssdk" val sttpClient3Org = "com.softwaremill.sttp.client3" + val sttpClient4Org = "com.softwaremill.sttp.client4" // zio lazy val zioCore = zioOrg %% "zio" % Versions.zio @@ -166,6 +168,8 @@ object Dependencies { // json lazy val playJson = playOrg %% "play-json" % Versions.playJson lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test + lazy val sttpCirceJson = sttpClient4Org %% "circe" % Versions.sttpCirceJson % Test + // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb @@ -201,6 +205,7 @@ object Dependencies { tapirStubServer, playJson, sttpPlayJson, + sttpCirceJson, awsSecretsManagerSdk, zioTest, zioTestSbt, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 4c63adbf8..b98f0e374 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -21,14 +21,13 @@ import sttp.tapir.{PublicEndpoint, endpoint} import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.play.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{PublicEndpoint, endpoint} import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse -//import za.co.absa.atum.server.model.PlayJsonImplicits._ -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.ErrorResponse +import za.co.absa.atum.server.model.PlayJsonImplicits._ +import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.CirceJsonImplicits._ +import sttp.tapir.{PublicEndpoint, endpoint} trait Endpoints extends BaseEndpoints { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala index 4a9c1824a..f1e15a501 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointIntegrationTests.scala @@ -17,22 +17,33 @@ package za.co.absa.atum.server.api.http import org.mockito.Mockito.{mock, when} -import sttp.client3._ -import sttp.client3.playJson._ -import sttp.client3.testing.SttpBackendStub +//import sttp.client3._ +//import sttp.client3.playJson._ +//import sttp.client3.testing.SttpBackendStub +import sttp.client4._ +import sttp.client4.circe._ +import io.circe.generic.auto._ +import sttp.client4.testing._ +//import scala.concurrent.Future +//import scala.concurrent.ExecutionContext.Implicits.global +//import sttp.client4._ +//import sttp.client4.testing._ +//import sttp.client4.circe._ +//import io.circe.generic.auto._ + import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} //import za.co.absa.atum.server.model.PlayJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo import zio.test._ -import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} +//import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} object CreatePartitioningEndpointIntegrationTests extends ZIOSpecDefault with Endpoints with TestData { @@ -51,10 +62,15 @@ object CreatePartitioningEndpointIntegrationTests extends ZIOSpecDefault with En createPartitioningEndpointV2.zServerLogic(PartitioningController.createPartitioningIfNotExistsV2) def spec: Spec[TestEnvironment with Scope, Any] = { - val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[PartitioningController])) - .whenServerEndpoint(createPartitioningServerEndpoint) - .thenRunLogic() - .backend() + val backendStub = SttpBackendStub(new RIOMonadError[PartitioningController]) + .whenRequestMatches(_ => true) + .thenRespondWrapped( + createPartitioningServerEndpoint.logic(_) + .foldM( + e => ZIO.succeed(Response(e, StatusCode.BadRequest)), + s => ZIO.succeed(Response.ok(s)) + ) + ) val request = basicRequest .post(uri"https://test.com/api/v2/createPartitioning") From 852d4fd6b5ac8403022779321a530ca198e0328b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 18 Jun 2024 10:57:42 +0200 Subject: [PATCH 03/11] using circe syntax in flow classes --- .../api/database/flows/functions/GetFlowCheckpoints.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 0b30ee8d3..b8102987e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -19,7 +19,6 @@ package za.co.absa.atum.server.api.database.flows.functions import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read -import play.api.libs.json.Json import za.co.absa.atum.model.dto.CheckpointQueryDTO import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.flows.Flows @@ -51,7 +50,7 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] override def sql(values: CheckpointQueryDTO)(implicit read: Read[CheckpointFromDB]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = Json.toJson(partitioning).toString + val partitioningNormalized = partitioning.asJson.noSpaces sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( From 37b54348bdbd83dee95268608102911341b65d72 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 18 Jun 2024 11:24:51 +0200 Subject: [PATCH 04/11] using circe in api http module --- .../za/co/absa/atum/server/api/http/BaseEndpoints.scala | 4 ++-- .../scala/za/co/absa/atum/server/api/http/Endpoints.scala | 6 +++--- .../za/co/absa/atum/server/api/http/ServerOptions.scala | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index 4f067fbc5..422f044ef 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -18,12 +18,12 @@ package za.co.absa.atum.server.api.http import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass -import sttp.tapir.json.play.jsonBody +import sttp.tapir.json.circe.jsonBody +import za.co.absa.atum.server.model.{BadRequestResponse, ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import sttp.tapir.typelevel.MatchType import sttp.tapir.ztapir._ import sttp.tapir.{EndpointOutput, PublicEndpoint} import za.co.absa.atum.server.Constants.Endpoints.{Api, V1, V2} -import za.co.absa.atum.server.model.ErrorResponse._ import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index e7381f0bd..d7cd1dc1f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -16,15 +16,15 @@ package za.co.absa.atum.server.api.http + +import io.circe.generic.auto.{exportDecoder, exportEncoder} import sttp.model.StatusCode -import sttp.tapir.{PublicEndpoint, endpoint} import sttp.tapir.generic.auto.schemaForCaseClass -import sttp.tapir.json.play.jsonBody import sttp.tapir.ztapir._ +import sttp.tapir.json.circe.jsonBody import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.Constants.Endpoints._ import za.co.absa.atum.server.model.ErrorResponse -import za.co.absa.atum.server.model.PlayJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.CirceJsonImplicits._ import sttp.tapir.{PublicEndpoint, endpoint} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala index f6bbe79cd..c42f6343d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala @@ -18,7 +18,6 @@ package za.co.absa.atum.server.api.http import sttp.monad.MonadError import sttp.tapir.DecodeResult -import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.play.jsonBody import sttp.tapir.server.http4s.Http4sServerOptions import sttp.tapir.server.interceptor.DecodeFailureContext @@ -27,7 +26,7 @@ import sttp.tapir.server.interceptor.decodefailure.DefaultDecodeFailureHandler.r import sttp.tapir.server.interceptor.metrics.MetricsRequestInterceptor import sttp.tapir.server.model.ValuedEndpointOutput import sttp.tapir.ztapir.{headers, statusCode} -import za.co.absa.atum.server.model.ErrorResponse.BadRequestResponse +import za.co.absa.atum.server.model.BadRequestResponse import zio.interop.catz._ trait ServerOptions { From eb77ba5326e994a4a9821ffbd5dcab80f8d6b84f Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 18 Jun 2024 11:59:25 +0200 Subject: [PATCH 05/11] fix implicit import in ServerOptions --- .../scala/za/co/absa/atum/server/api/http/ServerOptions.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala index c42f6343d..1eefa98c1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/ServerOptions.scala @@ -18,7 +18,8 @@ package za.co.absa.atum.server.api.http import sttp.monad.MonadError import sttp.tapir.DecodeResult -import sttp.tapir.json.play.jsonBody +import sttp.tapir.generic.auto.schemaForCaseClass +import sttp.tapir.json.circe.jsonBody import sttp.tapir.server.http4s.Http4sServerOptions import sttp.tapir.server.interceptor.DecodeFailureContext import sttp.tapir.server.interceptor.decodefailure.DecodeFailureHandler From f6d2ece553c6a8f777bf8f9e3bfda27166531c5d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jun 2024 13:23:04 +0200 Subject: [PATCH 06/11] merge master, fix dependencies and conclude server implicit implementation --- project/Dependencies.scala | 28 +++++++++++---- .../api/controller/BaseController.scala | 2 +- .../api/controller/CheckpointController.scala | 2 +- .../controller/CheckpointControllerImpl.scala | 2 +- .../api/controller/FlowController.scala | 2 +- .../api/controller/FlowControllerImpl.scala | 2 +- .../controller/PartitioningController.scala | 2 +- .../PartitioningControllerImpl.scala | 2 +- .../CreateOrUpdateAdditionalData.scala | 1 - .../CreatePartitioningIfNotExists.scala | 1 - .../GetPartitioningAdditionalData.scala | 1 - .../GetPartitioningCheckpoints.scala | 1 - .../functions/GetPartitioningMeasures.scala | 1 - .../absa/atum/server/api/http/Endpoints.scala | 23 +++++------- .../server/model/CirceJsonImplicits.scala | 5 +-- .../atum/server/model/ErrorResponse.scala | 31 ---------------- .../atum/server/model/PartitioningForDB.scala | 35 +++++-------------- .../CheckpointControllerUnitTests.scala | 2 +- .../controller/FlowControllerUnitTests.scala | 2 +- .../PartitioningControllerUnitTests.scala | 4 +-- .../CreateCheckpointEndpointUnitTests.scala | 21 +++++------ .../CreatePartitioningEndpointUnitTests.scala | 19 ++++------ .../GetFlowCheckpointsEndpointUnitTests.scala | 8 ++--- 23 files changed, 72 insertions(+), 125 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 50e530e18..29769d1c0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -28,7 +28,6 @@ object Dependencies { val balta = "0.1.0" val jacksonModuleScala = "2.14.2" - val circeVersion = "0.14.5" val specs2 = "4.10.0" val typesafeConfig = "1.4.2" @@ -76,7 +75,8 @@ object Dependencies { val http4sPrometheus = "0.23.6" val playJson = "3.0.1" val sttpPlayJson = "3.9.3" - val sttpCirceJson = "4.0.0-M16" + val circeJson = "0.14.7" + val sttpCirceJson = "3.9.7" val awssdk = "2.23.15" @@ -117,8 +117,10 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided - lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeVersion - lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeVersion + // Circe dependencies + lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson + lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson + lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson Seq( jacksonModuleScala, @@ -128,6 +130,7 @@ object Dependencies { json4sNative, circeCore, circeParser, + circeGeneric, ) } @@ -141,7 +144,6 @@ object Dependencies { val logbackOrg = "ch.qos.logback" val awsSdkOrg = "software.amazon.awssdk" val sttpClient3Org = "com.softwaremill.sttp.client3" - val sttpClient4Org = "com.softwaremill.sttp.client4" // zio lazy val zioCore = zioOrg %% "zio" % Versions.zio @@ -165,11 +167,21 @@ object Dependencies { lazy val tapirPrometheus = tapirOrg %% "tapir-prometheus-metrics" % Versions.tapir lazy val tapirStubServer = tapirOrg %% "tapir-sttp-stub-server" % Versions.tapir % Test + lazy val tapirCirce = tapirOrg %% "tapir-json-circe" % Versions.tapir + lazy val tapirOpenApiDocs = tapirOrg %% "tapir-openapi-docs" % Versions.tapir + lazy val tapirOpenApiCirceYaml = tapirOrg %% "tapir-openapi-circe-yaml" % Versions.tapir + lazy val tapirHttp4sServer = tapirOrg %% "tapir-http4s-server" % Versions.tapir + lazy val tapirCore = tapirOrg %% "tapir-core" % Versions.tapir + lazy val tapirSwaggerUi = tapirOrg %% "tapir-swagger-ui-http4s" % Versions.tapir + // json lazy val playJson = playOrg %% "play-json" % Versions.playJson lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test - lazy val sttpCirceJson = sttpClient4Org %% "circe" % Versions.sttpCirceJson % Test + // STTP core and Circe integration + lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpCirceJson % Test + lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpCirceJson + lazy val clientBackend = sttpClient3Org %% "async-http-client-backend-zio" % Versions.sttpCirceJson // Fa-db lazy val faDbDoobie = faDbOrg %% "doobie" % Versions.fadb @@ -201,11 +213,13 @@ object Dependencies { tapirHttp4sZio, tapirSwagger, tapirPlayJson, + tapirCirce, tapirPrometheus, tapirStubServer, playJson, sttpPlayJson, - sttpCirceJson, + sttpCirce, + sttpCore, awsSecretsManagerSdk, zioTest, zioTestSbt, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 1877cc0b4..9bfe5c83e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.atum.server.model.ErrorResponse.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import za.co.absa.fadb.exceptions.StatusException import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala index 402b67884..6b547c3cb 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointController.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 69a140d55..8815a89c2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala index 20783e9f4..d122a93c2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowController.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointQueryDTO} -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.MultiSuccessResponse import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala index 23d12d8d2..8382b212c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/FlowControllerImpl.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointQueryDTO} import za.co.absa.atum.server.api.service.FlowService -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.MultiSuccessResponse import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala index c5e7dc737..2dfc53016 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningController.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.model.dto.{ CheckpointQueryDTO, PartitioningSubmitDTO } -import za.co.absa.atum.server.model.ErrorResponse.ErrorResponse +import za.co.absa.atum.server.model.ErrorResponse import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio.IO import zio.macros.accessible diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala index 964d57634..d37cc94a1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.PartitioningService -import za.co.absa.atum.server.model.ErrorResponse.{ErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{ErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} import zio._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala index 8447ebff5..f7b8da0d3 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreateOrUpdateAdditionalData.scala @@ -31,7 +31,6 @@ import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import doobie.postgres.implicits._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 5adb17aa5..3da9fd091 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -31,7 +31,6 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[PartitioningSubmitDTO, Unit, Task] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala index 9992b75b5..961076ac8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningAdditionalData.scala @@ -29,7 +29,6 @@ import za.co.absa.fadb.doobie.DoobieEngine import zio.interop.catz.asyncInstance import zio.{Task, URLayer, ZIO, ZLayer} import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.getMapWithOptionStringValues diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 4f043b43c..52a6832f8 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -29,7 +29,6 @@ import za.co.absa.fadb.doobie.DoobieFunction.DoobieMultipleResultFunction import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala index f2a6d7c97..5d541417b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningMeasures.scala @@ -29,7 +29,6 @@ import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ import io.circe.syntax._ -import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index d7cd1dc1f..f2d03fffd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -29,10 +29,10 @@ import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, Singl import za.co.absa.atum.server.model.CirceJsonImplicits._ import sttp.tapir.{PublicEndpoint, endpoint} + trait Endpoints extends BaseEndpoints { - protected val createCheckpointEndpointV1 - : PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { + protected val createCheckpointEndpointV1: PublicEndpoint[CheckpointDTO, ErrorResponse, CheckpointDTO, Any] = { apiV1.post .in(pathToAPIv1CompatibleFormat(CreateCheckpoint)) .in(jsonBody[CheckpointDTO]) @@ -40,8 +40,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[CheckpointDTO]) } - protected val createCheckpointEndpointV2 - : PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { + protected val createCheckpointEndpointV2: PublicEndpoint[CheckpointDTO, ErrorResponse, SingleSuccessResponse[CheckpointDTO], Any] = { apiV2.post .in(CreateCheckpoint) .in(jsonBody[CheckpointDTO]) @@ -49,8 +48,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[CheckpointDTO]]) } - protected val createPartitioningEndpointV1 - : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { + protected val createPartitioningEndpointV1: PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, AtumContextDTO, Any] = { apiV1.post .in(pathToAPIv1CompatibleFormat(CreatePartitioning)) .in(jsonBody[PartitioningSubmitDTO]) @@ -58,8 +56,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[AtumContextDTO]) } - protected val createPartitioningEndpointV2 - : PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, SingleSuccessResponse[AtumContextDTO], Any] = { + protected val createPartitioningEndpointV2: PublicEndpoint[PartitioningSubmitDTO, ErrorResponse, SingleSuccessResponse[AtumContextDTO], Any] = { apiV2.post .in(CreatePartitioning) .in(jsonBody[PartitioningSubmitDTO]) @@ -67,8 +64,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AtumContextDTO]]) } - protected val createOrUpdateAdditionalDataEndpointV2 - : PublicEndpoint[AdditionalDataSubmitDTO, ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO], Any] = { + protected val createOrUpdateAdditionalDataEndpointV2: PublicEndpoint[AdditionalDataSubmitDTO, ErrorResponse, SingleSuccessResponse[AdditionalDataSubmitDTO], Any] = { apiV2.post .in(CreateOrUpdateAdditionalData) .in(jsonBody[AdditionalDataSubmitDTO]) @@ -76,8 +72,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[SingleSuccessResponse[AdditionalDataSubmitDTO]]) } - protected val getPartitioningCheckpointsEndpointV2 - : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { + protected val getPartitioningCheckpointsEndpointV2: PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.get .in(GetPartitioningCheckpoints) .in(jsonBody[CheckpointQueryDTO]) @@ -85,8 +80,7 @@ trait Endpoints extends BaseEndpoints { .out(jsonBody[MultiSuccessResponse[CheckpointDTO]]) } - protected val getFlowCheckpointsEndpointV2 - : PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { + protected val getFlowCheckpointsEndpointV2: PublicEndpoint[CheckpointQueryDTO, ErrorResponse, MultiSuccessResponse[CheckpointDTO], Any] = { apiV2.post .in(GetFlowCheckpoints) .in(jsonBody[CheckpointQueryDTO]) @@ -100,5 +94,4 @@ trait Endpoints extends BaseEndpoints { protected val healthEndpoint: PublicEndpoint[Unit, Unit, Unit, Any] = endpoint.get.in(Health) - } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala index 450e2e56c..75df93885 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CirceJsonImplicits.scala @@ -16,8 +16,9 @@ package za.co.absa.atum.server.model -import io.circe._, io.circe.generic.semiauto._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} +import io.circe._ +import io.circe.generic.semiauto._ +import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType import za.co.absa.atum.model.dto._ object CirceJsonImplicits { diff --git a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala index 821fdcc56..bf17272b4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/ErrorResponse.scala @@ -16,37 +16,6 @@ package za.co.absa.atum.server.model -//import play.api.libs.json.{Json, Reads, Writes} -// -//sealed trait ErrorResponse { -// def message: String -//} -// -//object ErrorResponse { -// implicit val reads: Reads[ErrorResponse] = Json.reads[ErrorResponse] -// implicit val writes: Writes[ErrorResponse] = Json.writes[ErrorResponse] -//} -// -//final case class BadRequestResponse(message: String) extends ErrorResponse -// -//object BadRequestResponse { -// implicit val reads: Reads[BadRequestResponse] = Json.reads[BadRequestResponse] -// implicit val writes: Writes[BadRequestResponse] = Json.writes[BadRequestResponse] -//} -// -//final case class GeneralErrorResponse(message: String) extends ErrorResponse -// -//object GeneralErrorResponse { -// implicit val reads: Reads[GeneralErrorResponse] = Json.reads[GeneralErrorResponse] -// implicit val writes: Writes[GeneralErrorResponse] = Json.writes[GeneralErrorResponse] -//} -// -//final case class InternalServerErrorResponse(message: String) extends ErrorResponse -// -//object InternalServerErrorResponse { -// implicit val reads: Reads[InternalServerErrorResponse] = Json.reads[InternalServerErrorResponse] -// implicit val writes: Writes[InternalServerErrorResponse] = Json.writes[InternalServerErrorResponse] -//} import io.circe._ import io.circe.generic.semiauto._ diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index bb5a14006..ad987d659 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,30 +16,11 @@ package za.co.absa.atum.server.model -//import play.api.libs.json.{Json, Writes} -//import za.co.absa.atum.model.dto.PartitioningDTO -// -//private[server] case class PartitioningForDB private ( -// version: Int = 1, -// keys: Seq[String], -// keysToValues: Map[String, String] -//) -// -//object PartitioningForDB { -// -// def fromSeqPartitionDTO(partitioning: PartitioningDTO): PartitioningForDB = { -// val allKeys = partitioning.map(_.key) -// val mapOfKeysAndValues = partitioning.map(p => p.key -> p.value).toMap[String, String] -// -// PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) -// } -// -// implicit val writes: Writes[PartitioningForDB] = Json.writes -// -//} - -import io.circe._, io.circe.generic.semiauto._ + +import io.circe.generic.semiauto._ +import io.circe.{Decoder, Encoder} import za.co.absa.atum.model.dto.PartitioningDTO +import scala.collection.immutable.Seq private[server] case class PartitioningForDB private ( version: Int = 1, @@ -50,12 +31,12 @@ private[server] case class PartitioningForDB private ( object PartitioningForDB { def fromSeqPartitionDTO(partitioning: PartitioningDTO): PartitioningForDB = { - val allKeys = partitioning.map(_.key) - val mapOfKeysAndValues = partitioning.map(p => p.key -> p.value).toMap[String, String] + val allKeys: Seq[String] = partitioning.map(_.key) + val mapOfKeysAndValues: Map[String, String] = partitioning.map(p => p.key -> p.value).toMap PartitioningForDB(keys = allKeys, keysToValues = mapOfKeysAndValues) } - implicit val encodePartitioningForDB: Encoder[PartitioningForDB] = deriveEncoder - + implicit val encoder: Encoder[PartitioningForDB] = deriveEncoder + implicit val decoder: Decoder[PartitioningForDB] = deriveDecoder } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala index 80cdf125c..faa02bc01 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/CheckpointControllerUnitTests.scala @@ -20,7 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.CheckpointService -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.fadb.exceptions.ErrorInDataException import za.co.absa.fadb.status.FunctionStatus import zio.test.Assertion.failsWithA diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala index 98bd3a0f1..be3f586b7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/FlowControllerUnitTests.scala @@ -20,7 +20,7 @@ import org.mockito.Mockito.{mock, when} import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.FlowService -import za.co.absa.atum.server.model.ErrorResponse.InternalServerErrorResponse +import za.co.absa.atum.server.model.InternalServerErrorResponse import zio._ import zio.test.Assertion.failsWithA import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala index 340949d8e..7de77eb40 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/controller/PartitioningControllerUnitTests.scala @@ -21,8 +21,8 @@ import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.service.PartitioningService -import za.co.absa.atum.server.model.ErrorResponse.InternalServerErrorResponse -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} +import za.co.absa.atum.server.model.InternalServerErrorResponse +import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.{equalTo, failsWithA} import zio.test._ diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala index a3a2eb340..7fb322666 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreateCheckpointEndpointUnitTests.scala @@ -16,20 +16,22 @@ package za.co.absa.atum.server.api.http +import io.circe.generic.auto._ +import io.circe.syntax.EncoderOps import org.mockito.Mockito.{mock, when} -import sttp.client3._ -import sttp.client3.playJson._ import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.CheckpointController -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.CirceJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ -import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeCheckpointDTO, encodeCheckpointDTO} import zio.test.Assertion.equalTo import zio.test._ @@ -46,8 +48,8 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) - private val createCheckpointServerEndpoint = - createCheckpointEndpointV2.zServerLogic(CheckpointController.createCheckpointV2) + private val createCheckpointServerEndpoint = createCheckpointEndpointV2 + .zServerLogic(CheckpointController.createCheckpointV2) def spec: Spec[TestEnvironment with Scope, Any] = { val backendStub = TapirStubInterpreter(SttpBackendStub.apply(new RIOMonadError[CheckpointController])) @@ -62,7 +64,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w suite("CreateCheckpointEndpointSuite")( test("Returns expected CheckpointDTO") { val response = request - .body(checkpointDTO1) + .body(checkpointDTO1.asJson.noSpaces) .send(backendStub) val body = response.map(_.body) @@ -72,7 +74,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected BadRequest") { val response = request - .body(checkpointDTO2) + .body(checkpointDTO2.asJson.noSpaces) .send(backendStub) val statusCode = response.map(_.code) @@ -81,7 +83,7 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }, test("Returns expected InternalServerError") { val response = request - .body(checkpointDTO3) + .body(checkpointDTO3.asJson.noSpaces) .send(backendStub) val statusCode = response.map(_.code) @@ -92,5 +94,4 @@ object CreateCheckpointEndpointUnitTests extends ZIOSpecDefault with Endpoints w }.provide( checkpointControllerMockLayer ) - } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala index e0a9e16a0..5f7022bd7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/CreatePartitioningEndpointUnitTests.scala @@ -16,31 +16,24 @@ package za.co.absa.atum.server.api.http -import org.mockito.Mockito.{mock, when} -//import sttp.client3._ -//import sttp.client3.playJson._ -//import sttp.client3.testing.SttpBackendStub -import sttp.client4._ -import sttp.client4.circe._ import io.circe.generic.auto._ -import sttp.client4.testing._ -import sttp.client4._ +import org.mockito.Mockito.{mock, when} +import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.AtumContextDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.PartitioningController -import za.co.absa.atum.server.model.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} -//import za.co.absa.atum.server.model.PlayJsonImplicits._ -//import za.co.absa.atum.server.model.CirceJsonImplicits.{decodeAtumContextDTO, encodePartitioningSubmitDTO} +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} +import za.co.absa.atum.server.model.CirceJsonImplicits._ import za.co.absa.atum.server.model.SuccessResponse.SingleSuccessResponse import zio._ import zio.test.Assertion.equalTo import zio.test._ -import java.util.UUID - object CreatePartitioningEndpointUnitTests extends ZIOSpecDefault with Endpoints with TestData { private val createPartitioningEndpointMock = mock(classOf[PartitioningController]) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala index 699ae3a34..7b0ae8a26 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -16,18 +16,18 @@ package za.co.absa.atum.server.api.http +import io.circe.generic.auto._ import org.mockito.Mockito.{mock, when} -import sttp.client3._ -import sttp.client3.playJson._ import sttp.client3.testing.SttpBackendStub +import sttp.client3.{UriContext, basicRequest} +import sttp.client3.circe._ import sttp.model.StatusCode import sttp.tapir.server.stub.TapirStubInterpreter import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.controller.FlowController -import za.co.absa.atum.server.model.ErrorResponse.{GeneralErrorResponse, InternalServerErrorResponse} -import za.co.absa.atum.server.model.PlayJsonImplicits._ +import za.co.absa.atum.server.model.{GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.model.SuccessResponse.MultiSuccessResponse import zio._ import zio.test.Assertion.equalTo From 3c707e6e438b20be94b8dde8e8327628ea06d455 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 24 Jun 2024 14:14:40 +0200 Subject: [PATCH 07/11] removing play Json dependencies, and PlayJsonImplicits class --- .../atum/model/dto/MeasureResultDTO.scala | 1 + project/Dependencies.scala | 8 -- .../runs/functions/WriteCheckpoint.scala | 9 +- .../atum/server/model/CheckpointFromDB.scala | 1 + .../atum/server/model/PlayJsonImplicits.scala | 102 ------------------ .../WriteCheckpointIntegrationTests.scala | 4 +- 6 files changed, 12 insertions(+), 113 deletions(-) delete mode 100644 server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 61b26cac6..7bf65ca44 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -17,6 +17,7 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} +import io.circe.generic.auto._ case class MeasureResultDTO( mainValue: MeasureResultDTO.TypedValue, diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 29769d1c0..0f9a58aef 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -73,8 +73,6 @@ object Dependencies { val tapir = "1.9.6" val http4sBlazeBackend = "0.23.15" val http4sPrometheus = "0.23.6" - val playJson = "3.0.1" - val sttpPlayJson = "3.9.3" val circeJson = "0.14.7" val sttpCirceJson = "3.9.7" @@ -174,10 +172,6 @@ object Dependencies { lazy val tapirCore = tapirOrg %% "tapir-core" % Versions.tapir lazy val tapirSwaggerUi = tapirOrg %% "tapir-swagger-ui-http4s" % Versions.tapir - // json - lazy val playJson = playOrg %% "play-json" % Versions.playJson - lazy val sttpPlayJson = sttpClient3Org %% "play-json" % Versions.sttpPlayJson % Test - // STTP core and Circe integration lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpCirceJson % Test lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpCirceJson @@ -216,8 +210,6 @@ object Dependencies { tapirCirce, tapirPrometheus, tapirStubServer, - playJson, - sttpPlayJson, sttpCirce, sttpCore, awsSecretsManagerSdk, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index c7008f9e2..13372cbed 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -32,7 +32,12 @@ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ +import za.co.absa.atum.model.dto.MeasureResultDTO._ +import za.co.absa.atum.server.model.CirceJsonImplicits._ +import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get +import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ +import doobie.postgres.circe.jsonb.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task] @@ -48,7 +53,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) values.measurements.map(x => x.asJson.noSpaces) } - sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( + val sqlDebug = sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningNormalized @@ -64,6 +69,8 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ${values.measuredByAtumAgent}, ${values.author} ) ${Fragment.const(alias)};""" + println(sqlDebug) + sqlDebug } } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala index 705e6c319..f2eb606f0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/CheckpointFromDB.scala @@ -18,6 +18,7 @@ package za.co.absa.atum.server.model import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningDTO} import io.circe.{DecodingFailure, Json} +import za.co.absa.atum.server.model.CirceJsonImplicits._ import java.time.ZonedDateTime import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala deleted file mode 100644 index a4e32fe0f..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.model - -import play.api.libs.functional.syntax.toFunctionalBuilderOps -import play.api.libs.json._ -import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} -import za.co.absa.atum.model.dto._ -import za.co.absa.atum.server.model.SuccessResponse.{MultiSuccessResponse, SingleSuccessResponse} - -object PlayJsonImplicits { - - implicit val optionStringReads: Reads[Option[String]] = new Reads[Option[String]] { - def reads(json: JsValue): JsResult[Option[String]] = json match { - case JsNull => JsSuccess(None) - case JsString(s) => JsSuccess(Some(s)) - case _ => JsError("Expected JsString or JsNull") - } - } - - implicit val optionStringWrites: Writes[Option[String]] = new Writes[Option[String]] { - def writes(opt: Option[String]): JsValue = opt match { - case Some(s) => JsString(s) - case None => JsNull - } - } - - implicit val resultValueTypeReads: Reads[ResultValueType] = new Reads[ResultValueType] { - override def reads(json: JsValue): JsResult[ResultValueType] = json match { - case JsString("String") => JsSuccess(ResultValueType.String) - case JsString("Long") => JsSuccess(ResultValueType.Long) - case JsString("BigDecimal") => JsSuccess(ResultValueType.BigDecimal) - case JsString("Double") => JsSuccess(ResultValueType.Double) - case _ => JsError("Invalid ResultValueType") - } - } - - implicit val resultValueTypeWrites: Writes[ResultValueType] = new Writes[ResultValueType] { - def writes(resultValueType: ResultValueType): JsValue = resultValueType match { - case ResultValueType.String => Json.toJson("String") - case ResultValueType.Long => Json.toJson("Long") - case ResultValueType.BigDecimal => Json.toJson("BigDecimal") - case ResultValueType.Double => Json.toJson("Double") - } - } - - implicit val readsMeasureDTO: Reads[MeasureDTO] = Json.reads[MeasureDTO] - implicit val writesMeasureDTO: Writes[MeasureDTO] = Json.writes[MeasureDTO] - - implicit val readsTypedValue: Reads[MeasureResultDTO.TypedValue] = Json.reads[MeasureResultDTO.TypedValue] - implicit val writesTypedValue: Writes[MeasureResultDTO.TypedValue] = Json.writes[MeasureResultDTO.TypedValue] - - implicit val readsMeasureResultDTO: Reads[MeasureResultDTO] = { - ((__ \ "mainValue").read[MeasureResultDTO.TypedValue] and - (__ \ "supportValues").readNullable[Map[String, TypedValue]].map(_.getOrElse(Map.empty)) - )(MeasureResultDTO.apply _) - } - - implicit val writesMeasureResultDTO: Writes[MeasureResultDTO] = Json.writes[MeasureResultDTO] - - implicit val readsMeasurementDTO: Reads[MeasurementDTO] = Json.reads[MeasurementDTO] - implicit val writesMeasurementDTO: Writes[MeasurementDTO] = Json.writes[MeasurementDTO] - - implicit val readsPartitionDTO: Reads[PartitionDTO] = Json.reads[PartitionDTO] - implicit val writesPartitionDTO: Writes[PartitionDTO] = Json.writes[PartitionDTO] - - implicit val readsCheckpointDTO: Reads[CheckpointDTO] = Json.reads[CheckpointDTO] - implicit val writesCheckpointDTO: Writes[CheckpointDTO] = Json.writes[CheckpointDTO] - - implicit val readsPartitioningSubmitDTO: Reads[PartitioningSubmitDTO] = Json.reads[PartitioningSubmitDTO] - implicit val writesPartitioningSubmitDTO: Writes[PartitioningSubmitDTO] = Json.writes[PartitioningSubmitDTO] - - implicit val readsStringMap: Reads[Map[String, Option[String]]] = Reads.mapReads[Option[String]] - implicit val writesStringMap: OWrites[MapWrites.Map[String, Option[String]]] = - Writes.genericMapWrites[Option[String], MapWrites.Map] - - implicit val readsAdditionalDataSubmitDTO: Reads[AdditionalDataSubmitDTO] = Json.reads[AdditionalDataSubmitDTO] - implicit val writesAdditionalDataSubmitDTO: Writes[AdditionalDataSubmitDTO] = Json.writes[AdditionalDataSubmitDTO] - - implicit val readsAtumContextDTO: Reads[AtumContextDTO] = Json.reads[AtumContextDTO] - implicit val writesAtumContextDTO: Writes[AtumContextDTO] = Json.writes[AtumContextDTO] - - implicit def formatSingleSuccessResponse[T: Format]: Format[SingleSuccessResponse[T]] = Json.format[SingleSuccessResponse[T]] - implicit def formatMultiSuccessResponse[T: Format]: Format[MultiSuccessResponse[T]] = Json.format[MultiSuccessResponse[T]] - - implicit val readsCheckpointQueryDTO: Reads[CheckpointQueryDTO] = Json.reads[CheckpointQueryDTO] - implicit val writesCheckpointQueryDTO: Writes[CheckpointQueryDTO] = Json.writes[CheckpointQueryDTO] -} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index f76e044e8..054e86af9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -16,8 +16,8 @@ package za.co.absa.atum.server.api.database.runs.functions +import za.co.absa.atum.model.dto.{CheckpointDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitionDTO} import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} -import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider @@ -41,7 +41,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { author = "author", partitioning = Seq(PartitionDTO("key1", "val1"), PartitionDTO("key2", "val2")), processStartTime = ZonedDateTime.now(), - processEndTime = None, + processEndTime = Option(ZonedDateTime.now()), measurements = Set(MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.Long)))) ) From 935cde04af3457662c9bccd973e20689d428018d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 24 Jun 2024 14:18:23 +0200 Subject: [PATCH 08/11] removing play Json dependencies, and PlayJsonImplicits class --- .../main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala index 7bf65ca44..61b26cac6 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/MeasureResultDTO.scala @@ -17,7 +17,6 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} -import io.circe.generic.auto._ case class MeasureResultDTO( mainValue: MeasureResultDTO.TypedValue, From 3c2ca0041c59add7100638cd0805fa6a8641c5a8 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 25 Jun 2024 16:19:45 +0200 Subject: [PATCH 09/11] fixing bugs and using circe instead of Json4s dependency --- .../agent/dispatcher/HttpDispatcher.scala | 1 + .../atum/model/utils/SerializationUtils.scala | 82 +++++++++++-------- .../utils/SerializationUtilsUnitTests.scala | 5 +- .../server/api/database/DoobieImplicits.scala | 9 +- .../CreatePartitioningIfNotExists.scala | 4 +- .../runs/functions/WriteCheckpoint.scala | 11 ++- .../WriteCheckpointIntegrationTests.scala | 5 +- 7 files changed, 65 insertions(+), 52 deletions(-) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala index cc7ea6623..c773229e3 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala @@ -23,6 +23,7 @@ import sttp.model.Uri import za.co.absa.atum.agent.exception.AtumAgentException.HttpException import za.co.absa.atum.model.dto.{AdditionalDataSubmitDTO, AtumContextDTO, CheckpointDTO, PartitioningSubmitDTO} import za.co.absa.atum.model.utils.SerializationUtils +import io.circe.generic.auto._ class HttpDispatcher(config: Config) extends Dispatcher(config: Config) with Logging { import HttpDispatcher._ diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala index fcd95568d..67d4ea85c 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/SerializationUtils.scala @@ -16,33 +16,36 @@ package za.co.absa.atum.model.utils -import org.json4s.JsonAST.JString -import org.json4s.jackson.Serialization -import org.json4s.jackson.Serialization.{write, writePretty} -import org.json4s.{CustomSerializer, Formats, JNull, NoTypeHints, ext} -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType -import za.co.absa.atum.model.dto.MeasureResultDTO.ResultValueType._ +import io.circe.{Decoder, Encoder} +import io.circe.syntax._ +import io.circe.parser._ +import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import java.util.UUID object SerializationUtils { - implicit private val formatsJson: Formats = - Serialization.formats(NoTypeHints).withBigDecimal + - ext.UUIDSerializer + - ZonedDateTimeSerializer + - ResultValueTypeSerializer - // TODO "yyyy-MM-dd'T'hh:mm:ss.SSS'Z'" OR TODO "yyyy-MM-dd HH:mm:ss.SSSSSSX" val timestampFormat: DateTimeFormatter = DateTimeFormatter.ISO_ZONED_DATE_TIME + implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap[ZonedDateTime](_.format(timestampFormat)) + implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => + Right(ZonedDateTime.parse(str, timestampFormat)) + } + + implicit val encodeUUID: Encoder[UUID] = Encoder.encodeString.contramap[UUID](_.toString) + implicit val decodeUUID: Decoder[UUID] = Decoder.decodeString.emap { str => + Right(UUID.fromString(str)) + } + /** * The method returns arbitrary object as a Json string. * * @return A string representing the object in Json format */ - def asJson[T <: AnyRef](obj: T): String = { - write[T](obj) + def asJson[T: Encoder](obj: T): String = { + obj.asJson.noSpaces } /** @@ -50,8 +53,8 @@ object SerializationUtils { * * @return A string representing the object in Json format */ - def asJsonPretty[T <: AnyRef](obj: T): String = { - writePretty[T](obj) + def asJsonPretty[T: Encoder](obj: T): String = { + obj.asJson.spaces2 } /** @@ -59,27 +62,34 @@ object SerializationUtils { * * @return An object deserialized from the Json string */ - def fromJson[T <: AnyRef](jsonStr: String)(implicit m: Manifest[T]): T = { - Serialization.read[T](jsonStr) + def fromJson[T: Decoder](jsonStr: String): T = { + decode[T](jsonStr) match { + case Right(value) => value + case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + } } - private case object ResultValueTypeSerializer extends CustomSerializer[ResultValueType](format => ( - { - case JString(resultValType) => resultValType match { - case "String" => String - case "Long" => Long - case "BigDecimal" => BigDecimal - case "Double" => Double - } - case JNull => null - }, - { - case resultValType: ResultValueType => resultValType match { - case String => JString("String") - case Long => JString("Long") - case BigDecimal => JString("BigDecimal") - case Double => JString("Double") - } - })) + sealed trait ResultValueType + object ResultValueType { + case object String extends ResultValueType + case object Long extends ResultValueType + case object BigDecimal extends ResultValueType + case object Double extends ResultValueType + + implicit val encodeResultValueType: Encoder[ResultValueType] = Encoder.encodeString.contramap { + case ResultValueType.String => "String" + case ResultValueType.Long => "Long" + case ResultValueType.BigDecimal => "BigDecimal" + case ResultValueType.Double => "Double" + } + + implicit val decodeResultValueType: Decoder[ResultValueType] = Decoder.decodeString.emap { + case "String" => Right(ResultValueType.String) + case "Long" => Right(ResultValueType.Long) + case "BigDecimal" => Right(ResultValueType.BigDecimal) + case "Double" => Right(ResultValueType.Double) + case other => Left(s"Cannot decode $other as ResultValueType") + } + } } diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 91ebc9ab9..f43fa083f 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -16,6 +16,7 @@ package za.co.absa.atum.model.utils +import io.circe.generic.auto._ import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.dto.MeasureResultDTO.{ResultValueType, TypedValue} import za.co.absa.atum.model.dto._ @@ -41,7 +42,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { val expectedAdditionalDataJson = """ |{"partitioning":[{"key":"key","value":"val"}], - |"additionalData":{"key1":"val1","key2":"val2"}, + |"additionalData":{"key1":"val1","key2":"val2","key3":null}, |"author":"testAuthor"} |""".linearize val actualAdditionalDataJson = SerializationUtils.asJson(additionalDataDTO) @@ -343,7 +344,7 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { authorIfNew = "authorTest" ) - val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"authorIfNew":"authorTest"}""" + val expectedPartitioningDTOJson = """{"partitioning":[{"key":"key","value":"val"}],"parentPartitioning":null,"authorIfNew":"authorTest"}""" val actualPartitioningDTOJson = SerializationUtils.asJson(partitioningDTO) assert(actualPartitioningDTOJson == expectedPartitioningDTOJson) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index a1131e65b..5a002d2df 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -20,6 +20,7 @@ import cats.Show import cats.data.NonEmptyList import doobie.{Get, Put} import doobie.postgres.implicits._ +import io.circe.Json import org.postgresql.jdbc.PgArray import org.postgresql.util.PGobject @@ -42,7 +43,7 @@ object DoobieImplicits { object Json { - implicit val jsonArrayPutUsingString: Put[List[String]] = { + implicit val jsonArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("json[]") @@ -50,7 +51,7 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - o.setValue(a.mkString("{", ",", "}")) + o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) o } } @@ -100,7 +101,7 @@ object DoobieImplicits { object Jsonb { - implicit val jsonbArrayPutUsingString: Put[List[String]] = { + implicit val jsonbArrayPut: Put[List[Json]] = { Put.Advanced .other[PGobject]( NonEmptyList.of("jsonb[]") @@ -108,7 +109,7 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("jsonb[]") - o.setValue(a.mkString("{", ",", "}")) + o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) o } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 3da9fd091..708add14e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -50,8 +50,8 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningJsonString }, - ${values.authorIfNew}, - ${ + ${values.authorIfNew}, + ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString parentPartitioningJsonString } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 13372cbed..58f2f6b3a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.Fragment -import doobie.implicits._ +import doobie.implicits.toSqlInterpolator import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointDTO import za.co.absa.atum.server.model.PartitioningForDB @@ -31,13 +31,12 @@ import zio._ import zio.interop.catz._ import io.circe.syntax._ import io.circe.generic.auto._ - import za.co.absa.atum.model.dto.MeasureResultDTO._ import za.co.absa.atum.server.model.CirceJsonImplicits._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.circe.jsonb.implicits.jsonbGet import doobie.postgres.implicits._ -import doobie.postgres.circe.jsonb.implicits._ +import io.circe.Json class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieSingleResultFunctionWithStatus[CheckpointDTO, Unit, Task] @@ -50,7 +49,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) // List[String] containing json data has to be properly escaped // It would be safer to use Json data type and derive Put instance val measurementsNormalized = { - values.measurements.map(x => x.asJson.noSpaces) + values.measurements.toList.map(_.asJson) } val sqlDebug = sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( @@ -63,8 +62,8 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ${values.processStartTime}, ${values.processEndTime}, ${ - import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPutUsingString - measurementsNormalized.toList + import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbArrayPut + measurementsNormalized }, ${values.measuredByAtumAgent}, ${values.author} diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index 054e86af9..b9878b104 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -35,11 +35,12 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { suite("WriteCheckpointSuite")( test("Returns expected Left with DataNotFoundException as related partitioning is not in the database") { + val checkpointDTO = CheckpointDTO( id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key1", "val1"), PartitionDTO("key2", "val2")), + partitioning = Seq(PartitionDTO("key2", "value2")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = @@ -53,7 +54,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { ).provide( WriteCheckpoint.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithoutRollback ) } From 0f2b220c56956cb15ab5757dedd0b478f07143f7 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jun 2024 15:21:22 +0200 Subject: [PATCH 10/11] revised arrayPutUsing string implicit conversion --- .../server/api/database/DoobieImplicits.scala | 18 ++++++++++++++++-- .../runs/functions/WriteCheckpoint.scala | 4 +--- .../WriteCheckpointIntegrationTests.scala | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala index 5a002d2df..3a1969ed2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/DoobieImplicits.scala @@ -51,7 +51,14 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("json[]") - o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) + val arrayElements = a.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s"\"$escapedJsonString\"" + } + // Join all elements into a single string wrapped in curly braces + o.setValue(arrayElements.mkString("{", ",", "}")) o } } @@ -109,7 +116,14 @@ object DoobieImplicits { .tcontramap { a => val o = new PGobject o.setType("jsonb[]") - o.setValue(a.map(x => s"\"${x.noSpaces.replaceAll("\"", """\"""")}\"").mkString("{", ",", "}")) + val arrayElements = a.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s"\"$escapedJsonString\"" + } + // Join all elements into a single string wrapped in curly braces + o.setValue(arrayElements.mkString("{", ",", "}")) o } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index 58f2f6b3a..a6587f296 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -52,7 +52,7 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) values.measurements.toList.map(_.asJson) } - val sqlDebug = sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( + sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString partitioningNormalized @@ -68,8 +68,6 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) ${values.measuredByAtumAgent}, ${values.author} ) ${Fragment.const(alias)};""" - println(sqlDebug) - sqlDebug } } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala index b9878b104..6f4c4c2dc 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpointIntegrationTests.scala @@ -40,7 +40,7 @@ object WriteCheckpointIntegrationTests extends ConfigProviderTest { id = UUID.randomUUID(), name = "name", author = "author", - partitioning = Seq(PartitionDTO("key2", "value2")), + partitioning = Seq(PartitionDTO("key4", "value4")), processStartTime = ZonedDateTime.now(), processEndTime = Option(ZonedDateTime.now()), measurements = From ad5999901e32f4fdfd8273843d05e7f83c373ce8 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 27 Jun 2024 10:35:56 +0200 Subject: [PATCH 11/11] removed json4s and replaced its usage --- .../model/utils/ZonedDateTimeSerializer.scala | 24 +++++++++-------- project/Dependencies.scala | 26 ------------------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala b/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala index 6fe76dd61..97f300321 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/ZonedDateTimeSerializer.scala @@ -16,17 +16,19 @@ package za.co.absa.atum.model.utils -import org.json4s.JsonAST.JString -import org.json4s.{CustomSerializer, JNull} - +import io.circe.{Decoder, Encoder} import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter + +object ZonedDateTimeSerializer { + implicit val encodeZonedDateTime: Encoder[ZonedDateTime] = Encoder.encodeString.contramap( + _.format(DateTimeFormatter.ISO_ZONED_DATE_TIME)) -case object ZonedDateTimeSerializer extends CustomSerializer[ZonedDateTime](_ => ( - { - case JString(s) => ZonedDateTime.parse(s, SerializationUtils.timestampFormat) - case JNull => null - }, - { - case d: ZonedDateTime => JString(SerializationUtils.timestampFormat.format(d)) + implicit val decodeZonedDateTime: Decoder[ZonedDateTime] = Decoder.decodeString.emap { str => + try { + Right(ZonedDateTime.parse(str, DateTimeFormatter.ISO_ZONED_DATE_TIME)) + } catch { + case e: Throwable => Left(e.getMessage) + } } -)) +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0f9a58aef..41e855530 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -46,22 +46,6 @@ object Dependencies { val fadb = "0.3.0" - val json4s_spark2 = "3.5.3" - val json4s_spark3 = "3.7.0-M11" - def json4s(scalaVersion: Version): String = { - // TODO done this impractical way until https://github.com/AbsaOSS/commons/issues/134 - val maj2 = Component("2") - val min11 = Component("11") - val min12 = Component("12") - val min13 = Component("13") - scalaVersion.components match { - case Seq(`maj2`, `min11`, _) => json4s_spark2 - case Seq(`maj2`, `min12`, _) => json4s_spark3 - case Seq(`maj2`, `min13`, _) => json4s_spark3 - case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12, and 2.13 are currently supported.") - } - } - val logback = "1.2.3" val zio = "2.0.19" @@ -106,15 +90,9 @@ object Dependencies { } private def jsonSerdeDependencies(scalaVersion: Version): Seq[ModuleID] = { - val json4sVersion = Versions.json4s(scalaVersion) lazy val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jacksonModuleScala - lazy val json4sExt = "org.json4s" %% "json4s-ext" % json4sVersion - lazy val json4sCore = "org.json4s" %% "json4s-core" % json4sVersion - lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion - lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided - // Circe dependencies lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson @@ -122,10 +100,6 @@ object Dependencies { Seq( jacksonModuleScala, - json4sExt, - json4sCore, - json4sJackson, - json4sNative, circeCore, circeParser, circeGeneric,