diff --git a/CHANGELOG b/CHANGELOG index dc12dc6b4..7b9ea673a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +Version 5.7.1 (2023-08-07) +-------------------------- +Lower sensitivity of cats-effect responsiveness warning (#1309) +Reduce log level for test suite (#1307) +Exclude zookeeper transitive dependency from loaders (#1305) +Batch Transformer: make it possible to skip schemas with all transformations (#1300) +Bump Snowplow Events Manifest to 0.4.0 (#1303) +transformer-kafka: add semi-automatic test scenarios using cloud resources (#1302) + Version 5.7.0 (2023-07-26) -------------------------- Add Snowplow Community License (#1297) diff --git a/README.md b/README.md index a730050b7..e0c136ec0 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Licensed under the [Snowplow Community License](https://docs.snowplow.io/communi [build-image]: https://github.com/snowplow/snowplow-rdb-loader/workflows/CI/badge.svg [build]: https://github.com/snowplow/snowplow-rdb-loader/actions/workflows/ci.yml -[release-image]: https://img.shields.io/badge/release-5.7.0-blue.svg?style=flat +[release-image]: https://img.shields.io/badge/release-5.7.1-blue.svg?style=flat [releases]: https://github.com/snowplow/snowplow-rdb-loader/releases [license]: https://docs.snowplow.io/docs/contributing/community-license-faq/ diff --git a/modules/common-transformer-stream/src/test/resources/simplelogger.properties b/modules/common-transformer-stream/src/test/resources/simplelogger.properties index 306a747b4..d9d02cecf 100644 --- a/modules/common-transformer-stream/src/test/resources/simplelogger.properties +++ b/modules/common-transformer-stream/src/test/resources/simplelogger.properties @@ -1,3 +1,5 @@ org.slf4j.simpleLogger.showThreadName=false org.slf4j.simpleLogger.showDateTime=true -org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file +org.slf4j.simpleLogger.showShortLogName=true +org.slf4j.simpleLogger.defaultLogLevel=warn +org.slf4j.simpleLogger.log.com.snowplowanalytics=info diff --git a/modules/databricks-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/databricks/Main.scala b/modules/databricks-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/databricks/Main.scala index ad141c476..778ec3fed 100644 --- a/modules/databricks-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/databricks/Main.scala +++ b/modules/databricks-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/databricks/Main.scala @@ -9,12 +9,16 @@ package com.snowplowanalytics.snowplow.loader.databricks import cats.effect.{ExitCode, IO, IOApp} -import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking +import scala.concurrent.duration.DurationInt +import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking import com.snowplowanalytics.snowplow.rdbloader.Runner object Main extends IOApp { + override def runtimeConfig = + super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds) + def run(args: List[String]): IO[ExitCode] = Runner.run[IO, Unit]( args, diff --git a/modules/redshift-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/redshift/Main.scala b/modules/redshift-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/redshift/Main.scala index c988c2fb4..e004ba57e 100644 --- a/modules/redshift-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/redshift/Main.scala +++ b/modules/redshift-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/redshift/Main.scala @@ -10,12 +10,16 @@ package com.snowplowanalytics.snowplow.loader.redshift import cats.effect.{ExitCode, IO, IOApp} import cats.effect.implicits._ -import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking +import scala.concurrent.duration.DurationInt +import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking import com.snowplowanalytics.snowplow.rdbloader.Runner object Main extends IOApp { + override def runtimeConfig = + super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds) + def run(args: List[String]): IO[ExitCode] = Runner.run[IO, Unit]( args, diff --git a/modules/snowflake-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/snowflake/Main.scala b/modules/snowflake-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/snowflake/Main.scala index 9179e3873..cabbaa81c 100644 --- a/modules/snowflake-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/snowflake/Main.scala +++ b/modules/snowflake-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/snowflake/Main.scala @@ -10,10 +10,15 @@ package com.snowplowanalytics.snowplow.loader.snowflake import cats.effect.{ExitCode, IO, IOApp} import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking - import com.snowplowanalytics.snowplow.rdbloader.Runner +import scala.concurrent.duration.DurationInt + object Main extends IOApp { + + override def runtimeConfig = + super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds) + def run(args: List[String]): IO[ExitCode] = Runner.run[IO, Snowflake.InitQueryResult]( args, diff --git a/modules/transformer-batch/src/test/resources/simplelogger.properties b/modules/transformer-batch/src/test/resources/simplelogger.properties new file mode 100644 index 000000000..8583d8cdb --- /dev/null +++ b/modules/transformer-batch/src/test/resources/simplelogger.properties @@ -0,0 +1,2 @@ +org.slf4j.simpleLogger.defaultLogLevel=warn +org.slf4j.simpleLogger.log.com.snowplowanalytics=info diff --git a/modules/transformer-kafka/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/Main.scala b/modules/transformer-kafka/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/Main.scala index bd86ca2b1..c30dbea57 100644 --- a/modules/transformer-kafka/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/Main.scala +++ b/modules/transformer-kafka/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/Main.scala @@ -16,8 +16,13 @@ import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.kafka.generat import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking import org.apache.hadoop.conf.Configuration +import scala.concurrent.duration.DurationInt + object Main extends IOApp { + override def runtimeConfig = + super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds) + def run(args: List[String]): IO[ExitCode] = Run.run[IO, KafkaCheckpointer[IO]]( args, diff --git a/modules/transformer-kinesis/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/Main.scala b/modules/transformer-kinesis/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/Main.scala index bb7960bc4..94f4bf246 100644 --- a/modules/transformer-kinesis/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/Main.scala +++ b/modules/transformer-kinesis/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/Main.scala @@ -14,12 +14,16 @@ import com.snowplowanalytics.snowplow.rdbloader.aws._ import com.snowplowanalytics.snowplow.rdbloader.common.cloud.{BlobStorage, Queue} import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.{Config, Run} import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.kinesis.generated.BuildInfo - import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking +import scala.concurrent.duration.DurationInt + object Main extends IOApp { final val QueueMessageGroupId = "shredding" + override def runtimeConfig = + super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds) + def run(args: List[String]): IO[ExitCode] = Run.run[IO, KinesisCheckpointer[IO]]( args, diff --git a/modules/transformer-pubsub/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/Main.scala b/modules/transformer-pubsub/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/Main.scala index 4ec93b52d..5273431e6 100644 --- a/modules/transformer-pubsub/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/Main.scala +++ b/modules/transformer-pubsub/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/Main.scala @@ -23,11 +23,15 @@ import com.snowplowanalytics.snowplow.rdbloader.gcp.{GCS, Pubsub} import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.pubsub.generated.BuildInfo import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Run - import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking +import scala.concurrent.duration.DurationInt + object Main extends IOApp { + override def runtimeConfig = + super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds) + implicit private def logger[F[_]: Sync]: Logger[F] = Slf4jLogger.getLogger[F] def run(args: List[String]): IO[ExitCode] = diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2a2d5a7ff..0630b4c8e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -142,6 +142,7 @@ object Dependencies { val parquet4s = "com.github.mjakubowski84" %% "parquet4s-fs2" % V.parquet4s val hadoopCommon = ("org.apache.hadoop" % "hadoop-common" % V.hadoopClient) .exclude("com.jcraft", "jsch") + .exclude("org.apache.zookeeper", "zookeeper") val hadoop = "org.apache.hadoop" % "hadoop-client" % V.hadoopClient val parquetHadoop = "org.apache.parquet" % "parquet-hadoop" % V.parquetHadoop val hadoopAws = ("org.apache.hadoop" % "hadoop-aws" % V.hadoopClient % Runtime) @@ -358,4 +359,3 @@ object Dependencies { ExclusionRule(organization = "org.apache.hadoop.thirdparty", name = "hadoop-shaded-protobuf_3_7"), ) } -