Merge 0458a75 into c00b7dd

snowplow · Aug 7, 2023 · 090476f · 090476f
2 parents c00b7dd + 0458a75
commit 090476f
Show file tree

Hide file tree

Showing 11 changed files with 47 additions and 8 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,12 @@
+Version 5.7.1 (2023-08-07)
+--------------------------
+Lower sensitivity of cats-effect responsiveness warning (#1309)
+Reduce log level for test suite (#1307)
+Exclude zookeeper transitive dependency from loaders (#1305)
+Batch Transformer: make it possible to skip schemas with all transformations (#1300)
+Bump Snowplow Events Manifest to 0.4.0 (#1303)
+transformer-kafka: add semi-automatic test scenarios using cloud resources (#1302)
+
 Version 5.7.0 (2023-07-26)
 --------------------------
 Add Snowplow Community License (#1297)

diff --git a/README.md b/README.md
@@ -65,7 +65,7 @@ Licensed under the [Snowplow Community License](https://docs.snowplow.io/communi
 [build-image]: https://github.com/snowplow/snowplow-rdb-loader/workflows/CI/badge.svg
 [build]: https://github.com/snowplow/snowplow-rdb-loader/actions/workflows/ci.yml
 
-[release-image]: https://img.shields.io/badge/release-5.7.0-blue.svg?style=flat
+[release-image]: https://img.shields.io/badge/release-5.7.1-blue.svg?style=flat
 [releases]: https://github.com/snowplow/snowplow-rdb-loader/releases
 
 [license]: https://docs.snowplow.io/docs/contributing/community-license-faq/

diff --git a/modules/common-transformer-stream/src/test/resources/simplelogger.properties b/modules/common-transformer-stream/src/test/resources/simplelogger.properties
@@ -1,3 +1,5 @@
 org.slf4j.simpleLogger.showThreadName=false
 org.slf4j.simpleLogger.showDateTime=true
-org.slf4j.simpleLogger.showShortLogName=true
+org.slf4j.simpleLogger.showShortLogName=true
+org.slf4j.simpleLogger.defaultLogLevel=warn
+org.slf4j.simpleLogger.log.com.snowplowanalytics=info
diff --git a/...abricks-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/databricks/Main.scala b/...abricks-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/databricks/Main.scala
@@ -9,12 +9,16 @@ package com.snowplowanalytics.snowplow.loader.databricks
 
 import cats.effect.{ExitCode, IO, IOApp}
 
-import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
+import scala.concurrent.duration.DurationInt
 
+import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
 import com.snowplowanalytics.snowplow.rdbloader.Runner
 
 object Main extends IOApp {
 
+  override def runtimeConfig =
+    super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds)
+
   def run(args: List[String]): IO[ExitCode] =
     Runner.run[IO, Unit](
       args,

diff --git a/.../redshift-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/redshift/Main.scala b/.../redshift-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/redshift/Main.scala
@@ -10,12 +10,16 @@ package com.snowplowanalytics.snowplow.loader.redshift
 import cats.effect.{ExitCode, IO, IOApp}
 import cats.effect.implicits._
 
-import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
+import scala.concurrent.duration.DurationInt
 
+import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
 import com.snowplowanalytics.snowplow.rdbloader.Runner
 
 object Main extends IOApp {
 
+  override def runtimeConfig =
+    super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds)
+
   def run(args: List[String]): IO[ExitCode] =
     Runner.run[IO, Unit](
       args,

diff --git a/...nowflake-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/snowflake/Main.scala b/...nowflake-loader/src/main/scala/com/snowplowanalytics/snowplow/loader/snowflake/Main.scala
@@ -10,10 +10,15 @@ package com.snowplowanalytics.snowplow.loader.snowflake
 import cats.effect.{ExitCode, IO, IOApp}
 
 import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
-
 import com.snowplowanalytics.snowplow.rdbloader.Runner
 
+import scala.concurrent.duration.DurationInt
+
 object Main extends IOApp {
+
+  override def runtimeConfig =
+    super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds)
+
   def run(args: List[String]): IO[ExitCode] =
     Runner.run[IO, Snowflake.InitQueryResult](
       args,

diff --git a/modules/transformer-batch/src/test/resources/simplelogger.properties b/modules/transformer-batch/src/test/resources/simplelogger.properties
@@ -0,0 +1,2 @@
+org.slf4j.simpleLogger.defaultLogLevel=warn
+org.slf4j.simpleLogger.log.com.snowplowanalytics=info
diff --git a/...c/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/Main.scala b/...c/main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kafka/Main.scala
@@ -16,8 +16,13 @@ import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.kafka.generat
 import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
 import org.apache.hadoop.conf.Configuration
 
+import scala.concurrent.duration.DurationInt
+
 object Main extends IOApp {
 
+  override def runtimeConfig =
+    super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds)
+
   def run(args: List[String]): IO[ExitCode] =
     Run.run[IO, KafkaCheckpointer[IO]](
       args,

diff --git a/...main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/Main.scala b/...main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/kinesis/Main.scala
@@ -14,12 +14,16 @@ import com.snowplowanalytics.snowplow.rdbloader.aws._
 import com.snowplowanalytics.snowplow.rdbloader.common.cloud.{BlobStorage, Queue}
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.{Config, Run}
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.kinesis.generated.BuildInfo
-
 import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
 
+import scala.concurrent.duration.DurationInt
+
 object Main extends IOApp {
   final val QueueMessageGroupId = "shredding"
 
+  override def runtimeConfig =
+    super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds)
+
   def run(args: List[String]): IO[ExitCode] =
     Run.run[IO, KinesisCheckpointer[IO]](
       args,

diff --git a/.../main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/Main.scala b/.../main/scala/com/snowplowanalytics/snowplow/rdbloader/transformer/stream/pubsub/Main.scala
@@ -23,11 +23,15 @@ import com.snowplowanalytics.snowplow.rdbloader.gcp.{GCS, Pubsub}
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Config
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.pubsub.generated.BuildInfo
 import com.snowplowanalytics.snowplow.rdbloader.transformer.stream.common.Run
-
 import com.snowplowanalytics.snowplow.scalatracker.emitters.http4s.ceTracking
 
+import scala.concurrent.duration.DurationInt
+
 object Main extends IOApp {
 
+  override def runtimeConfig =
+    super.runtimeConfig.copy(cpuStarvationCheckInterval = 10.seconds)
+
   implicit private def logger[F[_]: Sync]: Logger[F] = Slf4jLogger.getLogger[F]
 
   def run(args: List[String]): IO[ExitCode] =

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -142,6 +142,7 @@ object Dependencies {
   val parquet4s         = "com.github.mjakubowski84"         %% "parquet4s-fs2"            % V.parquet4s
   val hadoopCommon      = ("org.apache.hadoop"               % "hadoop-common"             % V.hadoopClient)
     .exclude("com.jcraft", "jsch")
+    .exclude("org.apache.zookeeper", "zookeeper")
   val hadoop            = "org.apache.hadoop"                % "hadoop-client"             % V.hadoopClient
   val parquetHadoop     = "org.apache.parquet"               % "parquet-hadoop"            % V.parquetHadoop
   val hadoopAws         = ("org.apache.hadoop"               % "hadoop-aws"                % V.hadoopClient % Runtime)
@@ -358,4 +359,3 @@ object Dependencies {
       ExclusionRule(organization = "org.apache.hadoop.thirdparty", name = "hadoop-shaded-protobuf_3_7"),
     )
 }
-