Manually fetch container image before running StatsD spec

Previously, we relied on org.testcontainers' method of fetching container image for StatsD. This was counted towards test timeout and rarely actually triggered the tests. Now, the image is fetched as part of beforeAll step and handled as part of setup time that's not counted towards the timeout.
snowplow-incubator · Mar 27, 2024 · ac7b58f · ac7b58f
1 parent edc75ea
commit ac7b58f
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 5 deletions.
diff --git a/modules/it/src/test/scala/com/snowplowanalytics/snowplow/runtime/MetricsSpec.scala b/modules/it/src/test/scala/com/snowplowanalytics/snowplow/runtime/MetricsSpec.scala
@@ -12,12 +12,35 @@ import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import cats.effect.{IO, Ref, Resource}
 import cats.effect.testing.specs2.CatsResource
 import org.specs2.mutable.SpecificationLike
+import org.specs2.specification.BeforeAll
 import org.testcontainers.containers.GenericContainer
+import com.github.dockerjava.core.DockerClientBuilder
+import com.github.dockerjava.api.command.PullImageResultCallback
 
 import retry.syntax.all._
 import retry.RetryPolicies
+import com.github.dockerjava.api.model.PullResponseItem
 
-class MetricsSpec extends CatsResource[IO, (GenericContainer[_], StatsdAPI[IO])] with SpecificationLike {
+class MetricsSpec extends CatsResource[IO, (GenericContainer[_], StatsdAPI[IO])] with SpecificationLike with BeforeAll {
+
+  override def beforeAll(): Unit = {
+    // blocking the main thread to fetch before we start creating a container
+    DockerClientBuilder
+      .getInstance()
+      .build()
+      .pullImageCmd(Statsd.image)
+      .withTag(Statsd.tag)
+      .withPlatform("linux/amd64")
+      .exec(new PullImageResultCallback() {
+        override def onNext(item: PullResponseItem) = {
+          println(s"StatsD image: ${item.getStatus()}")
+          super.onNext(item)
+        }
+      })
+      .awaitCompletion()
+      .onComplete()
+    super.beforeAll()
+  }
 
   override val resource: Resource[IO, (GenericContainer[_], StatsdAPI[IO])] =
     for {

diff --git a/modules/it/src/test/scala/com/snowplowanalytics/snowplow/runtime/Statsd.scala b/modules/it/src/test/scala/com/snowplowanalytics/snowplow/runtime/Statsd.scala
@@ -17,12 +17,13 @@ import com.github.dockerjava.api.model.ExposedPort
 import com.github.dockerjava.api.model.Ports
 
 object Statsd {
-
+  val image = "dblworks/statsd" // the official statsd/statsd size is monstrous
+  val tag   = "v0.10.1"
   def resource(
     loggerName: String
   ): Resource[IO, GenericContainer[_]] =
     Resource.make {
-      val statsd: GenericContainer[_] = new GenericContainer("statsd/statsd:v0.10.1")
+      val statsd: GenericContainer[_] = new GenericContainer(s"$image:$tag")
       statsd.addExposedPort(8126)
       statsd.setWaitStrategy(Wait.forLogMessage("""^(.*)server is up(.+)$""", 1))
       statsd.withCreateContainerCmdModifier { cmd =>
@@ -33,7 +34,7 @@ object Statsd {
         cmd.getHostConfig().withPortBindings(ports)
         ()
       }
-      IO(start(statsd, loggerName))
+      IO.blocking(start(statsd, loggerName))
     }(ls => IO.blocking(ls.stop()))
 
   private def start(statsd: GenericContainer[_], loggerName: String): GenericContainer[_] = {

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -49,6 +49,7 @@ object Dependencies {
     val catsEffectSpecs2 = "1.5.0"
     val localstack       = "1.19.0"
     val eventGen         = "0.7.0"
+    val dockerJava       = "3.3.6"
   }
 
   val catsEffectKernel  = "org.typelevel"          %% "cats-effect-kernel"      % V.catsEffect
@@ -104,6 +105,7 @@ object Dependencies {
   val catsEffectTestkit = "org.typelevel"         %% "cats-effect-testkit"           % V.catsEffect       % Test
   val catsEffectSpecs2  = "org.typelevel"         %% "cats-effect-testing-specs2"    % V.catsEffectSpecs2 % Test
   val eventGen          = "com.snowplowanalytics" %% "snowplow-event-generator-core" % V.eventGen         % Test
+  val dockerJava        = "com.github.docker-java" % "docker-java"                   % V.dockerJava
 
   val streamsDependencies = Seq(
     cats,
@@ -140,7 +142,8 @@ object Dependencies {
     catsEffectSpecs2It  % Test,
     catsRetry           % Test,
     localstackIt        % Test,
-    slf4jIt             % Test
+    slf4jIt             % Test,
+    dockerJava          % Test
   )
 
   val kafkaDependencies = Seq(