Release cleanup (#19)

* Push SNAPSHOT docker images * Release and licensing updates * Grafana dashboard updates * License masthead update * Allow docker repo and username overrides * README updates * Test code license masthead * Increase reference.conf defaults for polling interval and lookup table size * README polish
seglo · May 9, 2019 · f5675a4 · f5675a4
1 parent 7e2cd47
commit f5675a4
Show file tree

Hide file tree

Showing 34 changed files with 272 additions and 101 deletions.
diff --git a/README.md b/README.md
diff --git a/build.sbt b/build.sbt
@@ -1,9 +1,10 @@
-import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd}
 import Dependencies._
-import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport.{dockerCommands, dockerUsername}
+import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd}
+import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport._
 import ReleaseTransformations._
 import ReleasePlugin.autoImport._
 import ReleaseKeys._
+
 import scala.sys.process._
 
 lazy val kafkaLagExporter =
@@ -33,7 +34,8 @@ lazy val kafkaLagExporter =
         AlpakkaKafkaTestKit,
         AkkaHttp
       ),
-      dockerUsername := Some("lightbend"),
+      dockerRepository := Option(System.getenv("DOCKER_REPOSITORY")).orElse(None),
+      dockerUsername := Option(System.getenv("DOCKER_USERNAME")).orElse(Some("lightbend")),
       // Based on best practices found in OpenShift Creating images guidelines
       // https://docs.openshift.com/container-platform/3.10/creating_images/guidelines.html
       dockerCommands := Seq(
@@ -46,13 +48,11 @@ lazy val kafkaLagExporter =
         Cmd("USER",           "1001"),
         ExecCmd("CMD",        "/opt/docker/bin/kafka-lag-exporter", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap"),
       ),
-      updateHelmChartVersions := {
+      updateHelmChart := {
         import scala.sys.process._
-        s"./scripts/update_chart.sh ${version.value}" !
+        val repo = dockerAlias.value.withTag(None).toString
+        s"./scripts/update_chart.sh ${version.value} $repo" !
       },
-      //compile in Compile := (compile in Compile).dependsOn(updateHelmChartVersions).value,
-      publishArtifact in (Compile, packageDoc) := false,
-      publishArtifact in (Compile, packageSrc) := false,
       skip in publish := true,
       releaseProcess := Seq[ReleaseStep](
         lintHelmChart,                          // Lint the Helm Chart for errors
@@ -61,13 +61,13 @@ lazy val kafkaLagExporter =
         runClean,
         runTest,
         setReleaseVersion,
-        updateHelmChart,                        // Update the Helm Chart with the release version
-        publishDockerImage,                     // Publish the Docker images used by the chart to dockerhub
+        updateHelmChartRelease,                 // Update the Helm Chart
+        publishDockerImage,                     // Publish the Docker images used by the chart
         commitReleaseVersion,
         tagRelease,
         githubReleaseDraft,                     // Create a GitHub release draft
         setNextVersion,
-        updateHelmChart,                        // Update the Helm Chart with the next snapshot version
+        updateHelmChartRelease,                 // Update the Helm Chart with the next snapshot version
         commitNextVersion,
         pushChanges
       )
@@ -95,13 +95,13 @@ lazy val commonSettings = Seq(
   licenses += ("Apache-2.0", new URL("https://www.apache.org/licenses/LICENSE-2.0.txt")),
   headerLicense := Some(
     HeaderLicense.Custom(
-      """|Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
-         |""".stripMargin
+      s"""|Copyright (C) ${startYear.value.get} Lightbend Inc. <http://www.lightbend.com>
+          |""".stripMargin
     )
   ),
 )
 
-lazy val updateHelmChartVersions = taskKey[Unit]("Update Helm Chart versions")
+lazy val updateHelmChart = taskKey[Unit]("Update Helm Chart")
 
 def exec(cmd: String, errorMessage: String): Unit = {
   val e = cmd.!
@@ -113,9 +113,11 @@ lazy val lintHelmChart = ReleaseStep(action = st => {
   st
 })
 
-lazy val updateHelmChart = ReleaseStep(action = st => {
+lazy val updateHelmChartRelease = ReleaseStep(action = st => {
   val (releaseVersion, _) = st.get(versions).getOrElse(sys.error("No versions are set! Was this release part executed before inquireVersions?"))
-  exec(s"./scripts/update_chart.sh $releaseVersion", "Error while updating Helm Chart versions")
+  val extracted = Project.extract(st)
+  val repo = extracted.get(dockerAlias in thisProjectRef).withTag(None)
+  exec(s"./scripts/update_chart.sh $releaseVersion $repo", "Error while updating Helm Chart")
   st
 })
 

diff --git a/charts/kafka-lag-exporter/Chart.yaml b/charts/kafka-lag-exporter/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
-appVersion: "0.4.0"
+appVersion: "0.4.0-SNAPSHOT"
 description: Kafka Lag Exporter
 name: kafka-lag-exporter
 version: 0.4.0

diff --git a/docs/consumer_lag.png b/docs/consumer_lag.png
diff --git a/docs/extrapolation.png b/docs/extrapolation.png
diff --git a/docs/interpolation.png b/docs/interpolation.png
diff --git a/grafana/consumer_group_max_time_lag.png b/grafana/consumer_group_max_time_lag.png
diff --git a/grafana/max_consumer_group_time_lag_over_offset_lag.png b/grafana/max_consumer_group_time_lag_over_offset_lag.png
diff --git a/grafana/max_consumer_group_time_lag_over_summed_offsets.png b/grafana/max_consumer_group_time_lag_over_summed_offsets.png
diff --git a/grafana/offset_lag_time_over_summed_offsets.png b/grafana/offset_lag_time_over_summed_offsets.png
diff --git a/grafana/offset_lag_time_with_offset_lag.png b/grafana/offset_lag_time_with_offset_lag.png
diff --git a/scripts/update_chart.sh b/scripts/update_chart.sh
@@ -3,7 +3,10 @@
 set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
 APP_VERSION=$1
+DOCKER_REPOSITORY=$2
+DOCKER_REPOSITORY_ESCAPED=$(echo $DOCKER_REPOSITORY | sed -e 's/\\/\\\\/g; s/\//\\\//g; s/&/\\\&/g')
 VERSION=`echo $1|sed -r 's/(\S+)-(\S+)/\1/'` # Strip out -SNAPSHOT if it exists
 SNAPSHOT=`echo $1|sed -r 's/(\S+)-(\S+)/\2/'` # Capture SNAPSHOT if it exists
 
@@ -14,6 +17,9 @@ sed -i -r "s/^(\s*)(version\s*:\s.*$)/\1version: ${VERSION}/" $DIR/../charts/kaf
 echo Update values.yaml docker image tag to $APP_VERSION
 sed -i -r "s/^(\s*)(tag\s*:\s.*$)/\1tag: ${APP_VERSION}/" $DIR/../charts/kafka-lag-exporter/values.yaml
 
+echo Update values.yaml docker repository to $DOCKER_REPOSITORY
+sed -i -r "s/^(\s*)(repository\s*:\s.*$)/\1repository: ${DOCKER_REPOSITORY_ESCAPED}/" $DIR/../charts/kafka-lag-exporter/values.yaml
+
 if [[ SNAPSHOT == "SNAPSHOT" ]]; then
     sed -i -r "s/^(\s*)(pullPolicy\s*:\s.*$)/\1pullPolicy: Always/" $DIR/../charts/kafka-lag-exporter/values.yaml
 else

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
@@ -1,9 +1,9 @@
 kafka-lag-exporter {
   port = 8000
   port = ${?KAFKA_LAG_EXPORTER_PORT}
-  poll-interval = 5 seconds
+  poll-interval = 30 seconds
   poll-interval = ${?KAFKA_LAG_EXPORTER_POLL_INTERVAL_SECONDS}
-  lookup-table-size = 20
+  lookup-table-size = 60
   lookup-table-size = ${?KAFKA_LAG_EXPORTER_LOOKUP_TABLE_SIZE}
   client-group-id = "kafkalagexporter"
   client-group-id = ${?KAFKA_LAG_EXPORTER_CLIENT_GROUP_ID}

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/AppConfig.scala b/src/main/scala/com/lightbend/kafkalagexporter/AppConfig.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/ConsumerGroupCollector.scala b/src/main/scala/com/lightbend/kafkalagexporter/ConsumerGroupCollector.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter
@@ -148,7 +148,7 @@ object ConsumerGroupCollector {
       mostRecentPoint <- tables(gtp.topicPartition).mostRecentPoint().toOption
     } yield {
       val timeLag = tables(gtp.topicPartition).lookup(groupPoint.offset) match {
-        case Prediction(pxTime) => (newOffsets.timestamp.toDouble - pxTime) / 1000
+        case Prediction(pxTime) => (groupPoint.time.toDouble - pxTime) / 1000
         case LagIsZero => 0d
         case TooFewPoints => Double.NaN
       }

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/Domain.scala b/src/main/scala/com/lightbend/kafkalagexporter/Domain.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/KafkaClient.scala b/src/main/scala/com/lightbend/kafkalagexporter/KafkaClient.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/KafkaClusterManager.scala b/src/main/scala/com/lightbend/kafkalagexporter/KafkaClusterManager.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/LookupTable.scala b/src/main/scala/com/lightbend/kafkalagexporter/LookupTable.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter
@@ -74,8 +74,10 @@ object LookupTable {
 
       points.toList match {
         case p if p.length < 2 => TooFewPoints
-        // if we only have 2 points at the same offset
-        case p1 :: p2 :: Nil if p1.offset == p2.offset => LagIsZero
+        // if we only have 2 points the same as the current offset then report
+        // lag as zero
+        case p1 :: p2 :: Nil if p1.offset == p2.offset &&
+                                p2.offset == offset => LagIsZero
         case _ => estimate()
 
       }

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/MainApp.scala b/src/main/scala/com/lightbend/kafkalagexporter/MainApp.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/Metrics.scala b/src/main/scala/com/lightbend/kafkalagexporter/Metrics.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/MetricsReporter.scala b/src/main/scala/com/lightbend/kafkalagexporter/MetricsReporter.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/MetricsSink.scala b/src/main/scala/com/lightbend/kafkalagexporter/MetricsSink.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/PrometheusEndpointSink.scala b/src/main/scala/com/lightbend/kafkalagexporter/PrometheusEndpointSink.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/watchers/StrimziClient.scala b/src/main/scala/com/lightbend/kafkalagexporter/watchers/StrimziClient.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter.watchers

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/watchers/StrimziClusterWatcher.scala b/src/main/scala/com/lightbend/kafkalagexporter/watchers/StrimziClusterWatcher.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter.watchers

diff --git a/src/main/scala/com/lightbend/kafkalagexporter/watchers/Watcher.scala b/src/main/scala/com/lightbend/kafkalagexporter/watchers/Watcher.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 - 2019 Lightbend Inc. <http://www.lightbend.com>
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
  */
 
 package com.lightbend.kafkalagexporter.watchers

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/ConsumerGroupCollectorSpec.scala b/src/test/scala/com/lightbend/kafkalagexporter/ConsumerGroupCollectorSpec.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter
 
 import java.time.{Clock, Instant, ZoneId}

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/LookupTableSpec.scala b/src/test/scala/com/lightbend/kafkalagexporter/LookupTableSpec.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter
 
 import com.lightbend.kafkalagexporter.LookupTable.Table.{LagIsZero, Prediction, TooFewPoints}
@@ -76,7 +80,7 @@ class LookupTableSpec extends FreeSpec with Matchers {
         table.lookup(301) shouldBe Prediction(9000.3)
       }
 
-      "lookups when table only contains flat section" in {
+      "lookups when table only contains a flat section with offsets same as lookup" in {
         val table = Table(5)
 
         table.addPoint(Point(0, 0))
@@ -101,8 +105,8 @@ class LookupTableSpec extends FreeSpec with Matchers {
           fail(s"Expected compressed table to have last timestamp $table")
         }
 
-        table.lookup(99) shouldBe LagIsZero
-        table.lookup(101) shouldBe LagIsZero
+        table.lookup(99) shouldBe Prediction(Double.NegativeInfinity)
+        table.lookup(101) shouldBe Prediction(Double.PositiveInfinity)
       }
 
       "normal case, table truncates, steady timestamps, different val rates" in {

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/TestData.scala b/src/test/scala/com/lightbend/kafkalagexporter/TestData.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter
 
 import com.lightbend.kafkalagexporter.Domain._

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/integration/IntegrationSpec.scala b/src/test/scala/com/lightbend/kafkalagexporter/integration/IntegrationSpec.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter.integration
 
 import akka.actor.testkit.typed.scaladsl.ActorTestKit

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/integration/LagSim.scala b/src/test/scala/com/lightbend/kafkalagexporter/integration/LagSim.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter.integration
 
 import akka.actor.Cancellable

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/integration/PrometheusUtils.scala b/src/test/scala/com/lightbend/kafkalagexporter/integration/PrometheusUtils.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter.integration
 
 import akka.actor.ActorSystem

diff --git a/src/test/scala/com/lightbend/kafkalagexporter/integration/SpecBase.scala b/src/test/scala/com/lightbend/kafkalagexporter/integration/SpecBase.scala
@@ -1,3 +1,7 @@
+/*
+ * Copyright (C) 2019 Lightbend Inc. <http://www.lightbend.com>
+ */
+
 package com.lightbend.kafkalagexporter.integration
 
 import akka.actor.typed.ActorSystem
@@ -43,6 +47,8 @@ abstract class SpecBase(kafkaPort: Int, val exporterPort: Int)
                                             |      bootstrap-brokers: "localhost:$kafkaPort"
                                             |    }
                                             |  ]
+                                            |  poll-interval = 5 seconds
+                                            |  lookup-table-size = 20
                                             |}""".stripMargin).withFallback(ConfigFactory.load())
 
   override def beforeEach(): Unit = {