refactor(stats): add a simpler way to collect more data for stats (#943)

* refactor(stats): convert some of the tests to JUnit/Kotlin

I didn't convert the ones that deal with the stats event content because I'm going to refactor how that works in a later commit.

* feat(stats): add a TelemetryEventDataProvider interface

Implementations can add their own data to the stats Event object. This allows us to pull data from a variety of sources without having to cram all that logic into TelemetryEventListener.

* refactor(stats): convert TelemetryService and TelemetryConfig to Kotlin

Some of the Groovy/Java code gets a little weirder because TelemetryConfigProps is in Kotlin, but after the next commit, all its clients will be in Kotlin, so it'll make more sense.

* refactor(stats): move the data collection out of TelemetryEventListener

Now it is split between two different ExecutionDataProviders. For now, this is a little arbitrary, but I anticipate adding more complicated data providers that do RPCs and things, so this should make those future data providers much simpler to reason about and test.

* fix(stats): fix the default stats logging URL

* fix(stats): don't use the term "whitelist"

echo-telemetry/echo-telemetry.gradle

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+apply from: "${project.rootDir}/gradle/kotlin.gradle"
+
 dependencies {
   api project(':echo-api')
   implementation project(':echo-model')
@@ -25,4 +27,12 @@ dependencies {
   implementation 'com.squareup.retrofit:converter-jackson'
   implementation 'de.huxhorn.sulky:de.huxhorn.sulky.ulid'
   implementation "org.apache.commons:commons-lang3"
+  testImplementation 'io.mockk:mockk'
+  testImplementation 'io.strikt:strikt-core'
+}
+
+test {
+  useJUnitPlatform {
+    includeEngines("junit-vintage", "junit-jupiter")
+  }
 }

echo-telemetry/src/main/java/com/netflix/spinnaker/echo/config/TelemetryConfig.kt

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.echo.config
+
+import com.netflix.spinnaker.echo.config.TelemetryConfig.TelemetryConfigProps
+import com.netflix.spinnaker.echo.telemetry.TelemetryService
+import com.netflix.spinnaker.retrofit.RetrofitConfigurationProperties
+import com.netflix.spinnaker.retrofit.Slf4jRetrofitLogger
+import com.squareup.okhttp.OkHttpClient
+import de.huxhorn.sulky.ulid.ULID
+import java.util.concurrent.TimeUnit
+import org.slf4j.LoggerFactory
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import retrofit.RestAdapter
+import retrofit.client.OkClient
+import retrofit.converter.JacksonConverter
+
+@Configuration
+@ConditionalOnProperty("stats.enabled")
+@EnableConfigurationProperties(TelemetryConfigProps::class)
+open class TelemetryConfig {
+
+  companion object {
+    private val log = LoggerFactory.getLogger(TelemetryConfig::class.java)
+  }
+
+  @Bean
+  open fun telemetryService(
+    retrofitConfigurationProperties: RetrofitConfigurationProperties,
+    configProps: TelemetryConfigProps
+  ): TelemetryService {
+    log.info("Telemetry service loaded")
+    return RestAdapter.Builder()
+      .setEndpoint(configProps.endpoint)
+      .setConverter(JacksonConverter())
+      .setClient(telemetryOkClient(configProps))
+      .setLogLevel(retrofitConfigurationProperties.logLevel)
+      .setLog(Slf4jRetrofitLogger(TelemetryService::class.java))
+      .build()
+      .create(TelemetryService::class.java)
+  }
+
+  private fun telemetryOkClient(configProps: TelemetryConfigProps): OkClient {
+    val httpClient = OkHttpClient()
+    httpClient.setConnectTimeout(configProps.connectionTimeoutMillis.toLong(), TimeUnit.MILLISECONDS)
+    httpClient.setReadTimeout(configProps.readTimeoutMillis.toLong(), TimeUnit.MILLISECONDS)
+    return OkClient(httpClient)
+  }
+
+  @ConfigurationProperties(prefix = "stats")
+  class TelemetryConfigProps {
+
+    companion object {
+      const val DEFAULT_TELEMETRY_ENDPOINT = "https://stats.spinnaker.io"
+    }
+
+    var enabled = false
+    var endpoint = DEFAULT_TELEMETRY_ENDPOINT
+    var instanceId = ULID().nextULID()
+    var spinnakerVersion = "unknown"
+    var deploymentMethod = DeploymentMethod()
+    var connectionTimeoutMillis = 3000
+    var readTimeoutMillis = 5000
+
+    class DeploymentMethod {
+      var type: String? = null
+      var version: String? = null
+    }
+  }
+}

echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/ExecutionDataProvider.kt

@@ -0,0 +1,142 @@
+/*
+ * Copyright 2020 Google, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.echo.telemetry
+
+import com.netflix.spinnaker.echo.api.events.Event as EchoEvent
+import com.netflix.spinnaker.echo.jackson.EchoObjectMapper
+import com.netflix.spinnaker.kork.proto.stats.CloudProvider
+import com.netflix.spinnaker.kork.proto.stats.Event as StatsEvent
+import com.netflix.spinnaker.kork.proto.stats.Execution
+import com.netflix.spinnaker.kork.proto.stats.Stage
+import com.netflix.spinnaker.kork.proto.stats.Status
+import java.util.ArrayList
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.stereotype.Component
+
+@Component
+@ConditionalOnProperty("stats.enabled")
+class ExecutionDataProvider : TelemetryEventDataProvider {
+
+  private val objectMapper = EchoObjectMapper.getInstance()
+
+  override fun populateData(echoEvent: EchoEvent, statsEvent: StatsEvent): StatsEvent {
+    val content = objectMapper.convertValue(echoEvent.getContent(), EventContent::class.java)
+    val execution = content.execution
+
+    var executionType = getExecutionType(execution.type)
+
+    if (execution.source.type.equals("templatedPipeline", ignoreCase = true)) {
+      if (execution.source.version.equals("v1", ignoreCase = true)) {
+        executionType = Execution.Type.MANAGED_PIPELINE_TEMPLATE_V1
+      } else if (execution.source.version.equals("v2", ignoreCase = true)) {
+        executionType = Execution.Type.MANAGED_PIPELINE_TEMPLATE_V2
+      }
+    }
+
+    val executionStatus = getStatus(execution.status)
+    val triggerType = getTriggerType(execution.trigger.type)
+
+    val protoStages = execution.stages.flatMap { toStages(it) }
+
+    val executionBuilder = Execution.newBuilder()
+      .setType(executionType)
+      .setStatus(executionStatus)
+      .setTrigger(Execution.Trigger.newBuilder().setType(triggerType))
+      .addAllStages(protoStages)
+    val executionId: String = execution.id
+    if (executionId.isNotEmpty()) {
+      executionBuilder.id = hash(executionId)
+    }
+    val executionProto = executionBuilder.build()
+
+    return statsEvent.toBuilder()
+      .setExecution(statsEvent.execution.toBuilder().mergeFrom(executionProto))
+      .build()
+  }
+
+  private fun getExecutionType(type: String): Execution.Type =
+    Execution.Type.valueOf(Execution.Type.getDescriptor().findMatchingValue(type))
+
+  private fun getStatus(status: String): Status =
+    Status.valueOf(Status.getDescriptor().findMatchingValue(status))
+
+  private fun getTriggerType(type: String): Execution.Trigger.Type =
+    Execution.Trigger.Type.valueOf(Execution.Trigger.Type.getDescriptor().findMatchingValue(type))
+
+  private fun toStages(stage: EventContent.Stage): List<Stage> {
+    // Only interested in user-configured stages.
+    if (stage.isSyntheticStage()) {
+      return listOf()
+    }
+
+    val stageStatus = getStatus(stage.status)
+    val stageBuilder = Stage.newBuilder().setType(stage.type).setStatus(stageStatus)
+    val returnList: MutableList<Stage> = ArrayList()
+    val cloudProvider = stage.context.cloudProvider
+    if (!cloudProvider.isNullOrEmpty()) {
+      stageBuilder.cloudProvider = getCloudProvider(cloudProvider)
+      returnList.add(stageBuilder.build())
+    } else if (!stage.context.newState.cloudProviders.isNullOrEmpty()) {
+      // Create and Update Application operations can specify multiple cloud providers in 1
+      // operation.
+      val cloudProviders = stage.context.newState.cloudProviders.split(",")
+      for (cp in cloudProviders) {
+        returnList.add(stageBuilder.clone().setCloudProvider(getCloudProvider(cp)).build())
+      }
+    } else {
+      returnList.add(stageBuilder.build())
+    }
+    return returnList
+  }
+
+  private fun getCloudProvider(cloudProvider: String): CloudProvider {
+    val cloudProviderId =
+      CloudProvider.ID.valueOf(CloudProvider.ID.getDescriptor().findMatchingValue(cloudProvider))
+    return CloudProvider.newBuilder().setId(cloudProviderId).build()
+  }
+
+  data class EventContent(val execution: Execution = Execution()) {
+
+    data class Execution(
+      val id: String = "",
+      val type: String = "UNKNOWN",
+      val status: String = "UNKNOWN",
+      val trigger: Trigger = Trigger(),
+      val source: Source = Source(),
+      val stages: List<Stage> = listOf()
+    )
+
+    data class Trigger(val type: String = "UNKNOWN")
+
+    data class Source(val type: String? = null, val version: String? = null)
+
+    data class Stage(
+      val status: String = "UNKNOWN",
+      val type: String = "UNKNOWN",
+      val syntheticStageOwner: String? = null,
+      val context: Context = Context()
+    ) {
+
+      fun isSyntheticStage() = !syntheticStageOwner.isNullOrEmpty()
+    }
+
+    data class Context(val cloudProvider: String? = null, val newState: NewState = NewState())
+
+    data class NewState(val cloudProviders: String? = null)
+  }
+}

echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/ProtobufEnums.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 Google, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.netflix.spinnaker.echo.telemetry
+
+import com.google.protobuf.Descriptors.EnumDescriptor
+import com.google.protobuf.Descriptors.EnumValueDescriptor
+
+internal fun EnumDescriptor.findMatchingValue(name: String): EnumValueDescriptor {
+  // If we can't find it, just return the "UNKNOWN" value
+  return findValueByName(name.toUpperCase()) ?: findValueByNumber(0)
+}