From 16b5ab48dea092b5651382814f732b7fdd80afb1 Mon Sep 17 00:00:00 2001 From: Stacie Graves Date: Thu, 22 Aug 2019 15:59:12 -0600 Subject: [PATCH] feat(telemetry): adds listener for telemetry metrics --- build.gradle | 1 + echo-proto/echo-proto.gradle | 47 +++++++++ echo-proto/src/main/proto/event.proto | 83 ++++++++++++++++ echo-telemetry/echo-telemetry.gradle | 25 +++++ .../echo/config/TelemetryConfig.java | 65 +++++++++++++ .../telemetry/TelemetryEventListener.java | 97 +++++++++++++++++++ .../echo/telemetry/TelemetryService.java | 26 +++++ .../echo/TelemetryEventListenerSpec.groovy | 54 +++++++++++ echo-web/echo-web.gradle | 2 + settings.gradle | 4 +- 10 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 echo-proto/echo-proto.gradle create mode 100644 echo-proto/src/main/proto/event.proto create mode 100644 echo-telemetry/echo-telemetry.gradle create mode 100644 echo-telemetry/src/main/java/com/netflix/spinnaker/echo/config/TelemetryConfig.java create mode 100644 echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryEventListener.java create mode 100644 echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryService.java create mode 100644 echo-telemetry/src/test/groovy/com/netflix/spinnaker/echo/TelemetryEventListenerSpec.groovy diff --git a/build.gradle b/build.gradle index 332062675..6de8830aa 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,7 @@ buildscript { // this override is needed to omit compileOnly dependencies from generated pom.xml classpath "com.netflix.nebula:nebula-publishing-plugin:12.0.1" } + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.8' } } diff --git a/echo-proto/echo-proto.gradle b/echo-proto/echo-proto.gradle new file mode 100644 index 000000000..3ae7e3cad --- /dev/null +++ b/echo-proto/echo-proto.gradle @@ -0,0 +1,47 @@ +apply plugin: 'java' +apply plugin: 'com.google.protobuf' + +dependencies { + implementation 'com.google.protobuf:protobuf-java:3.0.0' + implementation 'com.google.guava:guava:23.5-jre' + api 'com.google.api.grpc:grpc-google-common-protos:1.0.5' + implementation 'io.grpc:grpc-all:1.8.0' +} + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.5.1-1' + } + plugins { + grpc { + artifact = "io.grpc:protoc-gen-grpc-java:1.9.0" + } + } + + generateProtoTasks { + ofSourceSet('main').each { task -> + task.builtins { + java{ + outputSubDir = 'java' + } + } + task.plugins { + grpc { + outputSubDir = 'java' + } + } + task.descriptorSetOptions.includeImports = true + } + } + generatedFilesBaseDir = "${projectDir}/build-gen/proto" +} + +idea { + module { + sourceDirs += file("${projectDir}/build-gen/proto/main/java") + } +} + +clean { + delete "${projectDir}/build-gen" +} diff --git a/echo-proto/src/main/proto/event.proto b/echo-proto/src/main/proto/event.proto new file mode 100644 index 000000000..274ec6b48 --- /dev/null +++ b/echo-proto/src/main/proto/event.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package spinnaker.echo; + +option java_package = "com.netflix.spinnaker.echo.proto"; +option java_multiple_files = true; + +message EventProto { + SpinnakerInstance spinnaker_instance = 1; +} + +message SpinnakerInstance { + string id = 1; + string version = 2; + Application application = 3; +} + +message Application { + string id = 1; + Execution execution = 2; +} + +message Execution { + string id = 1; + Type type = 2; + + enum Type { + UNKNOWN = 0; + PIPELINE = 1; + ORCHESTRATION = 2; + MANAGED_PIPELINE_TEMPLATE_V1 = 3; + MANAGED_PIPELINE_TEMPLATE_V2 = 4; + } + + Trigger trigger = 3; + + message Trigger { + Type type = 1; + + // Sourced from https://github.com/spinnaker/echo/tree/master/echo-model/src/main/java/com/netflix/spinnaker/echo/model/trigger + enum Type { + UNKNOWN = 0; + ARTIFACTORY = 1; + BUILD = 2; + DOCKER = 3; + GIT = 4; + MANUAL = 5; + PUBSUB = 6; + WEBHOOK = 7; + } + } + + repeated Stage stages = 4; + Status status = 5; +} + +message Stage { + string type = 1; + Status status = 2; + CloudProvider cloud_provider = 3; +} + +message CloudProvider { + string id = 1; + string variant = 2; +} + +// Sourced from https://github.com/spinnaker/orca/blob/master/orca-core/src/main/java/com/netflix/spinnaker/orca/ExecutionStatus.java +enum Status { + UNKNOWN = 0; + NOT_STARTED = 1; + RUNNING = 2; + PAUSED = 3; + SUSPENDED = 4; + SUCCEEDED = 5; + FAILED_CONTINUE = 6; + TERMINAL = 7; + CANCELED = 8; + REDIRECT = 9; + STOPPED = 10; + SKIPPED = 11; + BUFFERED = 12; +} \ No newline at end of file diff --git a/echo-telemetry/echo-telemetry.gradle b/echo-telemetry/echo-telemetry.gradle new file mode 100644 index 000000000..bbec71baf --- /dev/null +++ b/echo-telemetry/echo-telemetry.gradle @@ -0,0 +1,25 @@ +/* + * Copyright 2019 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(':echo-model') + implementation project(':echo-notifications') + implementation project(':echo-proto') + implementation 'com.squareup.retrofit:retrofit' + implementation 'com.squareup.retrofit:converter-jackson' + implementation 'com.netflix.spinnaker.kork:kork-web' + implementation 'com.google.protobuf:protobuf-java-util:3.7.1' +} diff --git a/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/config/TelemetryConfig.java b/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/config/TelemetryConfig.java new file mode 100644 index 000000000..07dceaa1e --- /dev/null +++ b/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/config/TelemetryConfig.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.echo.config; + +import static retrofit.Endpoints.newFixedEndpoint; + +import com.netflix.spinnaker.echo.telemetry.TelemetryService; +import com.netflix.spinnaker.retrofit.Slf4jRetrofitLogger; +import groovy.transform.CompileStatic; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import retrofit.Endpoint; +import retrofit.RestAdapter; +import retrofit.client.Client; +import retrofit.converter.JacksonConverter; + +@Slf4j +@Configuration +@ConditionalOnProperty("telemetry.enabled") +@CompileStatic +class TelemetryConfig { + + @Value("${telemetry.endpoint}") + String endpoint; + + @Bean + Endpoint telemetryEndpoint() { + return newFixedEndpoint(endpoint); + } + + @Bean + public TelemetryService telemetryService( + Endpoint telemetryEndpoint, Client retrofitClient, RestAdapter.LogLevel retrofitLogLevel) { + log.info("Telemetry service loaded"); + + TelemetryService client = + new RestAdapter.Builder() + .setEndpoint(telemetryEndpoint) + .setConverter(new JacksonConverter()) + .setClient(retrofitClient) + .setLogLevel(RestAdapter.LogLevel.FULL) + .setLog(new Slf4jRetrofitLogger(TelemetryService.class)) + .build() + .create(TelemetryService.class); + + return client; + } +} diff --git a/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryEventListener.java b/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryEventListener.java new file mode 100644 index 000000000..db50d26be --- /dev/null +++ b/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryEventListener.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.echo.telemetry; + +import com.google.protobuf.util.JsonFormat; +import com.netflix.spinnaker.echo.events.EchoEventListener; +import com.netflix.spinnaker.echo.model.Event; +import com.netflix.spinnaker.echo.proto.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@ConditionalOnProperty("telemetry.enabled") +public class TelemetryEventListener implements EchoEventListener { + private final TelemetryService telemetryService; + + @Autowired + public TelemetryEventListener(TelemetryService telemetryService) { + this.telemetryService = telemetryService; + } + + @Value("${telemetry.instanceId}") + String instanceId; + + @Override + public void processEvent(Event event) { + try { + if (event.getDetails() == null + || !event.getDetails().getType().equals("orca:pipeline:complete")) { + return; + } + + Map execution = (Map) event.content.get("execution"); + Execution.Builder executionBuilder = getExecutionBuilder(execution); + + List stages = (ArrayList) execution.get("stages"); + for (Map stage : stages) { + executionBuilder.addStages(getStageBuilder(stage)); + } + + Application.Builder applicationBuilder = + Application.newBuilder() + .setId(event.details.getApplication()) + .setExecution(executionBuilder); + + SpinnakerInstance.Builder spinnakerInstance = + SpinnakerInstance.newBuilder().setId(instanceId).setApplication(applicationBuilder); + + EventProto.Builder eventProto = + EventProto.newBuilder().setSpinnakerInstance(spinnakerInstance); + + telemetryService.sendMessage(JsonFormat.printer().print(eventProto)); + + } catch (Exception e) { + log.error("Could not send Telemetry event {}", event, e); + } + } + + private Execution.Builder getExecutionBuilder(Map execution) { + Map trigger = (Map) execution.get("trigger"); + return Execution.newBuilder() + .setId(execution.get("id").toString()) + .setType(Execution.Type.valueOf(execution.get("type").toString().toUpperCase())) + .setTrigger( + Execution.Trigger.newBuilder() + .setType( + Execution.Trigger.Type.valueOf(trigger.get("type").toString().toUpperCase()))) + .setStatus(Status.valueOf(execution.get("status").toString().toUpperCase())); + } + + private Stage.Builder getStageBuilder(Map stage) { + return Stage.newBuilder() + .setType(stage.get("type").toString()) + .setStatus(Status.valueOf(stage.get("status").toString().toUpperCase())); + } +} diff --git a/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryService.java b/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryService.java new file mode 100644 index 000000000..f5bf25d2c --- /dev/null +++ b/echo-telemetry/src/main/java/com/netflix/spinnaker/echo/telemetry/TelemetryService.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.echo.telemetry; + +import retrofit.client.Response; +import retrofit.http.Body; +import retrofit.http.POST; + +public interface TelemetryService { + @POST("/}") + Response sendMessage(@Body String body); +} diff --git a/echo-telemetry/src/test/groovy/com/netflix/spinnaker/echo/TelemetryEventListenerSpec.groovy b/echo-telemetry/src/test/groovy/com/netflix/spinnaker/echo/TelemetryEventListenerSpec.groovy new file mode 100644 index 000000000..74aafd1d9 --- /dev/null +++ b/echo-telemetry/src/test/groovy/com/netflix/spinnaker/echo/TelemetryEventListenerSpec.groovy @@ -0,0 +1,54 @@ +package com.netflix.spinnaker.echo + +import com.netflix.spinnaker.echo.model.Event +import com.netflix.spinnaker.echo.telemetry.TelemetryEventListener +import com.netflix.spinnaker.echo.telemetry.TelemetryService +import spock.lang.Specification +import spock.lang.Subject + +class TelemetryEventListenerSpec extends Specification { + def service = Mock(TelemetryService) + + @Subject + def listener = new TelemetryEventListener(service) + + void setup() { + listener.instanceId = "test-instance" + } + + def "send a telemetry event"() { + given: + Event event = new Event( + details: [ + type : "orca:pipeline:complete", + application: "some-application" + ], + content: [ + execution: [ + id : "execution_id", + type : "PIPELINE", + status : "SUCCEEDED", + trigger: [ + type: "GIT" + ], + stages : [ + [ + type : "deploy", + status: "SUCCEEDED" + ], + [ + type : "wait", + status: "SUCCEEDED" + ], + ] + ] + ] + ) + + when: + listener.processEvent(event) + + then: + 1 * service.sendMessage('{\n "spinnakerInstance": {\n "id": "test-instance",\n "application": {\n "id": "some-application",\n "execution": {\n "id": "execution_id",\n "type": "PIPELINE",\n "trigger": {\n "type": "GIT"\n },\n "stages": [{\n "type": "deploy",\n "status": "SUCCEEDED"\n }, {\n "type": "wait",\n "status": "SUCCEEDED"\n }],\n "status": "SUCCEEDED"\n }\n }\n }\n}') + } +} diff --git a/echo-web/echo-web.gradle b/echo-web/echo-web.gradle index da4dc5116..04b00e880 100644 --- a/echo-web/echo-web.gradle +++ b/echo-web/echo-web.gradle @@ -35,6 +35,8 @@ dependencies { implementation project(':echo-pubsub-core') implementation project(':echo-pubsub-aws') implementation project(':echo-pubsub-google') + implementation project(':echo-proto') + implementation project(':echo-telemetry') implementation "com.netflix.spinnaker.fiat:fiat-api:$fiatVersion" implementation "com.netflix.spinnaker.fiat:fiat-core:$fiatVersion" implementation "org.springframework.boot:spring-boot-starter-web" diff --git a/settings.gradle b/settings.gradle index 01d953529..9687449d7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -27,7 +27,9 @@ include 'echo-artifacts', 'echo-pubsub-aws', 'echo-pubsub-google', 'echo-test', - 'echo-bom' + 'echo-bom', + 'echo-telemetry', + 'echo-proto' rootProject.name = 'echo'