Spectatorize (#139)

Additional metrics for atlas queries and pipeline failure metric
spinnaker · Dec 1, 2017 · 9821a37 · 9821a37
1 parent dd062ce
commit 9821a37
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 10 deletions.
diff --git a/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java b/kayenta-atlas/src/main/java/com/netflix/kayenta/atlas/metrics/AtlasMetricsService.java
@@ -31,13 +31,16 @@
 import com.netflix.kayenta.retrofit.config.RemoteService;
 import com.netflix.kayenta.retrofit.config.RetrofitClientFactory;
 import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.spectator.api.Registry;
+import com.netflix.spectator.api.Timer;
 import com.squareup.okhttp.OkHttpClient;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.Singular;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import javax.inject.Inject;
 import javax.validation.constraints.NotNull;
 import java.io.IOException;
 import java.time.Duration;
@@ -58,13 +61,16 @@ public class AtlasMetricsService implements MetricsService {
   private List<String> accountNames;
 
   @Autowired
-  AccountCredentialsRepository accountCredentialsRepository;
+  private final AccountCredentialsRepository accountCredentialsRepository;
 
   @Autowired
-  RetrofitClientFactory retrofitClientFactory;
+  private final RetrofitClientFactory retrofitClientFactory;
 
   @Autowired
-  AtlasSSEConverter atlasSSEConverter;
+  private final AtlasSSEConverter atlasSSEConverter;
+
+  @Autowired
+  private final Registry registry;
 
   @Override
   public String getType() {
@@ -121,12 +127,20 @@ public List<MetricSet> queryMetrics(String accountName,
     AtlasCanaryMetricSetQueryConfig atlasMetricSetQuery = (AtlasCanaryMetricSetQueryConfig)canaryMetricConfig.getQuery();
     String decoratedQuery = atlasMetricSetQuery.getQ() + "," + atlasCanaryScope.cq();
     String isoStep = Duration.of(atlasCanaryScope.getStep(), SECONDS) + "";
-    List<AtlasResults> atlasResultsList = atlasRemoteService.fetch(decoratedQuery,
-                                                                   atlasCanaryScope.getStart().toEpochMilli(),
-                                                                   atlasCanaryScope.getEnd().toEpochMilli(),
-                                                                   isoStep,
-                                                                   credentials.getFetchId(),
-                                                                   UUID.randomUUID() + "");
+
+    long start = registry.clock().monotonicTime();
+    List < AtlasResults > atlasResultsList;
+    try {
+      atlasResultsList = atlasRemoteService.fetch(decoratedQuery,
+                                                  atlasCanaryScope.getStart().toEpochMilli(),
+                                                  atlasCanaryScope.getEnd().toEpochMilli(),
+                                                  isoStep,
+                                                  credentials.getFetchId(),
+                                                  UUID.randomUUID() + "");
+    } finally {
+      long end = registry.clock().monotonicTime();
+      registry.timer("atlas.fetchTime").record(end - start, TimeUnit.NANOSECONDS);
+    }
     Map<String, AtlasResults> idToAtlasResultsMap = AtlasResultsHelper.merge(atlasResultsList);
     List<MetricSet> metricSetList = new ArrayList<>();
 

diff --git a/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryConfigController.java b/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryConfigController.java
@@ -24,6 +24,7 @@
 import com.netflix.kayenta.storage.ObjectType;
 import com.netflix.kayenta.storage.StorageService;
 import com.netflix.kayenta.storage.StorageServiceRepository;
+import com.netflix.spectator.api.Registry;
 import com.netflix.spinnaker.kork.web.exceptions.NotFoundException;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;

diff --git a/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java b/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java
@@ -28,6 +28,7 @@
 import com.netflix.kayenta.storage.ObjectType;
 import com.netflix.kayenta.storage.StorageService;
 import com.netflix.kayenta.storage.StorageServiceRepository;
+import com.netflix.spectator.api.Id;
 import com.netflix.spectator.api.Registry;
 import com.netflix.spinnaker.orca.ExecutionStatus;
 import com.netflix.spinnaker.orca.pipeline.ExecutionLauncher;
@@ -64,6 +65,8 @@ public class CanaryController {
   private final List<CanaryScopeFactory> canaryScopeFactories;
   private final Registry registry;
   private final ObjectMapper kayentaObjectMapper;
+  private final Id pipelineRunId;
+  private final Id failureId;
 
   @Autowired
   public CanaryController(String currentInstanceId,
@@ -88,6 +91,8 @@ public CanaryController(String currentInstanceId,
 
     this.registry = registry;
     this.kayentaObjectMapper = kayentaObjectMapper;
+    this.pipelineRunId = registry.createId("canary.pipelines.initiated");
+    this.failureId = registry.createId("canary.pipelines.startupFailed");
   }
 
   //
@@ -136,7 +141,7 @@ public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) fi
         .filter((f) -> f.handles(serviceType)).findFirst()
         .orElseThrow(() -> new IllegalArgumentException("Unable to resolve canary scope factory for '" + serviceType + "'."));
 
-    registry.counter(registry.createId("canary.pipelines.initiated")).increment();
+    registry.counter(pipelineRunId.withTag("canaryConfigId", canaryConfigId).withTag("canaryConfigName", canaryConfig.getName())).increment();
 
     CanaryScope controlScopeModel = canaryScopeFactory.buildCanaryScope(canaryExecutionRequest.getControlScope());
     CanaryScope experimentScopeModel = canaryScopeFactory.buildCanaryScope(canaryExecutionRequest.getExperimentScope());
@@ -314,6 +319,8 @@ private Execution handleStartupFailure(Execution execution, Throwable failure) {
     executionRepository.updateStatus(execution.getId(), status);
     executionRepository.cancel(execution.getId(), canceledBy, reason);
 
+    registry.counter(failureId).increment();
+
     return executionRepository.retrieve(execution.getType(), execution.getId());
   }
 }