feat(standalone-canary-analysis): add SCAPE complete event listener a…

…nd archive the status response object in the object store. (#593)
spinnaker · Jul 31, 2019 · 5cc068a · 5cc068a
1 parent 49c1c2c
commit 5cc068a
Show file tree

Hide file tree

Showing 29 changed files with 208 additions and 81 deletions.
diff --git a/.../CanaryAnalysisAdhocExecutionRequest.java → .../CanaryAnalysisAdhocExecutionRequest.java b/.../CanaryAnalysisAdhocExecutionRequest.java → .../CanaryAnalysisAdhocExecutionRequest.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import com.netflix.kayenta.canary.CanaryConfig;
 import io.swagger.annotations.ApiModel;

diff --git a/...omain/CanaryAnalysisExecutionRequest.java → ...lysis/CanaryAnalysisExecutionRequest.java b/...omain/CanaryAnalysisExecutionRequest.java → ...lysis/CanaryAnalysisExecutionRequest.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.netflix.kayenta.canary.CanaryClassifierThresholdsConfig;

diff --git a/.../CanaryAnalysisExecutionRequestScope.java → .../CanaryAnalysisExecutionRequestScope.java b/.../CanaryAnalysisExecutionRequestScope.java → .../CanaryAnalysisExecutionRequestScope.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.swagger.annotations.ApiModel;

diff --git a/...main/CanaryAnalysisExecutionResponse.java → ...ysis/CanaryAnalysisExecutionResponse.java b/...main/CanaryAnalysisExecutionResponse.java → ...ysis/CanaryAnalysisExecutionResponse.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;

diff --git a/...domain/CanaryAnalysisExecutionResult.java → ...alysis/CanaryAnalysisExecutionResult.java b/...domain/CanaryAnalysisExecutionResult.java → ...alysis/CanaryAnalysisExecutionResult.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;

diff --git a/...anaryAnalysisExecutionStatusResponse.java → ...anaryAnalysisExecutionStatusResponse.java b/...anaryAnalysisExecutionStatusResponse.java → ...anaryAnalysisExecutionStatusResponse.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.netflix.kayenta.canary.CanaryConfig;
@@ -125,4 +125,12 @@ public String status() {
       value =
           "endTimeIso is an ISO 8061 string and refers to the time the pipeline ended, either successfully or unsuccessfully.")
   protected String endTimeIso;
+
+  @NotNull
+  @ApiModelProperty(value = "The resolved storage account name.")
+  protected String storageAccountName;
+
+  @NotNull
+  @ApiModelProperty(value = "The resolved metrics account name.")
+  protected String metricsAccountName;
 }
diff --git a/...nalysis/domain/CanaryExecutionResult.java → ...canaryanalysis/CanaryExecutionResult.java b/...nalysis/domain/CanaryExecutionResult.java → ...canaryanalysis/CanaryExecutionResult.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.netflix.kayenta.canary.results.CanaryResult;

diff --git a/.../canaryanalysis/domain/StageMetadata.java → ...andalonecanaryanalysis/StageMetadata.java b/.../canaryanalysis/domain/StageMetadata.java → ...andalonecanaryanalysis/StageMetadata.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.domain.standalonecanaryanalysis;
 
 import com.netflix.spinnaker.orca.ExecutionStatus;
 import io.swagger.annotations.ApiModel;

diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/storage/ObjectType.java b/kayenta-core/src/main/java/com/netflix/kayenta/storage/ObjectType.java
@@ -19,6 +19,7 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.netflix.kayenta.canary.CanaryConfig;
 import com.netflix.kayenta.canary.CanaryExecutionStatusResponse;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionStatusResponse;
 import com.netflix.kayenta.metrics.MetricSet;
 import com.netflix.kayenta.metrics.MetricSetPair;
 import java.util.List;
@@ -28,13 +29,17 @@
 @AllArgsConstructor
 public enum ObjectType {
   CANARY_CONFIG(new TypeReference<CanaryConfig>() {}, "canary_config", "canary_config.json"),
-  METRIC_SET_LIST(new TypeReference<List<MetricSet>>() {}, "metrics", "metric_sets.json"),
-  METRIC_SET_PAIR_LIST(
-      new TypeReference<List<MetricSetPair>>() {}, "metric_pairs", "metric_set_pairs.json"),
   CANARY_RESULT_ARCHIVE(
       new TypeReference<CanaryExecutionStatusResponse>() {},
       "canary_archive",
-      "canary_archive.json");
+      "canary_archive.json"),
+  METRIC_SET_LIST(new TypeReference<List<MetricSet>>() {}, "metrics", "metric_sets.json"),
+  METRIC_SET_PAIR_LIST(
+      new TypeReference<List<MetricSetPair>>() {}, "metric_pairs", "metric_set_pairs.json"),
+  STANDALONE_CANARY_RESULT_ARCHIVE(
+      new TypeReference<CanaryAnalysisExecutionStatusResponse>() {},
+      "standalone_canary_archive",
+      "standalone_canary_archive.json");
 
   @Getter final TypeReference typeReference;
 

diff --git a/...t/java/com/netflix/kayenta/signalfx/EndToEndStandaloneCanaryAnalysisIntegrationTests.java b/...t/java/com/netflix/kayenta/signalfx/EndToEndStandaloneCanaryAnalysisIntegrationTests.java
@@ -25,10 +25,10 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.netflix.kayenta.canary.CanaryClassifierThresholdsConfig;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisAdhocExecutionRequest;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionRequest;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionRequestScope;
-import com.netflix.kayenta.canaryanalysis.domain.StageMetadata;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisAdhocExecutionRequest;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionRequest;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionRequestScope;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.StageMetadata;
 import io.restassured.response.ValidatableResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.BeforeClass;

diff --git a/...analysis/domain/CanaryAnalysisConfig.java → ...ecanaryanalysis/CanaryAnalysisConfig.java b/...analysis/domain/CanaryAnalysisConfig.java → ...ecanaryanalysis/CanaryAnalysisConfig.java
@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.standalonecanaryanalysis;
 
 import com.netflix.kayenta.canary.CanaryConfig;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionRequest;
 import javax.validation.constraints.NotNull;
 import lombok.AllArgsConstructor;
 import lombok.Builder;

diff --git a/...oneCanaryAnalysisModuleConfiguration.java → ...oneCanaryAnalysisModuleConfiguration.java b/...oneCanaryAnalysisModuleConfiguration.java → ...oneCanaryAnalysisModuleConfiguration.java
@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.config;
+package com.netflix.kayenta.standalonecanaryanalysis.config;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 
 @Configuration
-@ConditionalOnProperty("kayenta.standalone-canary-analysis.enabled")
-@ComponentScan({"com.netflix.kayenta.canaryanalysis"})
+@ConditionalOnProperty("kayenta.standalonecanaryanalysis-canary-analysis.enabled")
+@ComponentScan({"com.netflix.kayenta.standalonecanaryanalysis"})
 @Slf4j
 public class StandaloneCanaryAnalysisModuleConfiguration {}
diff --git a/...r/StandaloneCanaryAnalysisController.java → ...r/StandaloneCanaryAnalysisController.java b/...r/StandaloneCanaryAnalysisController.java → ...r/StandaloneCanaryAnalysisController.java
@@ -14,22 +14,22 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.controller;
+package com.netflix.kayenta.standalonecanaryanalysis.controller;
 
 import static org.springframework.web.bind.annotation.RequestMethod.GET;
 import static org.springframework.web.bind.annotation.RequestMethod.POST;
 
 import com.netflix.kayenta.canary.CanaryConfig;
 import com.netflix.kayenta.canary.providers.metrics.QueryConfigUtils;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisAdhocExecutionRequest;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisConfig;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionRequest;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionResponse;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionStatusResponse;
-import com.netflix.kayenta.canaryanalysis.service.CanaryAnalysisService;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisAdhocExecutionRequest;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionRequest;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionResponse;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionStatusResponse;
 import com.netflix.kayenta.security.AccountCredentials;
 import com.netflix.kayenta.security.AccountCredentialsRepository;
 import com.netflix.kayenta.security.CredentialsHelper;
+import com.netflix.kayenta.standalonecanaryanalysis.CanaryAnalysisConfig;
+import com.netflix.kayenta.standalonecanaryanalysis.service.CanaryAnalysisService;
 import com.netflix.kayenta.storage.ObjectType;
 import com.netflix.kayenta.storage.StorageService;
 import com.netflix.kayenta.storage.StorageServiceRepository;
@@ -251,8 +251,15 @@ public CanaryAnalysisExecutionResponse initiateCanaryAnalysisExecutionWithConfig
   @RequestMapping(value = "/{canaryAnalysisExecutionId:.+}", method = GET)
   public CanaryAnalysisExecutionStatusResponse getCanaryAnalysisExecution(
       @ApiParam(value = "The id for the Canary Analysis Execution") @PathVariable
-          String canaryAnalysisExecutionId) {
+          final String canaryAnalysisExecutionId,
+      @ApiParam(
+              value =
+                  "The account to use to try and find the execution if not found in the execution repo",
+              example = "some-storage-account")
+          @RequestParam(required = false)
+          final String storageAccountName) {
 
-    return canaryAnalysisService.getCanaryAnalysisExecution(canaryAnalysisExecutionId);
+    return canaryAnalysisService.getCanaryAnalysisExecution(
+        canaryAnalysisExecutionId, storageAccountName);
   }
 }
diff --git a/...anaryAnalysisExecutionCompletedEvent.java → ...anaryAnalysisExecutionCompletedEvent.java b/...anaryAnalysisExecutionCompletedEvent.java → ...anaryAnalysisExecutionCompletedEvent.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.event;
+package com.netflix.kayenta.standalonecanaryanalysis.event;
 
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionStatusResponse;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionStatusResponse;
 import org.springframework.context.ApplicationEvent;
 
 public class StandaloneCanaryAnalysisExecutionCompletedEvent extends ApplicationEvent {

diff --git a/...ryAnalysisExecutionCompletedProducer.java → ...ryAnalysisExecutionCompletedProducer.java b/...ryAnalysisExecutionCompletedProducer.java → ...ryAnalysisExecutionCompletedProducer.java
@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.event;
+package com.netflix.kayenta.standalonecanaryanalysis.event;
 
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionStatusResponse;
-import com.netflix.kayenta.canaryanalysis.service.CanaryAnalysisService;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionStatusResponse;
 import com.netflix.kayenta.events.AbstractExecutionCompleteEventProcessor;
+import com.netflix.kayenta.standalonecanaryanalysis.service.CanaryAnalysisService;
 import com.netflix.spinnaker.orca.pipeline.model.Execution;
 import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository;
 import org.springframework.context.ApplicationEventPublisher;
@@ -47,7 +47,7 @@ public boolean shouldProcessExecution(Execution execution) {
   @Override
   public void processCompletedPipelineExecution(Execution execution) {
     CanaryAnalysisExecutionStatusResponse canaryAnalysisExecution =
-        canaryAnalysisService.getCanaryAnalysisExecution(execution.getId());
+        canaryAnalysisService.getCanaryAnalysisExecution(execution.getId(), null);
     applicationEventPublisher.publishEvent(
         new StandaloneCanaryAnalysisExecutionCompletedEvent(this, canaryAnalysisExecution));
   }

diff --git a/...alonecanaryanalysis/event/listener/StandaloneCanaryAnalysisExecutionArchivalListener.java b/...alonecanaryanalysis/event/listener/StandaloneCanaryAnalysisExecutionArchivalListener.java
@@ -0,0 +1,59 @@
+package com.netflix.kayenta.standalonecanaryanalysis.event.listener;
+
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.kayenta.security.CredentialsHelper;
+import com.netflix.kayenta.standalonecanaryanalysis.event.StandaloneCanaryAnalysisExecutionCompletedEvent;
+import com.netflix.kayenta.storage.ObjectType;
+import com.netflix.kayenta.storage.StorageServiceRepository;
+import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+public class StandaloneCanaryAnalysisExecutionArchivalListener {
+
+  private final AccountCredentialsRepository accountCredentialsRepository;
+  private final StorageServiceRepository storageServiceRepository;
+
+  @Autowired
+  public StandaloneCanaryAnalysisExecutionArchivalListener(
+      AccountCredentialsRepository accountCredentialsRepository,
+      StorageServiceRepository storageServiceRepository) {
+    this.accountCredentialsRepository = accountCredentialsRepository;
+    this.storageServiceRepository = storageServiceRepository;
+  }
+
+  @EventListener
+  public void onApplicationEvent(StandaloneCanaryAnalysisExecutionCompletedEvent event) {
+    val response = event.getCanaryAnalysisExecutionStatusResponse();
+
+    Optional.ofNullable(response.getStorageAccountName())
+        .ifPresent(
+            storageAccountName -> {
+              val resolvedStorageAccountName =
+                  CredentialsHelper.resolveAccountByNameOrType(
+                      storageAccountName,
+                      AccountCredentials.Type.OBJECT_STORE,
+                      accountCredentialsRepository);
+
+              val storageService =
+                  storageServiceRepository
+                      .getOne(resolvedStorageAccountName)
+                      .orElseThrow(
+                          () ->
+                              new IllegalArgumentException(
+                                  "No storage service was configured; unable to archive results."));
+
+              storageService.storeObject(
+                  resolvedStorageAccountName,
+                  ObjectType.STANDALONE_CANARY_RESULT_ARCHIVE,
+                  response.getPipelineId(),
+                  response);
+            });
+  }
+}
diff --git a/...s/domain/MonitorKayentaCanaryContext.java → ...sis/orca/MonitorKayentaCanaryContext.java b/...s/domain/MonitorKayentaCanaryContext.java → ...sis/orca/MonitorKayentaCanaryContext.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.standalonecanaryanalysis.orca;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.netflix.kayenta.canary.CanaryClassifierThresholdsConfig;

diff --git a/...naryanalysis/domain/RunCanaryContext.java → ...canaryanalysis/orca/RunCanaryContext.java b/...naryanalysis/domain/RunCanaryContext.java → ...canaryanalysis/orca/RunCanaryContext.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.standalonecanaryanalysis.orca;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.netflix.kayenta.canary.CanaryClassifierThresholdsConfig;

diff --git a/.../kayenta/canaryanalysis/domain/Stats.java → .../standalonecanaryanalysis/orca/Stats.java b/.../kayenta/canaryanalysis/domain/Stats.java → .../standalonecanaryanalysis/orca/Stats.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.domain;
+package com.netflix.kayenta.standalonecanaryanalysis.orca;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import lombok.AllArgsConstructor;

diff --git a/...ge/GenerateCanaryAnalysisResultStage.java → ...ge/GenerateCanaryAnalysisResultStage.java b/...ge/GenerateCanaryAnalysisResultStage.java → ...ge/GenerateCanaryAnalysisResultStage.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.orca.stage;
+package com.netflix.kayenta.standalonecanaryanalysis.orca.stage;
 
-import com.netflix.kayenta.canaryanalysis.orca.task.GenerateCanaryAnalysisResultTask;
+import com.netflix.kayenta.standalonecanaryanalysis.orca.task.GenerateCanaryAnalysisResultTask;
 import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
 import com.netflix.spinnaker.orca.pipeline.TaskNode;
 import com.netflix.spinnaker.orca.pipeline.model.Stage;

diff --git a/...ryanalysis/orca/stage/RunCanaryStage.java → ...ryanalysis/orca/stage/RunCanaryStage.java b/...ryanalysis/orca/stage/RunCanaryStage.java → ...ryanalysis/orca/stage/RunCanaryStage.java
@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.orca.stage;
+package com.netflix.kayenta.standalonecanaryanalysis.orca.stage;
 
-import com.netflix.kayenta.canaryanalysis.orca.task.MonitorCanaryTask;
-import com.netflix.kayenta.canaryanalysis.orca.task.RunCanaryTask;
+import com.netflix.kayenta.standalonecanaryanalysis.orca.task.MonitorCanaryTask;
+import com.netflix.kayenta.standalonecanaryanalysis.orca.task.RunCanaryTask;
 import com.netflix.spinnaker.orca.CancellableStage;
 import com.netflix.spinnaker.orca.ExecutionStatus;
 import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;

diff --git a/...a/stage/SetupAndExecuteCanariesStage.java → ...a/stage/SetupAndExecuteCanariesStage.java b/...a/stage/SetupAndExecuteCanariesStage.java → ...a/stage/SetupAndExecuteCanariesStage.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package com.netflix.kayenta.canaryanalysis.orca.stage;
+package com.netflix.kayenta.standalonecanaryanalysis.orca.stage;
 
-import static com.netflix.kayenta.canaryanalysis.service.CanaryAnalysisService.CANARY_ANALYSIS_CONFIG_CONTEXT_KEY;
+import static com.netflix.kayenta.standalonecanaryanalysis.service.CanaryAnalysisService.CANARY_ANALYSIS_CONFIG_CONTEXT_KEY;
 import static java.time.Duration.ZERO;
 import static java.time.Instant.now;
 import static java.time.temporal.ChronoUnit.MINUTES;
@@ -25,9 +25,9 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.kayenta.canary.CanaryScope;
 import com.netflix.kayenta.canary.CanaryScopePair;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisConfig;
-import com.netflix.kayenta.canaryanalysis.domain.CanaryAnalysisExecutionRequest;
-import com.netflix.kayenta.canaryanalysis.domain.RunCanaryContext;
+import com.netflix.kayenta.domain.standalonecanaryanalysis.CanaryAnalysisExecutionRequest;
+import com.netflix.kayenta.standalonecanaryanalysis.CanaryAnalysisConfig;
+import com.netflix.kayenta.standalonecanaryanalysis.orca.RunCanaryContext;
 import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
 import com.netflix.spinnaker.orca.pipeline.TaskNode;
 import com.netflix.spinnaker.orca.pipeline.WaitStage;