Support multiple scopes per metric. (#173)

kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryExecutionRequest.java

@@ -18,9 +18,14 @@
 import lombok.Data;
 
 import javax.validation.constraints.NotNull;
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 @Data
 public class CanaryExecutionRequest {
+  protected Map<String, CanaryScopePair> scopes;
 
   @NotNull
   protected CanaryScope experimentScope;
@@ -29,4 +34,27 @@ public class CanaryExecutionRequest {
   protected CanaryScope controlScope;
 
   protected CanaryClassifierThresholdsConfig thresholds;
+
+  public Duration calculateDuration() {
+    Set<Duration> durationsFound = new HashSet<>();
+
+    if (experimentScope != null) {
+      durationsFound.add(experimentScope.calculateDuration());
+    }
+    if (controlScope != null) {
+      durationsFound.add(controlScope.calculateDuration());
+    }
+    if (scopes != null) {
+      scopes.values().forEach(scope -> {
+        durationsFound.add(scope.controlScope.calculateDuration());
+        durationsFound.add(scope.experimentScope.calculateDuration());
+      });
+    }
+    if (durationsFound.size() == 1) {
+      return durationsFound.stream()
+        .findFirst()
+        .orElse(null);
+    }
+    return null;  // cannot find a single duration to represent this data
+  }
 }

kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryMetricConfig.java

@@ -47,4 +47,7 @@ public class CanaryMetricConfig {
   @Singular
   @Getter
   private Map<String, Map> analysisConfigurations;
+
+  @Getter
+  private String scopeName;
 }

kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryScope.java

@@ -20,6 +20,7 @@
 import lombok.NoArgsConstructor;
 
 import javax.validation.constraints.NotNull;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Map;
 
@@ -46,4 +47,8 @@ public class CanaryScope {
   // Metric source specific parameters which may be used to further
   // alter the canary scope.
   Map<String, String> extendedScopeParams;
+
+  public Duration calculateDuration() {
+    return Duration.between(start, end);
+  }
 }

kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryScopePair.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.kayenta.canary;
+
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+public class CanaryScopePair {
+  @NotNull
+  CanaryScope controlScope;
+
+  @NotNull
+  CanaryScope experimentScope;
+}

kayenta-core/src/main/java/com/netflix/kayenta/canary/orca/CanaryJudgeTask.java

@@ -129,6 +129,7 @@ public TaskResult execute(@Nonnull Stage stage) {
       .judgeResult(result)
       .config(canaryConfig)
       .canaryExecutionRequest(canaryExecutionRequest)
+      .canaryDuration(canaryExecutionRequest.calculateDuration())
       .metricSetPairListId(metricSetPairListId)
       .pipelineId(stage.getExecution().getId())
       .build();

kayenta-core/src/main/java/com/netflix/kayenta/canary/results/CanaryResult.java

@@ -21,6 +21,8 @@
 import com.netflix.kayenta.canary.CanaryExecutionRequest;
 import lombok.*;
 
+import java.time.Duration;
+
 @Builder
 @ToString
 @NoArgsConstructor
@@ -42,6 +44,9 @@ public class CanaryResult {
   @Getter
   String metricSetPairListId;
 
+  @Getter
+  Duration canaryDuration;
+
   @NonNull
   @Getter
   String pipelineId;

kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java

@@ -121,6 +121,20 @@ public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) fi
 
     registry.counter(pipelineRunId.withTag("canaryConfigId", canaryConfigId).withTag("canaryConfigName", canaryConfig.getName())).increment();
 
+    Set<String> requiredScopes = canaryConfig.getMetrics().stream()
+      .map(CanaryMetricConfig::getScopeName)
+      .filter(Objects::nonNull)
+      .collect(Collectors.toSet());
+
+    if (requiredScopes.size() > 0 && canaryExecutionRequest.getScopes() == null) {
+      throw new IllegalArgumentException("Canary metrics require scopes, but no scopes were provided in the execution request.");
+    }
+    Set<String> providedScopes = canaryExecutionRequest.getScopes() == null ? Collections.emptySet() : canaryExecutionRequest.getScopes().keySet();
+    requiredScopes.removeAll(providedScopes);
+    if (requiredScopes.size() > 0) {
+      throw new IllegalArgumentException("Canary metrics require scopes which were not provided in the execution request: " + requiredScopes);
+    }
+
     Map<String, Object> setupCanaryContext =
       Maps.newHashMap(
         new ImmutableMap.Builder<String, Object>()
@@ -151,6 +165,9 @@ public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) fi
     CanaryClassifierThresholdsConfig orchestratorScoreThresholds = canaryExecutionRequest.getThresholds();
 
     if (orchestratorScoreThresholds == null) {
+      if (canaryConfig.getClassifier() == null || canaryConfig.getClassifier().getScoreThresholds() == null) {
+        throw new IllegalArgumentException("Classifier thresholds must be specified in either the canary config, or the execution request.");
+      }
       // The score thresholds were not explicitly passed in from the orchestrator (i.e. Spinnaker), so just use the canary config values.
       orchestratorScoreThresholds = canaryConfig.getClassifier().getScoreThresholds();
     }
@@ -174,7 +191,6 @@ public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) fi
         .withName("Standard Canary Pipeline")
         .withPipelineConfigId(UUID.randomUUID() + "")
         .withStage("setupCanary", "Setup Canary", setupCanaryContext)
-        .withStage("setupCanary", "Setup Canary", setupCanaryContext)
         .withStage("metricSetMixer", "Mix Control and Experiment Results", mixMetricSetsContext)
         .withStage("canaryJudge", "Perform Analysis", canaryJudgeContext);
 
@@ -311,6 +327,19 @@ private CanaryScopeFactory getScopeFactoryForServiceType(String serviceType) {
       .orElseThrow(() -> new IllegalArgumentException("Unable to resolve canary scope factory for '" + serviceType + "'."));
   }
 
+  private CanaryScope getScopeForNamedScope(CanaryExecutionRequest executionRequest, String scopeName, boolean isCanary) {
+    if (scopeName == null) {
+      return isCanary ? executionRequest.getExperimentScope() : executionRequest.getControlScope();
+    }
+
+    CanaryScopePair canaryScopePair = executionRequest.getScopes().get(scopeName);
+    CanaryScope canaryScope = isCanary ? canaryScopePair.getExperimentScope() : canaryScopePair.getControlScope();
+    if (canaryScope == null) {
+      throw new IllegalArgumentException("Canary scope for named scope " + scopeName + " is missing experimentScope or controlScope keys");
+    }
+    return canaryScope;
+  }
+
   private List<Map<String, Object>> generateFetchScopes(CanaryConfig canaryConfig,
                                                         CanaryExecutionRequest executionRequest,
                                                         boolean isCanary,
@@ -321,7 +350,7 @@ private List<Map<String, Object>> generateFetchScopes(CanaryConfig canaryConfig,
         CanaryMetricConfig metric = canaryConfig.getMetrics().get(index);
         String serviceType = metric.getQuery().getServiceType();
         CanaryScopeFactory canaryScopeFactory = getScopeFactoryForServiceType(serviceType);
-        CanaryScope inspecificScope = (isCanary ? executionRequest.getExperimentScope() : executionRequest.getControlScope());
+        CanaryScope inspecificScope = getScopeForNamedScope(executionRequest, metric.getScopeName(), isCanary);
         CanaryScope scopeModel = canaryScopeFactory.buildCanaryScope(inspecificScope);
         String stagePrefix = (isCanary ? REFID_FETCH_EXPERIMENT_PREFIX : REFID_FETCH_CONTROL_PREFIX);
         String scopeJson;