From 7cca4c242bb7766658f7009e5d9ecc6c86d7890c Mon Sep 17 00:00:00 2001 From: Matt Duftler Date: Thu, 1 Feb 2018 14:17:23 -0500 Subject: [PATCH] feat(executions): Add support for retrieving list of canary execution results by application. (#221) --- .../canary/CanaryExecutionStatusResponse.java | 5 + .../kayenta/canary/orca/CanaryJudgeTask.java | 4 + .../kayenta/canary/results/CanaryResult.java | 7 + .../metrics/StackdriverMetricsService.java | 1 + .../kayenta/controllers/CanaryController.java | 138 ++++++++++++++---- 5 files changed, 124 insertions(+), 31 deletions(-) diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryExecutionStatusResponse.java b/kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryExecutionStatusResponse.java index ccbec2190..2f2a6c56a 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryExecutionStatusResponse.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/canary/CanaryExecutionStatusResponse.java @@ -26,6 +26,11 @@ @Builder public class CanaryExecutionStatusResponse { + // These are here (in addition to CanaryResult) so we can still correlate runs while the canary execution is in-flight, + // or in the case where it never reaches the judging stage. + protected String application; + protected String parentPipelineExecutionId; + @NotNull protected Map stageStatus; diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/canary/orca/CanaryJudgeTask.java b/kayenta-core/src/main/java/com/netflix/kayenta/canary/orca/CanaryJudgeTask.java index a1d0ec322..ea9763b57 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/canary/orca/CanaryJudgeTask.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/canary/orca/CanaryJudgeTask.java @@ -114,6 +114,8 @@ public TaskResult execute(@Nonnull Stage stage) { canaryJudge = canaryJudges.get(0); } + String application = (String)context.get("application"); + String parentPipelineExecutionId = (String)context.get("parentPipelineExecutionId"); String canaryExecutionRequestJSON = (String)context.get("canaryExecutionRequest"); CanaryExecutionRequest canaryExecutionRequest = null; try { @@ -127,6 +129,8 @@ public TaskResult execute(@Nonnull Stage stage) { String canaryJudgeResultId = UUID.randomUUID() + ""; CanaryResult canaryResult = CanaryResult.builder() + .application(application) + .parentPipelineExecutionId(parentPipelineExecutionId) .judgeResult(result) .config(canaryConfig) .canaryExecutionRequest(canaryExecutionRequest) diff --git a/kayenta-core/src/main/java/com/netflix/kayenta/canary/results/CanaryResult.java b/kayenta-core/src/main/java/com/netflix/kayenta/canary/results/CanaryResult.java index e06e78a0f..ec2f5d208 100644 --- a/kayenta-core/src/main/java/com/netflix/kayenta/canary/results/CanaryResult.java +++ b/kayenta-core/src/main/java/com/netflix/kayenta/canary/results/CanaryResult.java @@ -30,6 +30,13 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class CanaryResult { + // These are here (in addition to CanaryExecutionStatusResponse) so we can still correlate runs even if we lose + // the persisted execution. + @Getter + String application; + @Getter + String parentPipelineExecutionId; + @Getter CanaryJudgeResult judgeResult; diff --git a/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java b/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java index 2d39f7686..2cadfdf8b 100644 --- a/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java +++ b/kayenta-stackdriver/src/main/java/com/netflix/kayenta/stackdriver/metrics/StackdriverMetricsService.java @@ -84,6 +84,7 @@ public class StackdriverMetricsService implements MetricsService { @Autowired private final StackdriverConfigurationProperties stackdriverConfigurationProperties; + @Builder.Default private List metricDescriptorsCache = Collections.emptyList(); @Override diff --git a/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java b/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java index b9b691bf4..555373a9e 100644 --- a/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java +++ b/kayenta-web/src/main/java/com/netflix/kayenta/controllers/CanaryController.java @@ -39,6 +39,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -47,6 +48,7 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; @RestController @RequestMapping("/canary") @@ -58,6 +60,7 @@ public class CanaryController { private final String REFID_FETCH_EXPERIMENT_PREFIX = "fetchExperiment"; private final String REFID_MIX_METRICS = "mixMetrics"; private final String REFID_JUDGE = "judge"; + private final String AD_HOC = "ad-hoc"; private final String currentInstanceId; private final ExecutionLauncher executionLauncher; @@ -99,7 +102,9 @@ public CanaryController(String currentInstanceId, // TODO(duftler): Allow for user to be passed in. @ApiOperation(value = "Initiate a canary pipeline") @RequestMapping(value = "/{canaryConfigId:.+}", consumes = "application/json", method = RequestMethod.POST) - public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) final String metricsAccountName, + public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) final String application, + @RequestParam(required = false) final String parentPipelineExecutionId, + @RequestParam(required = false) final String metricsAccountName, @RequestParam(required = false) final String configurationAccountName, @RequestParam(required = false) final String storageAccountName, @ApiParam @RequestBody final CanaryExecutionRequest canaryExecutionRequest, @@ -120,7 +125,9 @@ public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) fi .orElseThrow(() -> new IllegalArgumentException("No configuration service was configured.")); CanaryConfig canaryConfig = configurationService.loadObject(resolvedConfigurationAccountName, ObjectType.CANARY_CONFIG, canaryConfigId); - return buildExecution(canaryConfigId, + return buildExecution(application, + parentPipelineExecutionId, + canaryConfigId, canaryConfig, resolvedConfigurationAccountName, resolvedMetricsAccountName, @@ -134,10 +141,9 @@ public CanaryExecutionResponse initiateCanary(@RequestParam(required = false) fi // TODO(duftler): Allow for user to be passed in. @ApiOperation(value = "Initiate a canary pipeline with CanaryConfig provided") @RequestMapping(consumes = "application/json", method = RequestMethod.POST) - public CanaryExecutionResponse initiateCanaryWithConfig( - @RequestParam(required = false) final String metricsAccountName, - @RequestParam(required = false) final String storageAccountName, - @ApiParam @RequestBody final CanaryAdhocExecutionRequest canaryAdhocExecutionRequest) throws JsonProcessingException { + public CanaryExecutionResponse initiateCanaryWithConfig(@RequestParam(required = false) final String metricsAccountName, + @RequestParam(required = false) final String storageAccountName, + @ApiParam @RequestBody final CanaryAdhocExecutionRequest canaryAdhocExecutionRequest) throws JsonProcessingException { String resolvedMetricsAccountName = CredentialsHelper.resolveAccountByNameOrType(metricsAccountName, AccountCredentials.Type.METRICS_STORE, @@ -153,7 +159,9 @@ public CanaryExecutionResponse initiateCanaryWithConfig( throw new IllegalArgumentException("executionRequest must be provided for ad-hoc requests"); } - return buildExecution("ad-hoc", + return buildExecution(AD_HOC, + AD_HOC, + AD_HOC, canaryAdhocExecutionRequest.getCanaryConfig(), null, resolvedMetricsAccountName, @@ -167,17 +175,23 @@ public CanaryExecutionResponse initiateCanaryWithConfig( @ApiOperation(value = "Retrieve status and results for a canary run") @RequestMapping(value = "/{canaryExecutionId:.+}", method = RequestMethod.GET) public CanaryExecutionStatusResponse getCanaryResults(@RequestParam(required = false) final String storageAccountName, - @PathVariable String canaryExecutionId) throws JsonProcessingException { + @PathVariable String canaryExecutionId) { String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName, AccountCredentials.Type.OBJECT_STORE, accountCredentialsRepository); + Execution pipeline = executionRepository.retrieve(Execution.ExecutionType.PIPELINE, canaryExecutionId); + + return getCanaryResults(resolvedStorageAccountName, pipeline); + } + private CanaryExecutionStatusResponse getCanaryResults(String storageAccountName, Execution pipeline) { StorageService storageService = storageServiceRepository - .getOne(resolvedStorageAccountName) + .getOne(storageAccountName) .orElseThrow(() -> new IllegalArgumentException("No storage service was configured; unable to retrieve results.")); - Execution pipeline = executionRepository.retrieve(Execution.ExecutionType.PIPELINE, canaryExecutionId); + String canaryExecutionId = pipeline.getId(); + Stage judgeStage = pipeline.getStages().stream() .filter(stage -> stage.getRefId().equals(REFID_JUDGE)) .findFirst() @@ -196,7 +210,10 @@ public CanaryExecutionStatusResponse getCanaryResults(@RequestParam(required = f .orElseThrow(() -> new IllegalArgumentException("Unable to find stage '" + REFID_MIX_METRICS + "' in pipeline ID '" + canaryExecutionId + "'")); Map mixerContext = mixerStage.getContext(); - CanaryExecutionStatusResponse.CanaryExecutionStatusResponseBuilder canaryExecutionStatusResponseBuilder = CanaryExecutionStatusResponse.builder(); + CanaryExecutionStatusResponse.CanaryExecutionStatusResponseBuilder canaryExecutionStatusResponseBuilder = + CanaryExecutionStatusResponse.builder() + .application((String)contextContext.get("application")) + .parentPipelineExecutionId((String)contextContext.get("parentPipelineExecutionId")); Map stageStatus = pipeline.getStages() .stream() @@ -204,32 +221,37 @@ public CanaryExecutionStatusResponse getCanaryResults(@RequestParam(required = f Boolean isComplete = pipeline.getStatus().isComplete(); String pipelineStatus = pipeline.getStatus().toString().toLowerCase(); - canaryExecutionStatusResponseBuilder.stageStatus(stageStatus); - canaryExecutionStatusResponseBuilder.complete(isComplete); - canaryExecutionStatusResponseBuilder.status(pipelineStatus); + + canaryExecutionStatusResponseBuilder + .stageStatus(stageStatus) + .complete(isComplete) + .status(pipelineStatus); Long buildTime = pipeline.getBuildTime(); if (buildTime != null) { - canaryExecutionStatusResponseBuilder.buildTimeMillis(buildTime); - canaryExecutionStatusResponseBuilder.buildTimeIso(Instant.ofEpochMilli(buildTime) + ""); + canaryExecutionStatusResponseBuilder + .buildTimeMillis(buildTime) + .buildTimeIso(Instant.ofEpochMilli(buildTime) + ""); } Long startTime = pipeline.getStartTime(); if (startTime != null) { - canaryExecutionStatusResponseBuilder.startTimeMillis(startTime); - canaryExecutionStatusResponseBuilder.startTimeIso(Instant.ofEpochMilli(startTime) + ""); + canaryExecutionStatusResponseBuilder + .startTimeMillis(startTime) + .startTimeIso(Instant.ofEpochMilli(startTime) + ""); } Long endTime = pipeline.getEndTime(); if (endTime != null) { - canaryExecutionStatusResponseBuilder.endTimeMillis(endTime); - canaryExecutionStatusResponseBuilder.endTimeIso(Instant.ofEpochMilli(endTime) + ""); + canaryExecutionStatusResponseBuilder + .endTimeMillis(endTime) + .endTimeIso(Instant.ofEpochMilli(endTime) + ""); } if (isComplete && pipelineStatus.equals("succeeded")) { if (judgeOutputs.containsKey("canaryJudgeResultId")) { String canaryJudgeResultId = (String)judgeOutputs.get("canaryJudgeResultId"); - canaryExecutionStatusResponseBuilder.result(storageService.loadObject(resolvedStorageAccountName, ObjectType.CANARY_RESULT, canaryJudgeResultId)); + canaryExecutionStatusResponseBuilder.result(storageService.loadObject(storageAccountName, ObjectType.CANARY_RESULT, canaryJudgeResultId)); } } @@ -246,7 +268,9 @@ public CanaryExecutionStatusResponse getCanaryResults(@RequestParam(required = f return canaryExecutionStatusResponseBuilder.build(); } - private CanaryExecutionResponse buildExecution(@NotNull String canaryConfigId, + private CanaryExecutionResponse buildExecution(String application, + String parentPipelineExecutionId, + @NotNull String canaryConfigId, @NotNull CanaryConfig canaryConfig, String resolvedConfigurationAccountName, @NotNull String resolvedMetricsAccountName, @@ -268,17 +292,28 @@ private CanaryExecutionResponse buildExecution(@NotNull String canaryConfigId, throw new IllegalArgumentException("Canary metrics require scopes which were not provided in the execution request: " + requiredScopes); } + // TODO: Will non-spinnaker users need to know what application to pass (probably should pass something, or else how will they group their runs)? + if (StringUtils.isEmpty(application)) { + application = "kayenta-" + currentInstanceId; + } + + if (StringUtils.isEmpty(parentPipelineExecutionId)) { + parentPipelineExecutionId = "no-parent-pipeline-execution"; + } + HashMap setupCanaryContext = Maps.newHashMap( new ImmutableMap.Builder() .put("refId", REFID_SET_CONTEXT) .put("user", "[anonymous]") + .put("application", application) + .put("parentPipelineExecutionId", parentPipelineExecutionId) .put("canaryConfigId", canaryConfigId) .build()); if (resolvedConfigurationAccountName != null) { setupCanaryContext.put("configurationAccountName", resolvedConfigurationAccountName); } - if (canaryConfigId.equalsIgnoreCase("adhoc")) { + if (canaryConfigId.equalsIgnoreCase(AD_HOC)) { setupCanaryContext.put("canaryConfig", canaryConfig); } @@ -329,25 +364,28 @@ private CanaryExecutionResponse buildExecution(@NotNull String canaryConfigId, .put("storageAccountName", resolvedStorageAccountName) .put("metricSetPairListId", "${ #stage('Mix Control and Experiment Results')['context']['metricSetPairListId']}") .put("orchestratorScoreThresholds", orchestratorScoreThresholds) + .put("application", application) + .put("parentPipelineExecutionId", parentPipelineExecutionId) .put("canaryExecutionRequest", canaryExecutionRequestJSON) .build()); + String canaryPipelineConfigId = application + "-standard-canary-pipeline"; PipelineBuilder pipelineBuilder = - new PipelineBuilder("kayenta-" + currentInstanceId) + new PipelineBuilder(application) .withName("Standard Canary Pipeline") - .withPipelineConfigId(UUID.randomUUID() + "") + .withPipelineConfigId(canaryPipelineConfigId) .withStage("setupCanary", "Setup Canary", setupCanaryContext) .withStage("metricSetMixer", "Mix Control and Experiment Results", mixMetricSetsContext) .withStage("canaryJudge", "Perform Analysis", canaryJudgeContext); controlFetchContexts.forEach((context) -> - pipelineBuilder.withStage((String)context.get("stageType"), - (String)context.get("refId"), - context)); + pipelineBuilder.withStage((String)context.get("stageType"), + (String)context.get("refId"), + context)); fetchExperimentContexts.forEach((context) -> - pipelineBuilder.withStage((String)context.get("stageType"), - (String)context.get("refId"), - context)); + pipelineBuilder.withStage((String)context.get("stageType"), + (String)context.get("refId"), + context)); Execution pipeline = pipelineBuilder .withLimitConcurrent(false) @@ -434,4 +472,42 @@ private List> generateFetchScopes(CanaryConfig canaryConfig, .build()); }).collect(Collectors.toList()); } + + @ApiOperation(value = "Retrieve a list of an application's canary results") + @RequestMapping(value = "/executions", method = RequestMethod.GET) + List getCanaryResultsByApplication(@RequestParam(required = false) String application, + @RequestParam(value = "limit", defaultValue = "20") int limit, + @RequestParam(value = "statuses", required = false) String statuses, + @RequestParam(required = false) final String storageAccountName) { + String resolvedStorageAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName, + AccountCredentials.Type.OBJECT_STORE, + accountCredentialsRepository); + + if (StringUtils.isEmpty(statuses)) { + statuses = Stream.of(ExecutionStatus.values()) + .map(s -> s.toString()) + .collect(Collectors.joining(",")); + } + + List statusesList = Stream.of(statuses.split(",")) + .map(s -> s.trim()) + .filter(s -> !StringUtils.isEmpty(s)) + .collect(Collectors.toList()); + ExecutionRepository.ExecutionCriteria executionCriteria = new ExecutionRepository.ExecutionCriteria() + .setLimit(limit) + .setStatuses(statusesList); + + // Users of the ad-hoc endpoint can either omit application or pass 'ad-hoc' explicitly. + if (StringUtils.isEmpty(application)) { + application = AD_HOC; + } + + String canaryPipelineConfigId = application + "-standard-canary-pipeline"; + List executions = executionRepository.retrievePipelinesForPipelineConfigId(canaryPipelineConfigId, executionCriteria).toList().toBlocking().single(); + + return executions + .stream() + .map(execution -> getCanaryResults(resolvedStorageAccountName, execution)) + .collect(Collectors.toList()); + } }