Skip to content

Commit

Permalink
feat(canary-v2): Support specifying multiple scopes per canary reques…
Browse files Browse the repository at this point in the history 
…t. (#1900)
  • Loading branch information
Matt Duftler committed Jan 22, 2018
1 parent 2f741fc commit 453e237
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 87 deletions.
104 changes: 76 additions & 28 deletions ...nta/src/main/groovy/com/netflix/spinnaker/orca/kayenta/pipeline/KayentaCanaryStage.groovy
View file
Expand Up @@ -50,26 +50,62 @@ class KayentaCanaryStage implements StageDefinitionBuilder {
String metricsAccountName = canaryConfig.metricsAccountName
String storageAccountName = canaryConfig.storageAccountName
String canaryConfigId = canaryConfig.canaryConfigId
String scopeName = canaryConfig.scopeName
String controlScope = canaryConfig.controlScope
String controlRegion = canaryConfig.controlRegion
String experimentScope = canaryConfig.experimentScope
String experimentRegion = canaryConfig.experimentRegion
String startTimeIso = canaryConfig.startTimeIso ?: Instant.now(clock).toString()
List<Map> configScopes = canaryConfig.scopes

if (!configScopes) {
throw new IllegalArgumentException("Canary stage configuration must contain at least one scope.")
}

Map<String, Map> requestScopes = [:]

configScopes.each { configScope ->
// TODO(duftler): Externalize these default values.
String scopeName = configScope.scopeName ?: "default"
String controlScope = configScope.controlScope
String controlRegion = configScope.controlRegion
String experimentScope = configScope.experimentScope
String experimentRegion = configScope.experimentRegion
String startTimeIso = configScope.startTimeIso
String endTimeIso = configScope.endTimeIso
String step = configScope.step ?: "60"
Map<String, String> extendedScopeParams = configScope.extendedScopeParams ?: [:]
Map requestScope = [
controlScope: [
scope: controlScope,
region: controlRegion,
start: startTimeIso,
end: endTimeIso,
step: step,
extendedScopeParams: extendedScopeParams
],
experimentScope: [
scope: experimentScope,
region: experimentRegion,
start: startTimeIso,
end: endTimeIso,
step: step,
extendedScopeParams: extendedScopeParams
]
]

requestScopes[scopeName] = requestScope
}

// Using time boundaries from just the first scope since it doesn't really make sense for each scope to have different boundaries.
// TODO(duftler): Add validation to log warning when time boundaries differ across scopes.
Map<String, Object> firstScope = configScopes[0]
String startTimeIso = firstScope.startTimeIso ?: Instant.now(clock).toString()
Instant startTimeInstant = Instant.parse(startTimeIso)
String endTimeIso = canaryConfig.endTimeIso
String endTimeIso = firstScope.endTimeIso
Instant endTimeInstant
// TODO(duftler): Externalize these default values.
String step = canaryConfig.step ?: "60"
Map<String, String> extendedScopeParams = canaryConfig.get("extendedScopeParams") ?: [:]
Map<String, String> scoreThresholds = canaryConfig.get("scoreThresholds")
String lifetimeHours = canaryConfig.lifetimeHours
long lifetimeMinutes
long beginCanaryAnalysisAfterMins = (canaryConfig.beginCanaryAnalysisAfterMins ?: "0").toLong()
long lookbackMins = (canaryConfig.lookbackMins ?: "0").toLong()

if (endTimeIso) {
endTimeInstant = Instant.parse(canaryConfig.endTimeIso)
endTimeInstant = Instant.parse(firstScope.endTimeIso)
lifetimeMinutes = startTimeInstant.until(endTimeInstant, ChronoUnit.MINUTES)
} else if (lifetimeHours) {
lifetimeMinutes = Duration.ofHours(lifetimeHours.toLong()).toMinutes()
Expand Down Expand Up @@ -108,26 +144,25 @@ class KayentaCanaryStage implements StageDefinitionBuilder {
metricsAccountName: metricsAccountName,
storageAccountName: storageAccountName,
canaryConfigId: canaryConfigId,
scopeName: scopeName,
controlScope: controlScope,
controlRegion: controlRegion,
experimentScope: experimentScope,
experimentRegion: experimentRegion,
step: step,
extendedScopeParams: extendedScopeParams,
scopes: deepCopy(requestScopes),
scoreThresholds: scoreThresholds
]

if (!endTimeIso) {
runCanaryContext.startTimeIso = startTimeInstant.plus(beginCanaryAnalysisAfterMins, ChronoUnit.MINUTES).toString()
runCanaryContext.endTimeIso = startTimeInstant.plus(beginCanaryAnalysisAfterMins + i * canaryAnalysisIntervalMins, ChronoUnit.MINUTES).toString()
} else {
runCanaryContext.startTimeIso = startTimeInstant.toString()
runCanaryContext.endTimeIso = startTimeInstant.plus(i * canaryAnalysisIntervalMins, ChronoUnit.MINUTES).toString()
}

if (lookbackMins) {
runCanaryContext.startTimeIso = Instant.parse(runCanaryContext.endTimeIso).minus(lookbackMins, ChronoUnit.MINUTES).toString()
runCanaryContext.scopes.each { _, contextScope ->
if (!endTimeIso) {
contextScope.controlScope.start = startTimeInstant.plus(beginCanaryAnalysisAfterMins, ChronoUnit.MINUTES).toString()
contextScope.controlScope.end = startTimeInstant.plus(beginCanaryAnalysisAfterMins + i * canaryAnalysisIntervalMins, ChronoUnit.MINUTES).toString()
} else {
contextScope.controlScope.start = startTimeInstant.toString()
contextScope.controlScope.end = startTimeInstant.plus(i * canaryAnalysisIntervalMins, ChronoUnit.MINUTES).toString()
}

if (lookbackMins) {
contextScope.controlScope.start = Instant.parse(contextScope.controlScope.end).minus(lookbackMins, ChronoUnit.MINUTES).toString()
}

contextScope.experimentScope.start = contextScope.controlScope.start
contextScope.experimentScope.end = contextScope.controlScope.end
}

stages << newStage(stage.execution, RunCanaryPipelineStage.STAGE_TYPE, "Run Canary #$i", runCanaryContext, stage, SyntheticStageOwner.STAGE_BEFORE)
Expand All @@ -136,6 +171,19 @@ class KayentaCanaryStage implements StageDefinitionBuilder {
return stages
}

static Object deepCopy(Object sourceObj) {
ByteArrayOutputStream baos = new ByteArrayOutputStream()
ObjectOutputStream oos = new ObjectOutputStream(baos)

oos.writeObject(sourceObj)
oos.flush()

ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray())
ObjectInputStream ois = new ObjectInputStream(bais)

return ois.readObject()
}

@Override
String getType() {
"kayentaCanary"
Expand Down
31 changes: 2 additions & 29 deletions ...enta/src/main/groovy/com/netflix/spinnaker/orca/kayenta/tasks/RunKayentaCanaryTask.groovy
View file
Expand Up @@ -40,37 +40,10 @@ class RunKayentaCanaryTask implements Task {
String metricsAccountName = (String)context.get("metricsAccountName")
String storageAccountName = (String)context.get("storageAccountName")
String canaryConfigId = (String)context.get("canaryConfigId")
String scopeName = (String)context.get("scopeName") ?: "default"
String controlScope = (String)context.get("controlScope")
String controlRegion = (String)context.get("controlRegion")
String experimentScope = (String)context.get("experimentScope")
String experimentRegion = (String)context.get("experimentRegion")
String startTimeIso = (String)context.get("startTimeIso")
String endTimeIso = (String)context.get("endTimeIso")
String step = (String)context.get("step")
Map<String, String> extendedScopeParams = (Map<String, String>)context.get("extendedScopeParams")
Map<String, Map> scopes = (Map<String, Map>)context.get("scopes")
Map<String, String> scoreThresholds = (Map<String, String>)context.get("scoreThresholds")
Map<String, String> canaryExecutionRequest = [
scopes: [
(scopeName): [
controlScope: [
scope: controlScope,
region: controlRegion,
start: startTimeIso,
end: endTimeIso,
step: step,
extendedScopeParams: extendedScopeParams
],
experimentScope: [
scope: experimentScope,
region: experimentRegion,
start: startTimeIso,
end: endTimeIso,
step: step,
extendedScopeParams: extendedScopeParams
]
]
],
scopes: scopes,
thresholds: [
pass: scoreThresholds?.pass,
marginal: scoreThresholds?.marginal
Expand Down
123 changes: 93 additions & 30 deletions ...src/test/groovy/com/netflix/spinnaker/orca/kayenta/pipeline/KayentaCanaryStageSpec.groovy
View file
Expand Up @@ -16,16 +16,18 @@

package com.netflix.spinnaker.orca.kayenta.pipeline

import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.time.ZoneId
import java.time.temporal.ChronoUnit
import com.netflix.spinnaker.orca.pipeline.WaitStage
import com.netflix.spinnaker.orca.pipeline.model.Stage
import spock.lang.Shared
import spock.lang.Specification
import spock.lang.Unroll

import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.time.ZoneId
import java.time.temporal.ChronoUnit

import static com.netflix.spinnaker.orca.test.model.ExecutionBuilder.stage

class KayentaCanaryStageSpec extends Specification {
Expand All @@ -42,10 +44,12 @@ class KayentaCanaryStageSpec extends Specification {
context = [
canaryConfig: [
canaryConfigId : "MySampleStackdriverCanaryConfig",
controlScope : "myapp-v010-",
experimentScope : "myapp-v021-",
startTimeIso : "2017-01-01T01:02:34.567Z",
endTimeIso : "2017-01-01T05:02:34.567Z",
scopes : [[
controlScope : "myapp-v010",
experimentScope: "myapp-v021",
startTimeIso : "2017-01-01T01:02:34.567Z",
endTimeIso : "2017-01-01T05:02:34.567Z",
]],
beginCanaryAnalysisAfterMins: beginCanaryAnalysisAfterMins
]
]
Expand Down Expand Up @@ -77,10 +81,12 @@ class KayentaCanaryStageSpec extends Specification {
context = [
canaryConfig: [
canaryConfigId : "MySampleStackdriverCanaryConfig",
controlScope : "myapp-v010-",
experimentScope : "myapp-v021-",
startTimeIso : "2017-01-01T01:02:34.567Z",
endTimeIso : "2017-01-01T05:02:34.567Z",
scopes : [[
controlScope : "myapp-v010",
experimentScope: "myapp-v021",
startTimeIso : "2017-01-01T01:02:34.567Z",
endTimeIso : "2017-01-01T05:02:34.567Z"
]],
beginCanaryAnalysisAfterMins: beginCanaryAnalysisAfterMins,
canaryAnalysisIntervalMins : canaryAnalysisIntervalMins,
lookbackMins : lookbackMins
Expand Down Expand Up @@ -139,6 +145,57 @@ class KayentaCanaryStageSpec extends Specification {
[minutesFromInitialStartToCanaryStart: 180, minutesFromInitialStartToCanaryEnd: 240, step: "60"]]
}

@Unroll
def "should use the first scope's time boundaries for all scopes"() {
given:
def kayentaCanaryStage = stage {
type = "kayentaCanary"
name = "Run Kayenta Canary"
context = [
canaryConfig: [
canaryConfigId: "MySampleStackdriverCanaryConfig",
scopes : [
[
scopeName : "default",
controlScope : "myapp-v010",
experimentScope: "myapp-v021",
startTimeIso : "2017-01-01T01:02:34.567Z",
endTimeIso : "2017-01-01T05:02:34.567Z"
],
[
scopeName : "otherScope",
controlScope : "myapp-v016",
experimentScope: "myapp-v028",
startTimeIso : "2017-02-03T04:02:34.567Z",
endTimeIso : "2017-02-03T08:02:34.567Z"
],
[
scopeName : "yetAnotherScope",
controlScope : "myapp-v023",
experimentScope: "myapp-v025",
startTimeIso : "2017-03-04T06:02:34.567Z",
endTimeIso : "2017-03-04T10:02:34.567Z"
]
]
]
]
}
def builder = new KayentaCanaryStage()
def startTimeInstant = Instant.parse("2017-01-01T01:02:34.567Z")
builder.clock = Clock.fixed(startTimeInstant, ZoneId.systemDefault())
builder.waitStage = waitStage
def aroundStages = builder.aroundStages(kayentaCanaryStage)

when:
def summary = collectSummary(aroundStages, startTimeInstant, scopeName)

then:
summary == [[minutesFromInitialStartToCanaryStart: 0, minutesFromInitialStartToCanaryEnd: 240, step: "60"]]

where:
scopeName << ["default", "otherScope", "yetAnotherScope"]
}

@Unroll
def "should start now and include warmupWait stage if necessary"() {
given:
Expand All @@ -148,8 +205,10 @@ class KayentaCanaryStageSpec extends Specification {
context = [
canaryConfig: [
canaryConfigId : "MySampleStackdriverCanaryConfig",
controlScope : "myapp-v010-",
experimentScope : "myapp-v021-",
scopes : [[
controlScope : "myapp-v010",
experimentScope: "myapp-v021"
]],
lifetimeHours : "1",
beginCanaryAnalysisAfterMins: beginCanaryAnalysisAfterMins
]
Expand All @@ -168,7 +227,7 @@ class KayentaCanaryStageSpec extends Specification {
!warmupWaitPeriodMinutes || aroundStages[0].context.waitTime == Duration.ofMinutes(warmupWaitPeriodMinutes).getSeconds()
aroundStages.find {
it.type == "runCanary"
}.context.startTimeIso == startTimeInstant.plus(warmupWaitPeriodMinutes, ChronoUnit.MINUTES).toString()
}.context.scopes.default.controlScope.start == startTimeInstant.plus(warmupWaitPeriodMinutes, ChronoUnit.MINUTES).toString()

where:
beginCanaryAnalysisAfterMins || expectedStageTypes | warmupWaitPeriodMinutes
Expand All @@ -187,8 +246,10 @@ class KayentaCanaryStageSpec extends Specification {
context = [
canaryConfig: [
canaryConfigId : "MySampleStackdriverCanaryConfig",
controlScope : "myapp-v010-",
experimentScope : "myapp-v021-",
scopes : [[
controlScope : "myapp-v010",
experimentScope: "myapp-v021"
]],
beginCanaryAnalysisAfterMins: beginCanaryAnalysisAfterMins,
canaryAnalysisIntervalMins : canaryAnalysisIntervalMins,
lookbackMins : lookbackMins,
Expand Down Expand Up @@ -301,12 +362,14 @@ class KayentaCanaryStageSpec extends Specification {
canaryConfig: [
metricsAccountName : "atlas-acct-1",
canaryConfigId : "MySampleAtlasCanaryConfig",
controlScope : "some.host.node",
experimentScope : "some.other.host.node",
scopes : [[
controlScope : "some.host.node",
experimentScope : "some.other.host.node",
step : "PT60S",
extendedScopeParams: [type: "node"]
]],
canaryAnalysisIntervalMins: Duration.ofHours(6).toMinutes(),
lifetimeHours : "12",
step : "PT60S",
extendedScopeParams : [type: "node"]
lifetimeHours : "12"
]
]
}
Expand All @@ -326,13 +389,13 @@ class KayentaCanaryStageSpec extends Specification {
[minutesFromInitialStartToCanaryStart: 0, minutesFromInitialStartToCanaryEnd: Duration.ofHours(12).toMinutes(), step: "PT60S", metricsAccountName: "atlas-acct-1", extendedScopeParams: [type: "node"]]]
}

def collectSummary(List<Stage> aroundStages, startTimeInstant) {
def collectSummary(List<Stage> aroundStages, Instant startTimeInstant, String scopeName = "default") {
return aroundStages.collect {
if (it.type == waitStage.type) {
return [wait: it.context.waitTime]
} else if (it.type == RunCanaryPipelineStage.STAGE_TYPE) {
Instant runCanaryPipelineStartInstant = Instant.parse(it.context.startTimeIso)
Instant runCanaryPipelineEndInstant = Instant.parse(it.context.endTimeIso)
Instant runCanaryPipelineStartInstant = Instant.parse(it.context.scopes[scopeName].controlScope.start)
Instant runCanaryPipelineEndInstant = Instant.parse(it.context.scopes[scopeName].controlScope.end)
Map ret = [
minutesFromInitialStartToCanaryStart: startTimeInstant.until(runCanaryPipelineStartInstant, ChronoUnit.MINUTES),
minutesFromInitialStartToCanaryEnd : startTimeInstant.until(runCanaryPipelineEndInstant, ChronoUnit.MINUTES)
Expand All @@ -342,12 +405,12 @@ class KayentaCanaryStageSpec extends Specification {
ret.metricsAccountName = it.context.metricsAccountName
}

if (it.context.step) {
ret.step = it.context.step
if (it.context.scopes[scopeName].controlScope.step) {
ret.step = it.context.scopes[scopeName].controlScope.step
}

if (it.context.extendedScopeParams) {
ret.extendedScopeParams = it.context.extendedScopeParams
if (it.context.scopes[scopeName].controlScope.extendedScopeParams) {
ret.extendedScopeParams = it.context.scopes[scopeName].controlScope.extendedScopeParams
}

return ret
Expand Down

0 comments on commit 453e237

Please sign in to comment.