Skip to content

Commit

Permalink
feat(cfn/changesets): introduce support CFN change sets (#2950)
Browse files Browse the repository at this point in the history 
* feat(cfn/changesets): introduce support CFN change sets

This patch adds supports in ORCA to check the status of a CFN change set
if instructed to do so. There will be a new field in the
DeployCloudFormationDescription specifying wether the deploy corresponds
to a Stack or to a ChangeSet.

The main changes are:
- Waiting for CFN completion now discerns between the status of a
regular Stack or a Change Set.
- ORCA instructs to force refresh the cache of a particular Stack being
updated, making the refresh operation much more efficient (clouddriver
won't have to iterate over all stacks on a given account/region).

* fixup: consider empty changesets as succeeded
  • Loading branch information
@xavileon @maggieneterval
xavileon authored and maggieneterval committed Jun 27, 2019
1 parent e29224c commit 60ed2ad
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 25 deletions.
5 changes: 5 additions & 0 deletions ...a/clouddriver/tasks/providers/aws/cloudformation/CloudFormationForceCacheRefreshTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ public TaskResult execute(@Nonnull Stage stage) {
data.put("region", regions);
}

String stackName = (String) stage.getContext().get("stackName");
if (stackName != null) {
data.put("stackName", stackName);
}

cacheService.forceCacheUpdate(cloudProvider, REFRESH_TYPE, data);

return TaskResult.SUCCEEDED;
Expand Down
69 changes: 59 additions & 10 deletions ...a/clouddriver/tasks/providers/aws/cloudformation/WaitForCloudFormationCompletionTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
import com.netflix.spinnaker.orca.TaskResult;
import com.netflix.spinnaker.orca.clouddriver.OortService;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -36,6 +38,15 @@ public class WaitForCloudFormationCompletionTask implements OverridableTimeoutRe

public static final String TASK_NAME = "waitForCloudFormationCompletion";

private enum CloudFormationStates {
NOT_YET_READY,
CREATE_COMPLETE,
UPDATE_COMPLETE,
IN_PROGRESS,
ROLLBACK_COMPLETE,
FAILED
}

private final long backoffPeriod = TimeUnit.SECONDS.toMillis(10);
private final long timeout = TimeUnit.HOURS.toMillis(2);

Expand All @@ -54,14 +65,24 @@ public TaskResult execute(@Nonnull Stage stage) {
+ stackId
+ " with status "
+ stack.get("stackStatus"));
if (isComplete(stack.get("stackStatus"))) {
boolean isChangeSet =
(boolean) Optional.ofNullable(stage.getContext().get("isChangeSet")).orElse(false);
log.info("Deploying a CloudFormation ChangeSet for stackId " + stackId + ": " + isChangeSet);
String status =
isChangeSet
? getChangeSetInfo(stack, stage.getContext(), "status")
: getStackInfo(stack, "stackStatus");
if (isComplete(status) || isEmptyChangeSet(stage, stack)) {
return TaskResult.builder(ExecutionStatus.SUCCEEDED).outputs(stack).build();
} else if (isInProgress(stack.get("stackStatus"))) {
} else if (isInProgress(status)) {
return TaskResult.RUNNING;
} else if (isFailed(stack.get("stackStatus"))) {
log.info(
"Cloud formation stack failed to completed. Status: " + stack.get("stackStatusReason"));
throw new RuntimeException((String) stack.get("stackStatusReason"));
} else if (isFailed(status)) {
String statusReason =
isChangeSet
? getChangeSetInfo(stack, stage.getContext(), "statusReason")
: getStackInfo(stack, "stackStatusReason");
log.info("Cloud formation stack failed to completed. Status: " + statusReason);
throw new RuntimeException(statusReason);
}
throw new RuntimeException("Unexpected stack status: " + stack.get("stackStatus"));
} catch (RetrofitError e) {
Expand All @@ -85,26 +106,54 @@ public long getTimeout() {
return timeout;
}

private String getStackInfo(Map stack, String field) {
return (String) stack.get(field);
}

private String getChangeSetInfo(Map stack, Map context, String field) {
String changeSetName = (String) context.get("changeSetName");
log.debug("Getting change set status from stack for changeset {}: {}", changeSetName, stack);
return Optional.ofNullable((List<Map<String, ?>>) stack.get("changeSets"))
.orElse(Collections.emptyList()).stream()
.filter(changeSet -> changeSet.get("name").equals(changeSetName))
.findFirst()
.map(changeSet -> (String) changeSet.get(field))
.orElse(CloudFormationStates.NOT_YET_READY.toString());
}

private boolean isEmptyChangeSet(Stage stage, Map<String, ?> stack) {
if ((boolean) Optional.ofNullable(stage.getContext().get("isChangeSet")).orElse(false)) {
String status = getChangeSetInfo(stack, stage.getContext(), "status");
String statusReason = getChangeSetInfo(stack, stage.getContext(), "statusReason");
return status.equals(CloudFormationStates.FAILED.toString())
&& statusReason.startsWith("The submitted information didn't contain changes");
} else {
return false;
}
}

private boolean isComplete(Object status) {
if (status instanceof String) {
return ((String) status).endsWith("CREATE_COMPLETE")
|| ((String) status).endsWith("UPDATE_COMPLETE");
return ((String) status).endsWith(CloudFormationStates.CREATE_COMPLETE.toString())
|| ((String) status).endsWith(CloudFormationStates.UPDATE_COMPLETE.toString());
} else {
return false;
}
}

private boolean isInProgress(Object status) {
if (status instanceof String) {
return ((String) status).endsWith("IN_PROGRESS");
return ((String) status).endsWith(CloudFormationStates.IN_PROGRESS.toString())
|| ((String) status).endsWith(CloudFormationStates.NOT_YET_READY.toString());
} else {
return false;
}
}

private boolean isFailed(Object status) {
if (status instanceof String) {
return ((String) status).endsWith("ROLLBACK_COMPLETE");
return ((String) status).endsWith(CloudFormationStates.ROLLBACK_COMPLETE.toString())
|| ((String) status).endsWith(CloudFormationStates.FAILED.toString());
} else {
return false;
}
Expand Down
13 changes: 8 additions & 5 deletions ...ddriver/tasks/providers/aws/cloudformation/CloudFormationForceCacheRefreshTaskSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class CloudFormationForceCacheRefreshTaskSpec extends Specification {
def stage = stage()
stage.context.put("credentials", credentials)
stage.context.put("regions", regions)
stage.context.put("stackName", stackName)

when:
task.execute(stage)
Expand All @@ -55,11 +56,13 @@ class CloudFormationForceCacheRefreshTaskSpec extends Specification {
1 * task.cacheService.forceCacheUpdate('aws', CloudFormationForceCacheRefreshTask.REFRESH_TYPE, expectedData)

where:
credentials | regions || expectedData
null | null || [:]
"credentials" | null || [credentials: "credentials"]
null | ["eu-west-1"] || [region: ["eu-west-1"]]
"credentials" | ["eu-west-1"] || [credentials: "credentials", region: ["eu-west-1"]]
credentials | regions | stackName || expectedData
null | null | null || [:]
"credentials" | null | null || [credentials: "credentials"]
null | ["eu-west-1"] | null || [region: ["eu-west-1"]]
"credentials" | ["eu-west-1"] | null || [credentials: "credentials", region: ["eu-west-1"]]
null | null | "stackName" || [stackName: "stackName"]
"credentials" | ["eu-west-1"] | "stackName" || [credentials: "credentials", region: ["eu-west-1"], stackName: "stackName"]


}
Expand Down
89 changes: 79 additions & 10 deletions ...ddriver/tasks/providers/aws/cloudformation/WaitForCloudFormationCompletionTaskSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,29 @@ class WaitForCloudFormationCompletionTaskSpec extends Specification {
def waitForCloudFormationCompletionTask = new WaitForCloudFormationCompletionTask(oortService: oortService)

@Unroll
def "should succeed if the stack creation is '#status'"() {
def "should succeed if the stack creation is '#status' and if isChangeSet property is '#isChangeSet'"() {
given:
def pipeline = Execution.newPipeline('orca')
def context = [
'credentials': 'creds',
'cloudProvider': 'aws',
'isChangeSet': isChangeSet,
'changeSetName': 'changeSetName',
'kato.tasks': [[resultObjects: [[stackId: 'stackId']]]]
]
def stage = new Stage(pipeline, 'test', 'test', context)
def stack = [stackId: 'stackId', stackStatus: status] as Map
def stack = [
stackId: 'stackId',
stackStatus: status,
stackStatusReason: statusReason,
changeSets: [
[
name: 'changeSetName',
status: status,
statusReason: statusReason
]
]
]

when:
def result = waitForCloudFormationCompletionTask.execute(stage)
Expand All @@ -53,9 +66,10 @@ class WaitForCloudFormationCompletionTaskSpec extends Specification {
result.outputs == stack

where:
status | expectedResult
'CREATE_COMPLETE' | ExecutionStatus.SUCCEEDED
'UPDATE_COMPLETE' | ExecutionStatus.SUCCEEDED
isChangeSet | status | statusReason || expectedResult
false | 'CREATE_COMPLETE' | 'ignored' || ExecutionStatus.SUCCEEDED
false | 'UPDATE_COMPLETE' | 'ignored' || ExecutionStatus.SUCCEEDED
true | 'FAILED' | 'The submitted information didn\'t contain changes' || ExecutionStatus.SUCCEEDED
}

@Unroll
Expand Down Expand Up @@ -105,29 +119,50 @@ class WaitForCloudFormationCompletionTaskSpec extends Specification {
result.outputs.isEmpty()
}
def "should error on unknown stack status"() {
@Unroll
def "should error on known error states or unknown stack status"() {
given:
def pipeline = Execution.newPipeline('orca')
def context = [
'credentials': 'creds',
'cloudProvider': 'aws',
'isChangeSet': isChangeSet,
'changeSetName': 'changeSetName',
'kato.tasks': [[resultObjects: [[stackId: 'stackId']]]]
]
def stage = new Stage(pipeline, 'test', 'test', context)
def stack = [stackStatus: 'UNKNOWN']
def stack = [
stackStatus: status,
stackStatusReason: "Stack failed",
changeSets: [
[
name: 'changeSetName',
status: status,
statusReason: "Change set failed"
]
]
]
when:
def result = waitForCloudFormationCompletionTask.execute(stage)
then:
1 * oortService.getCloudFormationStack('stackId') >> stack
RuntimeException ex = thrown()
ex.message.startsWith("Unexpected stack status")
result == null
ex.message.startsWith(expectedMessage)
where:
isChangeSet | status || expectedMessage
false | 'UNKNOWN' || 'Unexpected stack status'
false | 'ROLLBACK_COMPLETE' || 'Stack failed'
false | 'CREATE_FAILED' || 'Stack failed'
true | 'UNKNOWN' || 'Unexpected stack status'
true | 'ROLLBACK_COMPLETE' || 'Change set failed'
true | 'FAILED' || 'Change set failed'
}
def "should error when clouddriver responds with an error other than 404"() {
def pipeline = Execution.newPipeline('orca')
def pipeline = Execution.newPipeline('orca')
def context = [
'credentials': 'creds',
'cloudProvider': 'aws',
Expand All @@ -146,4 +181,38 @@ class WaitForCloudFormationCompletionTaskSpec extends Specification {
result == null
}
@Unroll
def "should get the change set status if it's a change set"() {
def pipeline = Execution.newPipeline('orca')
def context = [
'credentials': 'creds',
'cloudProvider': 'aws',
'isChangeSet': true,
'changeSetName': 'changeSetName',
'kato.tasks': [[resultObjects: [[stackId: 'stackId']]]]
]
def stage = new Stage(pipeline, 'test', 'test', context)
def stack = [
stackStatus: 'UPDATE_COMPLETE',
changeSets: [
[
name: 'changeSetName',
status: status
]
]
]
when:
def result = waitForCloudFormationCompletionTask.execute(stage)
then:
1 * oortService.getCloudFormationStack('stackId') >> stack
result.status == expectedResult
where:
status | expectedResult
'CREATE_IN_PROGRESS' | ExecutionStatus.RUNNING
'CREATE_COMPLETE' | ExecutionStatus.SUCCEEDED
}
}

0 comments on commit 60ed2ad

Please sign in to comment.