Skip to content
This repository has been archived by the owner on Jan 6, 2022. It is now read-only.

Commit

Permalink
Merge c1a516a into cfde1ec
Browse files Browse the repository at this point in the history
  • Loading branch information
karolyi committed Nov 24, 2015
2 parents cfde1ec + c1a516a commit 7e09233
Show file tree
Hide file tree
Showing 13 changed files with 631 additions and 156 deletions.
36 changes: 30 additions & 6 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,10 @@ def set_properties(project):
'lambda_file_access_control',
os.environ.get('LAMBDA_FILE_ACCESS_CONTROL'))

project.set_property('template_files',
[
('cfn-sphere/templates','crassus.yaml'),
('cfn-sphere/templates','crassus_integration_test_role.yaml'),
])
project.set_property('template_files', [
('cfn-sphere/templates', 'crassus.yaml'),
('cfn-sphere/templates', 'crassus_integration_test_role.yaml'),
])

project.set_property('distutils_classifiers', [
'Development Status :: 4 - Beta',
Expand Down Expand Up @@ -85,8 +84,33 @@ def set_properties_for_teamcity_builds(project):
os.environ.get('PYPIPROXY_URL'))


@init(environments='backchannel_temp')
def set_properties_for_backchannel_builds(project):
# For upload_zip_to_s3
project.set_property('bucket_prefix', 'backchannel_')
# For upload_cfn_to_s3
project.set_property('template_key_prefix', 'backchannel_')
project.set_property('template_files', [
('cfn-sphere/templates', 'crassus.yaml'),
])

project.version = '%s-%s' % (
project.version, os.environ.get('BUILD_NUMBER', 0))
project.set_property('bucket_name', 'crassus-lambda-zips')
project.set_property('lambda_file_access_control', 'public-read')
project.default_task = [
'clean',
'install_build_dependencies',
'publish',
'package_lambda_code',
'upload_zip_to_s3',
'upload_cfn_to_s3',
]
project.set_property('install_dependencies_index_url',
os.environ.get('PYPIPROXY_URL'))


@init(environments='integration_env')
def set_properties_for_teamcity_integration_test(project):
use_plugin("python.integrationtest")
project.set_property('integrationtest_inherit_environment', True)

130 changes: 78 additions & 52 deletions cfn-sphere/templates/crassus.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,22 @@ AWSTemplateFormatVersion: '2010-09-09'
Description: crassus (Cross Account Smart Software Update Service)
Outputs:
inputSnsTopicARN:
Description: Topic ARN to send stack update messages to
Description: Topic ARN to send stack update messages to.
Value:
Ref: inputSnsTopic
outputSnsTopicARN:
Description: Topic ARN to receive messages from CFN/lambdas
Value:
Ref: outputSnsTopic
outputSqsQueue:
Description: SQS queue URL for receiving messages from crassus
Description: SQS queue URL for receiving messages from crassus & CFN.
Value:
Ref: outputSqsQueue
lambdaFunctionName:
Description: Name of the crassus Lambda function, to be used in seperate API call to allow
subscription to input SNS topic
Description: Name of the crassus Lambda function, to be used in separate API call to allow
subscription to input SNS topic.
Value:
Ref: updateStackFunction
cfnOutputConverterFunctionName:
Description: The part of Crassus with converts received cloudformation messages to Gaius format.
Value:
Ref: cfnOutputConverterFunction
Parameters:
bucketName:
Default: is24-crassus
Expand Down Expand Up @@ -46,22 +46,7 @@ Resources:
TopicName:
'|join|-':
- Ref: AWS::StackName
- input
outputSnsTopic:
Type: AWS::SNS::Topic
Properties:
DisplayName: Topic ARN to receive messages from CFN/lambdas
TopicName:
'|join|-':
- Ref: AWS::StackName
- output
Subscription:
-
Endpoint:
Fn::GetAtt:
- outputSqsQueue
- Arn
Protocol: sqs
- snsinput
inputSnsTopicPolicy:
Type: AWS::SNS::TopicPolicy
Properties:
Expand Down Expand Up @@ -118,10 +103,11 @@ Resources:
- ""
-
- '{'
- '"topic_list":["'
- Ref: outputSnsTopic
- '"]'
- '}'
- '"result_queue":["'
- Ref: outputSqsQueue
- '"],"cfn_events":["'
- Ref: cfnOutputSnsTopic
- '"]}'
Handler: crassus_deployer_lambda.handler
Role:
Fn::GetAtt:
Expand All @@ -138,43 +124,83 @@ Resources:
Principal: sns.amazonaws.com
SourceArn:
Ref: inputSnsTopic
OutputSqsPolicy:
Type: "AWS::SQS::QueuePolicy"
cfnOutputConverterPermission:
Type: AWS::Lambda::Permission
Properties:
Action: lambda:invokeFunction
FunctionName:
Ref: cfnOutputConverterFunction
Principal: sns.amazonaws.com
SourceArn:
Ref: cfnOutputSnsTopic
cfnOutputConverterFunction:
Type: AWS::Lambda::Function
Properties:
Code:
S3Bucket:
Ref: bucketName
S3Key:
Ref: zipFile
Description:
Fn::Join:
- ""
-
- '{"result_queue":["'
- Ref: outputSqsQueue
- '"]}'
Handler: crassus_deployer_lambda.cfn_output_converter
Role:
Fn::GetAtt:
- stackUpdateRole
- Arn
Runtime: python2.7
Timeout: 15
cfnOutputSnsTopic:
Type: AWS::SNS::Topic
Properties:
DisplayName: Topic ARN to receive messages from cloudformation for format conversion.
Subscription:
- Endpoint:
Fn::GetAtt:
- cfnOutputConverterFunction
- Arn
Protocol: lambda
cfnOutputSnsTopicPolicy:
Type: AWS::SNS::TopicPolicy
Properties:
PolicyDocument:
Version: "2012-10-17"
Id: OutputSqsPolicy
Statement:
-
Sid: "Allow-SendMessage-to-outputSqsQueue-from-outputSnsTopic"
Effect: "Allow"
Principal: "*"
Action:
- "sqs:SendMessage"
Resource: "*"
Condition:
ArnEquals:
aws:SourceArn:
Ref: outputSnsTopic
Queues:
-
Ref: outputSqsQueue
OutputSqsReceivePolicy:
- Action: sns:Publish
Effect: Allow
Principal:
AWS:
Ref: triggeringUserArn
Resource:
Ref: cfnOutputSnsTopic
Version: '2012-10-17'
Topics:
- Ref: inputSnsTopic
outputSqsReceivePolicy:
Type: "AWS::SQS::QueuePolicy"
Properties:
PolicyDocument:
Version: "2012-10-17"
Id: OutputSqsReceivePolicy
Id: outputSqsReceivePolicy
Statement:
-
Sid: "Allow-ReceiveMessage-from-outputSqsQueue"
Sid: "Allow-All-from-outputSqsQueue"
Effect: "Allow"
Principal:
AWS:
Ref: triggeringUserArn
Action:
- "sqs:ReceiveMessage"
Resource: "*"
- "SQS:ReceiveMessage"
- "SQS:DeleteMessage"
- "SQS:GetQueueUrl"
Resource:
Fn::GetAtt:
- outputSqsQueue
- Arn
Queues:
-
Ref: outputSqsQueue
Ref: outputSqsQueue
11 changes: 6 additions & 5 deletions src/integrationtest/python/crassus_integration_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def run(self):


class CrassusIntegrationTest(unittest.TestCase):

def setUp(self):
fqdn = socket.gethostname()
hostname = fqdn[:fqdn.find('.')] if '.' in fqdn else fqdn
Expand Down Expand Up @@ -98,11 +99,11 @@ def assume_role(self, invoker_role):
credentials = self.sts_client.assume_role(
RoleArn=invoker_role.arn, RoleSessionName='crassus-it')['Credentials']

ec2 = boto3.client(service_name='ec2', region_name=REGION_NAME,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials[
'SecretAccessKey'],
aws_session_token=credentials['SessionToken'])
ec2 = boto3.client(
service_name='ec2', region_name=REGION_NAME,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'])
try:
ec2.describe_instances()
self.fail(
Expand Down
116 changes: 116 additions & 0 deletions src/main/python/crassus/aws_tools.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# -*- coding: utf-8 -*-

import json
import logging

import boto3
from crassus.deployment_response import DeploymentResponse

__doc__ = """AWS helper functions module."""

aws_cf = boto3.client('cloudformation')
aws_sqs = boto3.client('sqs')
aws_lambda = boto3.client('lambda')

logger = logging.getLogger(__name__)


class AwsToolsBaseException(Exception):
pass


class StackResourceException(AwsToolsBaseException):
pass


def get_physical_by_logical_id(stack_name, resource_id): # pragma: no cover
"""
Get a physical ID (mostly ARN) from a logical ID in a given stack's
parameters.
Raise StackResourceException if an error occurred, return None if
not found, and return the ID if found.
"""
stack_param_dict = aws_cf.list_stack_resources(StackName=stack_name)
stack_param_http_result = stack_param_dict.get(
'ResponseMetadata', {}).get('HTTPStatusCode')
if stack_param_http_result != 200:
# We got a weird/nonexistent response code
raise StackResourceException('Erroneous HTTPStatusCode.')
for parameters_item in stack_param_dict.get('StackResourceSummaries', []):
if parameters_item['LogicalResourceId'] == resource_id:
return parameters_item['PhysicalResourceId']


def get_my_stack_name_by_func_arn(invoked_function_arn): # pragma: no cover
"""
Get the stack name for an invoked function ARN.
Returns the stack name if found, None if not available or in case
of any errors.
"""
stack_dict = aws_cf.list_stacks(
StackStatusFilter=[
'CREATE_COMPLETE',
'UPDATE_COMPLETE',
])
stack_param_http_result = stack_dict.get(
'ResponseMetadata', {}).get('HTTPStatusCode')
if stack_param_http_result != 200:
# We got a weird/nonexistent response code
return
for item in stack_dict.get('StackSummaries', []):
stack_name = item['StackName']
try:
resource_id = get_physical_by_logical_id(
stack_name, invoked_function_arn)
except StackResourceException:
continue
if resource_id is not None:
# Got the resource id, our stack name is the current
# stack_name
return stack_name


def sqs_send_message(queue_url_list, message):
"""
Send an message to a given SQS queue. The function is not foolproof,
you should have the rights to transmit to the SQS queue.
"""
if type(message) is not DeploymentResponse:
logger.error(
'sqs_send_message: got wrong type of message parameter: {0}: {1}'
.format(type(message), repr(message)))
return
message_str = json.dumps(message)
for queue_url in queue_url_list:
aws_sqs.send_message(
QueueUrl=queue_url, MessageBody=message_str, DelaySeconds=0)


def get_lambda_config_property(context, property_name):
"""
Extract JSON properties from the JSON encoded description.
Return the value for the property, None if not found.
"""
function_arn = context.invoked_function_arn
qualifier = context.function_version
description = aws_lambda.get_function_configuration(
FunctionName=function_arn,
Qualifier=qualifier
)['Description']
try:
data = json.loads(description)
return_value = data[property_name]
logger.debug('Extracted {0} property: %{1}'.format(
property_name, repr(return_value)))
return return_value
except ValueError:
logger.error(
'Description of function must contain JSON, but was "{0}"'
.format(description))
except KeyError:
logger.error(
'Unable to find \'{0}\' property in the JSON description.'
.format(property_name))
Loading

0 comments on commit 7e09233

Please sign in to comment.