diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..013dd20 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[report] +show_missing = True diff --git a/.gitignore b/.gitignore index 2bf3cc8..65d1a07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ # Distribution / packaging +.coverage +.floo +.idea .Python +.requirements +.npmignore +ardenv/ env/ build/ develop-eggs/ @@ -14,6 +20,7 @@ sdist/ var/ *.egg-info/ .installed.cfg +*.pyc *.egg node_modules/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5ae2789 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: python +cache: pip +sudo: required +dist: precise + +matrix: + include: + - python: 2.7 + +install: +- pip install -r test-requirements.txt +- pip install ${CODECOV:+codecov} +script: +- nosetests -d tests -- ${CODECOV:+--with-coverage --cover-xml --cover-package=ardere} +after_success: +- ${CODECOV:+codecov} +notifications: + slack: + secure: vT9sWtUuxk28g6xYKAsQmiPZllErOYVfx5lcL+/jo1eRFrmbpYnyndT6s+FxGI1547oizZ0IqZbHVvB7BUoSJixXJyQJYXW2MchwN1UeHrey8mYpF1GNEaJT7FMfqSkxUU9gvAZ3IU7zstNeTLbfG1GkLuzybp0WAiHl/ocUTz8= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5cb6c02 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,77 @@ +# Contribution Guidelines + +Anyone is welcome to contribute to this project. Feel free to get in touch with +other community members on IRC, the mailing list or through issues here on +GitHub. + +[See the README](/README.md) for contact information. + +## Bug Reports + +You can file issues here on GitHub. Please try to include as much information as +you can and under what conditions you saw the issue. + +## Sending Pull Requests + +Patches should be submitted as pull requests (PR). + +Before submitting a PR: +- Your code must run and pass all the automated tests before you submit your PR + for review. "Work in progress" pull requests are allowed to be submitted, but + should be clearly labeled as such and should not be merged until all tests + pass and the code has been reviewed. +- Your patch should include new tests that cover your changes. It is your and + your reviewer's responsibility to ensure your patch includes adequate tests. + +When submitting a PR: +- You agree to license your code under the project's open source license + ([MPL 2.0](/LICENSE)). +- Base your branch off the current `master` (see below for an example workflow). +- Add both your code and new tests if relevant. +- Run the test suite to make sure your code passes linting and tests. +- Please do not include merge commits in pull requests; include only commits with the new relevant code. + +See the main [README.md](/README.md) for information on prerequisites, installing, running and testing. + +## Code Review + +This project is production Mozilla code and subject to our [engineering practices and quality standards](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Committing_Rules_and_Responsibilities). Every patch must be peer reviewed. + +## Git Commit Guidelines + +We loosely follow the [Angular commit guidelines](https://github.com/angular/angular.js/blob/master/CONTRIBUTING.md#type) of `(): ` where `type` must be one of: + +* **feat**: A new feature +* **fix**: A bug fix +* **docs**: Documentation only changes +* **style**: Changes that do not affect the meaning of the code (white-space, formatting, missing + semi-colons, etc) +* **refactor**: A code change that neither fixes a bug or adds a feature +* **perf**: A code change that improves performance +* **test**: Adding missing tests +* **chore**: Changes to the build process or auxiliary tools and libraries such as documentation + generation + +### Scope +The scope could be anything specifying place of the commit change. + +### Subject +The subject contains succinct description of the change: + +* use the imperative, present tense: "change" not "changed" nor "changes" +* don't capitalize first letter +* no dot (.) at the end + +###Body +In order to maintain a reference to the context of the commit, add +`fixes #` if it closes a related issue or `issue #` +if it's a partial fix. + +You can also write a detailed description of the commit: Just as in the +**subject**, use the imperative, present tense: "change" not "changed" nor +"changes" It should include the motivation for the change and contrast this with +previous behavior. + +###Footer +The footer should contain any information about **Breaking Changes** and is also +the place to reference GitHub issues that this commit **Closes**. diff --git a/Design.md b/Design.md deleted file mode 100644 index aedb27b..0000000 --- a/Design.md +++ /dev/null @@ -1,3 +0,0 @@ -# Requirements - -## TODO: Document new design. \ No newline at end of file diff --git a/README.md b/README.md index a781f4b..1dcf6b9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,47 @@ # ardere -AWS Serverless Service for Load-Testing +*AWS Serverless Service for Load-Testing* + +ardere runs as a serverless service using AWS to orchestrate +load-tests consisting of docker container configurations arranged as +test plans. + +## Installation + +To deploy ardere to your AWS account, you will need a fairly recent +install of Node, then install the Node packages required: + + $ npm install + +You will need to ensure your have AWS access and secret keys configured +for serverless: + + $ sls config + +To deploy the ardere lambda's and required AWS stack: + + $ sls deploy + +Then you can deploy the ardere Step Function: + + $ sls deploy stepf + + +## Developing + +ardere is written in Python and deployed via serverless to AWS. To an +extent testing it on AWS is the most reliable indicator it works as +intended. However, there are sets of tests that ensure the Python code +is valid and works with arguments as intended that may be run locally. + +Create a Python virtualenv, and install the test requirements: + + $ virtualenv ardenv + $ source ardenv/bin/activate + $ pip install -r test-requirements.txt + +The tests can now be run with nose: + + $ nosetests + +Note that **you cannot run the sls deploy while the virtualenv is active** +due to how the serverless Python requirements plugin operates. diff --git a/ardere/aws.py b/ardere/aws.py new file mode 100644 index 0000000..4baea56 --- /dev/null +++ b/ardere/aws.py @@ -0,0 +1,299 @@ +"""AWS Helper Classes""" +import logging +import os +import time +import uuid +from collections import defaultdict + +import boto3 +import botocore +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List # noqa + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# Shell script to load +dir_path = os.path.dirname(os.path.realpath(__file__)) +parent_dir_path = os.path.dirname(dir_path) +shell_path = os.path.join(parent_dir_path, "src", "shell", + "waitforcluster.sh") + +# Load the shell script +with open(shell_path, 'r') as f: + shell_script = f.read() + + +class ECSManager(object): + """ECS Manager queries and manages an ECS cluster""" + # For testing purposes + boto = boto3 + + # ECS optimized AMI id's + ecs_ami_ids = { + "us-east-1": "ami-b2df2ca4", + "us-east-2": "ami-832b0ee6", + "us-west-1": "ami-dd104dbd", + "us-west-2": "ami-022b9262" + } + + def __init__(self, plan): + # type: (str) -> None + """Create and return a ECSManager for a cluster of the given name.""" + self._ecs_client = self.boto.client('ecs') + self._ec2_client = self.boto.client('ec2') + self._ecs_name = plan["ecs_name"] + self._plan = plan + + if "plan_run_uuid" not in plan: + plan["plan_run_uuid"] = str(uuid.uuid4()) + + self._plan_uuid = plan["plan_run_uuid"] + + @property + def s3_ready_file(self): + return "https://s3.amazonaws.com/{bucket}/{key}".format( + bucket=os.environ["s3_ready_bucket"], + key="{}.ready".format(self._plan_uuid) + ) + + def family_name(self, step): + """Generate a consistent family name for a given step""" + return step["name"] + "-" + self._plan_uuid + + def query_active_instances(self): + # type: () -> Dict[str, int] + """Query EC2 for all the instances owned by ardere for this cluster.""" + instance_dict = defaultdict(int) + paginator = self._ec2_client.get_paginator('describe_instances') + response_iterator = paginator.paginate( + Filters=[ + { + "Name": "tag:Owner", + "Values": ["ardere"] + }, + { + "Name": "tag:ECSCluster", + "Values": [self._ecs_name] + } + ] + ) + for page in response_iterator: + for reservation in page["Reservations"]: + for instance in reservation["Instances"]: + # Determine if the instance is pending/running and count + # 0 = Pending, 16 = Running, > is all shutting down, etc. + if instance["State"]["Code"] <= 16: + instance_dict[instance["InstanceType"]] += 1 + return instance_dict + + def calculate_missing_instances(self, desired, current): + # type: (Dict[str, int], Dict[str, int]) -> Dict[str, int] + """Determine how many of what instance types are needed to ensure + the current instance dict has all the desired instance count/types.""" + needed = {} + for instance_type, instance_count in desired.items(): + cur = current.get(instance_type, 0) + if cur < instance_count: + needed[instance_type] = instance_count - cur + return needed + + def request_instances(self, instances): + # type: (Dict[str, int]) -> None + """Create requested types/quantities of instances for this cluster""" + ami_id = self.ecs_ami_ids["us-east-1"] + request_instances = [] + for instance_type, instance_count in instances.items(): + result = self._ec2_client.run_instances( + ImageId=ami_id, + KeyName="loads", + MinCount=instance_count, + MaxCount=instance_count, + InstanceType=instance_type, + UserData="#!/bin/bash \necho ECS_CLUSTER='" + self._ecs_name + + "' >> /etc/ecs/ecs.config", + IamInstanceProfile={"Arn": os.environ["ecs_profile"]} + ) + + # Track returned instances for tagging step + request_instances.extend([x["InstanceId"] for x in + result["Instances"]]) + + self._ec2_client.create_tags( + Resources=request_instances, + Tags=[ + dict(Key="Owner", Value="ardere"), + dict(Key="ECSCluster", Value=self._ecs_name) + ] + ) + + def create_service(self, step): + # type: (Dict[str, Any]) -> Dict[str, Any] + """Creates an ECS service for a step and returns its info""" + logger.info("CreateService called with: {}".format(step)) + + # Prep the shell command + shell_command = [ + 'sh', '-c', '"$WAITFORCLUSTER"', + 'waitforcluster.sh', self.s3_ready_file, + str(step.get("run_delay", 0)) + ] + shell_command2 = ' '.join(shell_command) + ' && ' + step[ + "additional_command_args"] + shell_command3 = ['sh', '-c', '{}'.format(shell_command2)] + + # ECS wants a family name for task definitions, no spaces, 255 chars + family_name = step["name"] + "-" + self._plan_uuid + task_response = self._ecs_client.register_task_definition( + family=family_name, + containerDefinitions=[ + { + "name": step["name"], + "image": step["container_name"], + "cpu": step["cpu_units"], + # using only memoryReservation sets no hard limit + "memoryReservation": 256, + "environment": [ + { + "name": "WAITFORCLUSTER", + "value": shell_script + } + ], + "entryPoint": shell_command3 + } + ], + placementConstraints=[ + # Ensure the service is confined to the right instance type + { + "type": "memberOf", + "expression": "attribute:ecs.instance-type == {}".format( + step["instance_type"]), + } + ] + ) + task_arn = task_response["taskDefinition"]["taskDefinitionArn"] + step["taskArn"] = task_arn + service_result = self._ecs_client.create_service( + cluster=self._ecs_name, + serviceName=step["name"], + taskDefinition=task_arn, + desiredCount=step["instance_count"], + deploymentConfiguration={ + "minimumHealthyPercent": 0, + "maximumPercent": 100 + }, + placementConstraints=[ + { + "type": "distinctInstance" + } + ] + ) + step["serviceArn"] = service_result["service"]["serviceArn"] + step["service_status"] = "STARTED" + return step + + def create_services(self, steps): + # type: (List[Dict[str, Any]]) -> None + """Create ECS Services given a list of steps""" + with ThreadPoolExecutor(max_workers=8) as executer: + results = executer.map(self.create_service, steps) + return list(results) + + def service_ready(self, step): + # type: (Dict[str, Any]) -> bool + """Query a service and return whether all its tasks are running""" + service_name = step["name"] + response = self._ecs_client.describe_services( + cluster=self._ecs_name, + services=[service_name] + ) + + try: + deploy = response["services"][0]["deployments"][0] + except (TypeError, IndexError): + return False + return deploy["desiredCount"] == deploy["runningCount"] + + def all_services_ready(self, steps): + # type: (List[Dict[str, Any]]) -> bool + """Queries all service ARN's in the plan to see if they're ready""" + with ThreadPoolExecutor(max_workers=8) as executer: + results = executer.map(self.service_ready, steps) + return all(results) + + def stop_finished_service(self, start_time, step): + # type: (start_time, Dict[str, Any]) -> None + """Stops a service if it needs to shutdown""" + if step["service_status"] == "STOPPED": + return + + # Calculate time + step_duration = step.get("run_delay", 0) + step["run_max_time"] + now = time.time() + if now < (start_time + step_duration): + return + + # Running long enough to shutdown + self._ecs_client.update_service( + cluster=self._ecs_name, + service=step["name"], + desiredCount=0 + ) + step["service_status"] = "STOPPED" + + def stop_finished_services(self, start_time, steps): + # type: (int, List[Dict[str, Any]]) -> None + """Shuts down any services that have run for their max time""" + for step in steps: + self.stop_finished_service(start_time, step) + + def shutdown_plan(self, steps): + """Terminate the entire plan, ensure all services and task + definitions are completely cleaned up and removed""" + # Locate all the services for the ECS Cluster + paginator = self._ecs_client.get_paginator('list_services') + response_iterator = paginator.paginate( + cluster=self._ecs_name + ) + + # Collect all the service ARN's + service_arns = [] + for page in response_iterator: + service_arns.extend(page["serviceArns"]) + + for service_arn in service_arns: + try: + self._ecs_client.update_service( + cluster=self._ecs_name, + service=service_arn, + desiredCount=0 + ) + except botocore.exceptions.ClientError: + continue + + try: + self._ecs_client.delete_service( + cluster=self._ecs_name, + service=service_arn + ) + except botocore.exceptions.ClientError: + pass + + # Locate all the task definitions for this plan + for step in steps: + try: + response = self._ecs_client.describe_task_definition( + taskDefinition=self.family_name(step) + ) + except botocore.exceptions.ClientError: + continue + + task_arn = response["taskDefinition"]["taskDefinitionArn"] + + # Deregister the task + try: + self._ecs_client.deregister_task_definition( + taskDefinition=task_arn + ) + except botocore.exceptions.ClientError: + pass diff --git a/ardere/exceptions.py b/ardere/exceptions.py new file mode 100644 index 0000000..55e5b08 --- /dev/null +++ b/ardere/exceptions.py @@ -0,0 +1,6 @@ +class ServicesStartingException(Exception): + """Exception to indicate Services are still Starting""" + + +class ShutdownPlanException(Exception): + """Exception to indicate the Plan should be Shutdown""" diff --git a/ardere/handler.py b/ardere/handler.py index 5063f8a..60911dd 100644 --- a/ardere/handler.py +++ b/ardere/handler.py @@ -1,314 +1,23 @@ import logging import os import time -import uuid from collections import defaultdict import boto3 import botocore -from concurrent.futures import ThreadPoolExecutor - from typing import Any, Dict, List # noqa +from aws import ECSManager +from exceptions import ( + ServicesStartingException, + ShutdownPlanException +) logger = logging.getLogger() logger.setLevel(logging.INFO) -# Shell script to load -dir_path = os.path.dirname(os.path.realpath(__file__)) -parent_dir_path = os.path.dirname(dir_path) -shell_path = os.path.join(parent_dir_path, "src", "shell", - "waitforcluster.sh") - - -class ServicesStartingException(Exception): - """Exception to indicate Services are still Starting""" - - -class ShutdownPlanException(Exception): - """Exception to indicate the Plan should be Shutdown""" - - -class ECSManager(object): - """ECS Manager queries and manages an ECS cluster""" - - # ECS optimized AMI id's - ecs_ami_ids = { - "us-east-1": "ami-b2df2ca4", - "us-east-2": "ami-832b0ee6", - "us-west-1": "ami-dd104dbd", - "us-west-2": "ami-022b9262" - } - - def __init__(self, plan): - # type: (str) -> None - """Create and return a ECSManager for a cluster of the given name.""" - self._ecs_client = boto3.client('ecs') - self._ec2_client = boto3.client('ec2') - self._ecs_name = plan["ecs_name"] - - if "plan_run_uuid" not in plan: - plan["plan_run_uuid"] = str(uuid.uuid4()) - - self._plan_uuid = plan["plan_run_uuid"] - - @property - def s3_ready_file(self): - return "https://s3.amazonaws.com/{bucket}/{key}".format( - bucket=os.environ["s3_ready_bucket"], - key="{}.ready".format(self._plan_uuid) - ) - - def family_name(self, step): - """Generate a consistent family name for a given step""" - return step["name"] + "-" + self._plan_uuid - - def query_active_instances(self): - # type: () -> Dict[str, int] - """Query EC2 for all the instances owned by ardere for this cluster.""" - instance_dict = defaultdict(int) - paginator = self._ec2_client.get_paginator('describe_instances') - response_iterator = paginator.paginate( - Filters=[ - { - "Name": "tag:Owner", - "Values": ["ardere"] - }, - { - "Name": "tag:ECSCluster", - "Values": [self._ecs_name] - } - ] - ) - for page in response_iterator: - for reservation in page["Reservations"]: - for instance in reservation["Instances"]: - # Determine if the instance is pending/running and count - # 0 = Pending, 16 = Running, > is all shutting down, etc. - if instance["State"]["Code"] <= 16: - instance_dict[instance["InstanceType"]] += 1 - return instance_dict - - def calculate_missing_instances(self, desired, current): - # type: (Dict[str, int], Dict[str, int]) -> Dict[str, int] - """Determine how many of what instance types are needed to ensure - the current instance dict has all the desired instance count/types.""" - needed = {} - for instance_type, instance_count in desired.items(): - cur = current.get(instance_type, 0) - if cur < instance_count: - needed[instance_type] = instance_count - cur - return needed - - def request_instances(self, instances): - # type: (Dict[str, int]) -> None - """Create requested types/quantities of instances for this cluster""" - ami_id = self.ecs_ami_ids["us-east-1"] - request_instances = [] - for instance_type, instance_count in instances.items(): - result = self._ec2_client.run_instances( - ImageId=ami_id, - KeyName="loads", - MinCount=instance_count, - MaxCount=instance_count, - InstanceType=instance_type, - UserData="#!/bin/bash \necho ECS_CLUSTER='" + self._ecs_name + - "' >> /etc/ecs/ecs.config", - IamInstanceProfile={"Arn": os.environ["ecs_profile"]} - ) - - # Track returned instances for tagging step - request_instances.extend([x["InstanceId"] for x in - result["Instances"]]) - - self._ec2_client.create_tags( - Resources=request_instances, - Tags=[ - dict(Key="Owner", Value="ardere"), - dict(Key="ECSCluster", Value=self._ecs_name) - ] - ) - - def create_service(self, step): - # type: (Dict[str, Any]) -> Dict[str, Any] - """Creates an ECS service for a step and returns its info""" - logger.info("CreateService called with: {}".format(step)) - - # Load the shell script - with open(shell_path, 'r') as f: - shell_script = f.read() - - # Prep the shell command - shell_command = [ - 'sh', '-c', '"$WAITFORCLUSTER"', - 'waitforcluster.sh', self.s3_ready_file, - str(step.get("run_delay", 0)) - ] - shell_command2 = ' '.join(shell_command) + ' && ' + step[ - "additional_command_args"] - shell_command3 = ['sh', '-c', '{}'.format(shell_command2)] - - # ECS wants a family name for task definitions, no spaces, 255 chars - family_name = step["name"] + "-" + self._plan_uuid - task_response = self._ecs_client.register_task_definition( - family=family_name, - containerDefinitions=[ - { - "name": step["name"], - "image": step["container_name"], - "cpu": step["cpu_units"], - # using only memoryReservation sets no hard limit - "memoryReservation": 256, - "environment": [ - { - "name": "WAITFORCLUSTER", - "value": shell_script - } - ], - "entryPoint": shell_command3 - } - ], - placementConstraints=[ - # Ensure the service is confined to the right instance type - { - "type": "memberOf", - "expression": "attribute:ecs.instance-type == {}".format( - step["instance_type"]), - } - ] - ) - task_arn = task_response["taskDefinition"]["taskDefinitionArn"] - step["taskArn"] = task_arn - service_result = self._ecs_client.create_service( - cluster=self._ecs_name, - serviceName=step["name"], - taskDefinition=task_arn, - desiredCount=step["instance_count"], - deploymentConfiguration={ - "minimumHealthyPercent": 0, - "maximumPercent": 100 - }, - placementConstraints=[ - { - "type": "distinctInstance" - }, - { - "type": "distinctInstance" - } - ] - ) - step["serviceArn"] = service_result["service"]["serviceArn"] - step["service_status"] = "STARTED" - return step - - def create_services(self, steps): - # type: (List[Dict[str, Any]]) -> None - """Create ECS Services given a list of steps""" - with ThreadPoolExecutor(max_workers=8) as executer: - results = executer.map(self.create_service, steps) - return list(results) - - def service_ready(self, step): - # type: (Dict[str, Any]) -> bool - """Query a service and return whether all its tasks are running""" - service_name = step["name"] - response = self._ecs_client.describe_services( - cluster=self._ecs_name, - services=[service_name] - ) - - try: - deploy = response["services"][0]["deployments"][0] - except TypeError: - return False - return deploy["desiredCount"] == deploy["runningCount"] - - def all_services_ready(self, steps): - # type: (List[Dict[str, Any]]) -> bool - """Queries all service ARN's in the plan to see if they're ready""" - with ThreadPoolExecutor(max_workers=8) as executer: - results = executer.map(self.service_ready, steps) - return all(results) - - def stop_finished_service(self, start_time, step): - # type: (start_time, Dict[str, Any]) -> None - """Stops a service if it needs to shutdown""" - if step["service_status"] == "STOPPED": - return - - # Calculate time - step_duration = step.get("run_delay", 0) + step["run_max_time"] - now = time.time() - if now < (start_time + step_duration): - return - - # Running long enough to shutdown - self._ecs_client.update_service( - cluster=self._ecs_name, - service=step["name"], - desiredCount=0 - ) - step["service_status"] = "STOPPED" - - def stop_finished_services(self, start_time, steps): - # type: (int, List[Dict[str, Any]]) -> None - """Shuts down any services that have run for their max time""" - for step in steps: - self.stop_finished_service(start_time, step) - - def shutdown_plan(self, steps): - """Terminate the entire plan, ensure all services and task - definitions are completely cleaned up and removed""" - # Locate all the services for the ECS Cluster - paginator = self._ecs_client.get_paginator('list_services') - response_iterator = paginator.paginate( - cluster=self._ecs_name - ) - - # Collect all the service ARN's - service_arns = [] - for page in response_iterator: - service_arns.extend(page["serviceArns"]) - - for service_arn in service_arns: - try: - self._ecs_client.update_service( - cluster=self._ecs_name, - service=service_arn, - desiredCount=0 - ) - except botocore.exceptions.ClientError: - continue - - try: - self._ecs_client.delete_service( - cluster=self._ecs_name, - service=service_arn - ) - except botocore.exceptions.ClientError: - pass - - # Locate all the task definitions for this plan - for step in steps: - try: - response = self._ecs_client.describe_task_definition( - taskDefinition=self.family_name(step) - ) - except botocore.exceptions.ClientError: - continue - - task_arn = response["taskDefinition"]["taskDefinitionArn"] - - # Deregister the task - try: - response = self._ecs_client.deregister_task_definition( - taskDefinition=task_arn - ) - except botocore.exceptions.ClientError: - continue - -def build_instance_map(test_plan): +def _build_instance_map(test_plan): """Given a JSON test-plan, build and return a dict of instance types and how many should exist for each type.""" instances = defaultdict(int) @@ -317,7 +26,7 @@ def build_instance_map(test_plan): return instances -def find_test_plan_duration(plan): +def _find_test_plan_duration(plan): # type: (Dict[str, Any]) -> int """Locates and calculates the longest test plan duration from its delay through its duration of the plan.""" @@ -330,7 +39,7 @@ def populate_missing_instances(event, context): logger.info("Called with {}".format(event)) logger.info("Environ: {}".format(os.environ)) ecs_manager = ECSManager(plan=event) - needed = build_instance_map(event) + needed = _build_instance_map(event) logger.info("Plan instances needed: {}".format(needed)) current_instances = ecs_manager.query_active_instances() missing_instances = ecs_manager.calculate_missing_instances( @@ -397,7 +106,7 @@ def check_for_cluster_done(event, context): # If we're totally done, exit. now = time.time() - plan_duration = find_test_plan_duration(event) + plan_duration = _find_test_plan_duration(event) if now > (start_time + plan_duration): raise ShutdownPlanException("Test Plan has completed") return event diff --git a/custom_resources/ecs.cloudform.template b/custom_resources/ecs.cloudform.template deleted file mode 100644 index 4ac6912..0000000 --- a/custom_resources/ecs.cloudform.template +++ /dev/null @@ -1,548 +0,0 @@ -{ - "Description": "AutoScale EC2 creation template", - "AWSTemplateFormatVersion": "2010-09-09", - "Metadata": { - "Comment": { - "Fn::Join": [ - "\n", - [ - "This template starts up a selected number of (small) instances that just run", - "a 'Hello world' sort of script.", - "Eventually this would call a bunch of CF Template snippets that would populate", - "the various load systems.", - "There are two systems, 'Parma*' which contains the system to test, and 'Telum*'", - "which contains the testing system. (parma is latin for shield, telum is latin for", - "arrow).", - "TODO: ", - "* expand parameter maps to include all allowed EC2 types", - "* add telum hosts", - "* use docker container references" - ] - ] - } - }, - "Parameters": { - "ParmaInstanceType": { - "Type": "String", - "Description": "Type of EC2 instance to run the targeted system on", - "Default": "t1.micro", - "AllowedValues": [ - "t1.micro", - "t2.nano", - "t2.micro", - "t2.small" - ], - "ConstraintDescription": "Must be a valid EC2 instance. " - }, - "ParmaInstanceCount": { - "Type": "Number", - "Description": "Number of instances for the targeted system", - "Default": 1, - "MaxValue": 10 - }, - "KeyPair": { - "Type": "AWS::EC2::KeyPair::KeyName", - "Description": "EC2 Key Pair for system access" - } - }, - "Mappings": { - "AWSInstanceType2Arch": { - "t1.micro": { - "Arch": "PV64" - }, - "t2.nano": { - "Arch": "HVM64" - }, - "t2.micro": { - "Arch": "HVM64" - }, - "t2.small": { - "Arch": "HVM64" - }, - "t2.medium": { - "Arch": "HVM64" - }, - "t2.large": { - "Arch": "HVM64" - }, - "m1.small": { - "Arch": "PV64" - }, - "m1.medium": { - "Arch": "PV64" - }, - "m1.large": { - "Arch": "PV64" - }, - "m1.xlarge": { - "Arch": "PV64" - }, - "m2.xlarge": { - "Arch": "PV64" - }, - "m2.2xlarge": { - "Arch": "PV64" - }, - "m2.4xlarge": { - "Arch": "PV64" - }, - "m3.medium": { - "Arch": "HVM64" - }, - "m3.large": { - "Arch": "HVM64" - }, - "m3.xlarge": { - "Arch": "HVM64" - }, - "m3.2xlarge": { - "Arch": "HVM64" - }, - "m4.large": { - "Arch": "HVM64" - }, - "m4.xlarge": { - "Arch": "HVM64" - }, - "m4.2xlarge": { - "Arch": "HVM64" - }, - "m4.4xlarge": { - "Arch": "HVM64" - }, - "m4.10xlarge": { - "Arch": "HVM64" - }, - "c1.medium": { - "Arch": "PV64" - }, - "c1.xlarge": { - "Arch": "PV64" - }, - "c3.large": { - "Arch": "HVM64" - }, - "c3.xlarge": { - "Arch": "HVM64" - }, - "c3.2xlarge": { - "Arch": "HVM64" - }, - "c3.4xlarge": { - "Arch": "HVM64" - }, - "c3.8xlarge": { - "Arch": "HVM64" - }, - "c4.large": { - "Arch": "HVM64" - }, - "c4.xlarge": { - "Arch": "HVM64" - }, - "c4.2xlarge": { - "Arch": "HVM64" - }, - "c4.4xlarge": { - "Arch": "HVM64" - }, - "c4.8xlarge": { - "Arch": "HVM64" - }, - "g2.2xlarge": { - "Arch": "HVMG2" - }, - "g2.8xlarge": { - "Arch": "HVMG2" - }, - "r3.large": { - "Arch": "HVM64" - }, - "r3.xlarge": { - "Arch": "HVM64" - }, - "r3.2xlarge": { - "Arch": "HVM64" - }, - "r3.4xlarge": { - "Arch": "HVM64" - }, - "r3.8xlarge": { - "Arch": "HVM64" - }, - "i2.xlarge": { - "Arch": "HVM64" - }, - "i2.2xlarge": { - "Arch": "HVM64" - }, - "i2.4xlarge": { - "Arch": "HVM64" - }, - "i2.8xlarge": { - "Arch": "HVM64" - }, - "d2.xlarge": { - "Arch": "HVM64" - }, - "d2.2xlarge": { - "Arch": "HVM64" - }, - "d2.4xlarge": { - "Arch": "HVM64" - }, - "d2.8xlarge": { - "Arch": "HVM64" - }, - "hi1.4xlarge": { - "Arch": "HVM64" - }, - "hs1.8xlarge": { - "Arch": "HVM64" - }, - "cr1.8xlarge": { - "Arch": "HVM64" - }, - "cc2.8xlarge": { - "Arch": "HVM64" - } - }, - "AWSRegionArch2AMI": { - "us-east-1": { - "PV64": "ami-2a69aa47", - "HVM64": "ami-6869aa05", - "HVMG2": "ami-648d9973" - }, - "us-west-2": { - "PV64": "ami-7f77b31f", - "HVM64": "ami-7172b611", - "HVMG2": "ami-09cd7a69" - }, - "us-west-1": { - "PV64": "ami-a2490dc2", - "HVM64": "ami-31490d51", - "HVMG2": "ami-1e5f0e7e" - }, - "us-east-2": { - "PV64": "NOT_SUPPORTED", - "HVM64": "ami-f6035893", - "HVMG2": "NOT_SUPPORTED" - } - } - }, - "Conditions": {}, - "Resources": { - "ParmaWebServerGroup": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "DependsOn": "ParmaLaunchConfig", - "Metadata": { - "Comment": { - "Fn::Join": ["\n", [ - "The ParmaLaunchConfig fn-init script will post an event to this", - "resource once it has initialized. You really want to make sure", - "that the name matches, else the build will fail." - ]] - } - }, - "CreationPolicy": { - "ResourceSignal": { - "Count": 1, - "Timeout": "PT15M" - } - }, - "Properties": { - "AvailabilityZones": { - "Fn::GetAZs": "" - }, - "LaunchConfigurationName": { - "Ref": "ParmaLaunchConfig" - }, - "MaxSize": { - "Ref": "ParmaInstanceCount" - }, - "MinSize": "1", - "LoadBalancerNames": [ - { - "Ref": "ParmaELB" - } - ] - } - }, - "ParmaLaunchConfig": { - "Type": "AWS::AutoScaling::LaunchConfiguration", - "Metadata": { - "Comment": "Simple App", - "AWS::CloudFormation::Init": { - "Metadata": { - "Comment": { - "Fn::Join": [ - " ", - [ - "TODO: Replace the following with pointers to the appropriate container inits.", - "Should probably leave the cfn-* bits alone." - ] - ] - } - }, - "config": { - "packages": { - "yum": { - "httpd": [] - } - }, - "files": { - "/var/www/html/index.html": { - "content": { - "Fn::Join": [ - "\n", - [ - "

Ok.

", - "stack = ", - { - "Ref": "AWS::StackName" - }, - " ", - { - "Ref": "AWS::StackId" - } - ] - ] - }, - "mode": "000644", - "owner": "root", - "group": "root" - }, - "/var/www/html/index.html": { - "content": { - "Fn::Join": [ - "\n", - [ - "

Ok.

", - "stack = ", - { - "Ref": "AWS::StackName" - }, - " ", - { - "Ref": "AWS::StackId" - } - ] - ] - }, - "mode": "000644", - "owner": "root", - "group": "root" - }, - "/var/www/html/status/index.html": { - "content": "ok", - "mode": "000644", - "owner": "root", - "group": "root" - }, - "/etc/cfn/cfn-hup.conf": { - "content": { - "Fn::Join": [ - "", - [ - "[main]\n", - "stack=", - { - "Ref": "AWS::StackId" - }, - "\n", - "region=", - { - "Ref": "AWS::Region" - }, - "\n" - ] - ] - }, - "mode": "000400", - "owner": "root", - "group": "root" - }, - "/etc/cfn/hooks.d/cfn-auto-reloader.conf": { - "content": { - "Fn::Join": [ - "", - [ - "[cfn-auto-reloader-hook]\n", - "triggers=post.update\n", - "path=Resources.LaunchConfig.Metadata.AWS::CloudFormation::Init\n", - "action=/opt/aws/bin/cfn-init -v ", - " --stack ", - { - "Ref": "AWS::StackName" - }, - " --resource LaunchConfig ", - " --region ", - { - "Ref": "AWS::Region" - }, - "\n", - "runas=root\n" - ] - ] - } - } - }, - "services": { - "sysvinit": { - "httpd": { - "enabled": "true", - "ensureRunning": "true" - }, - "cfn-hup": { - "enabled": "true", - "ensureRunning": "true", - "files": [ - "/etc/cfn/cfn-hup.conf", - "/etc/cfn/hooks.d/cfn-auto-reloader.conf" - ] - } - } - } - } - } - }, - "Properties": { - "KeyName": { - "Ref": "KeyPair" - }, - "ImageId": { - "Fn::FindInMap": [ - "AWSRegionArch2AMI", - { - "Ref": "AWS::Region" - }, - { - "Fn::FindInMap": [ - "AWSInstanceType2Arch", - { - "Ref": "ParmaInstanceType" - }, - "Arch" - ] - } - ] - }, - "SecurityGroups": [ - { - "Ref": "ParmaInstanceSecurityGroup" - } - ], - "InstanceType": { - "Ref": "ParmaInstanceType" - }, - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#!/bin/bash -xe\n", - "yum update -y aws-cfn-bootstrap\n", - "# Get the launch config for this set.\n", - "/opt/aws/bin/cfn-init -v ", - " --stack ", - { - "Ref": "AWS::StackName" - }, - " --resource ParmaLaunchConfig ", - " --region ", - { - "Ref": "AWS::Region" - }, - "\n", - "# Tell the server group that it's ok to continue.\n", - "/opt/aws/bin/cfn-signal -e $? ", - " --stack ", - { - "Ref": "AWS::StackName" - }, - " --resource ParmaWebServerGroup ", - " --region ", - { - "Ref": "AWS::Region" - }, - "\n" - ] - ] - } - } - } - }, - "ParmaInstanceSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "Enable SSH & HTTP access", - "SecurityGroupIngress": [ - { - "IpProtocol": "tcp", - "FromPort": "22", - "ToPort": "22", - "CidrIp": "0.0.0.0/0" - }, - { - "IpProtocol": "tcp", - "FromPort": "80", - "ToPort": "80", - "CidrIp": "0.0.0.0/0" - } - ] - } - }, - "ParmaELB": { - "Type": "AWS::ElasticLoadBalancing::LoadBalancer", - "Properties": { - "AvailabilityZones": { - "Fn::GetAZs": "" - }, - "CrossZone": "true", - "Listeners": [ - { - "LoadBalancerPort": "80", - "InstancePort": "80", - "Protocol": "HTTP" - } - ], - "HealthCheck": { - "Target": "http:80/status/", - "HealthyThreshold": "3", - "UnhealthyThreshold": "5", - "Interval": "30", - "Timeout": "5" - }, - "ConnectionDrainingPolicy": { - "Enabled": "true", - "Timeout": "300" - } - } - } - }, - "Outputs": { - "URL": { - "Description": "URL of the website", - "Value": { - "Fn::Join": [ - "", - [ - "http://", - { - "Fn::GetAtt": [ - "ParmaELB", - "DNSName" - ] - } - ] - ] - } - }, - "arn": { - "Description": "ARN", - "Value": { - "Ref": "AWS::StackId" - } - } - } -} \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index 39bb8a2..868224a 100644 --- a/serverless.yml +++ b/serverless.yml @@ -8,6 +8,7 @@ package: exclude: - node_modules/** - ardenv/** + - tests/** include: - ardere/** diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..68640ee --- /dev/null +++ b/setup.cfg @@ -0,0 +1,9 @@ +[nosetests] +verbose=True +verbosity=1 +detailed-errors=True +with-coverage=True +cover-erase=True +cover-package=ardere +cover-tests=True +cover-inclusive=True diff --git a/src/node/lib/start-test-plan.js b/src/node/lib/start-test-plan.js deleted file mode 100644 index 033ef4e..0000000 --- a/src/node/lib/start-test-plan.js +++ /dev/null @@ -1,25 +0,0 @@ -"use strict" -const AWS = require('aws-sdk'); -const path = require('path'); -const util = require('util'); - -const START_FN = 'RUN.TXT' - -exports.handler = (event, context, callback) => { - let s3 = new AWS.S3({apiVersion: '2006-03-01'}); - let key = path.posix.join(event.plan_uuid, event.run_uuid, START_FN); - let now = Math.round(new Date().getTime() / 1000); - let s3_url = util.format("s3://%s/%s", event.bucket, key); - - s3.upload({ - Bucket: event.bucket, - Key: key, - Body: new Buffer(now.toString(), 'binary'), - ACL: 'public-read' - }).promise().then((data) => { - console.log("Uploaded: ", s3_url); - callback(); - }).catch((err) => { - callback(err, "Error while uploading to: ", s3_url); - }); -} diff --git a/src/node/package.json b/src/node/package.json deleted file mode 100644 index cab51bb..0000000 --- a/src/node/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "ardere-lambda-functions", - "version": "0.0.1", - "description": "Ardere AWS Lambda functions", - "engines": { - "node": "^4.3.x", - "npm": "^3.5.x" - }, - "scripts": { - "test": "mocha" - }, - "author": "Mozilla Loads Project", - "license": "MPL", - "devDependencies": { - "aws-sdk-mock": "^1.6.1", - "chai": "^3.5.0", - "lambda-tester": "^2.7.0", - "mocha": "^3.2.0" - } -} diff --git a/src/node/test/test-start-test-plan.js b/src/node/test/test-start-test-plan.js deleted file mode 100644 index 188af16..0000000 --- a/src/node/test/test-start-test-plan.js +++ /dev/null @@ -1,34 +0,0 @@ -"use strict" -const AWS = require('aws-sdk-mock'); -const LambdaTester = require('lambda-tester'); - -const assert = require('chai').assert; -const handler = require('../lib/start-test-plan.js').handler; - -describe('start test plan', () => { - let upload; - - before(() => { - upload = AWS.mock('S3', 'upload', (params, callback) => { - callback(null, "Uploaded"); - }); - }); - - after(() => { - AWS.restore('S3'); - }); - - it('uploaded start file to s3', () => { - return LambdaTester(handler) - .event({bucket: 'foo', plan_uuid: 'abcd', run_uuid: 'efgh'}) - .expectResult((result) => { - assert(upload.stub.calledOnce); - let params = upload.stub.args[0][0]; - assert(params.Bucket == 'foo'); - assert(params.Key == 'abcd/efgh/RUN.TXT'); - assert(params.ACL.includes('public')); - let start_time = parseInt(params.Body.toString()); - assert(start_time <= Math.ceil(new Date().getTime() / 1000)); - }); - }); -}); diff --git a/templates/ecr.cloudform.template b/templates/ecr.cloudform.template deleted file mode 100644 index ac6e6e2..0000000 --- a/templates/ecr.cloudform.template +++ /dev/null @@ -1,742 +0,0 @@ -{ - "AWSTemplateFormatVersion":"2010-09-09", - "Metadata": { - "Comment": { - "Fn::Join": ["\n", [ - "This template is a barebones example of creating an EC2 cluster using a docker image pulled", - "from an AWS repository. This template is pre-populated with VPC and Subnet IDs for the ", - "`us-west-2` AZ. Abuse as you see fit." - ]] - } - }, - "Parameters":{ - "ImageURL": { - "Type": "String", - "Description": "URL for the repository to generate instances", - "Default": "927034868273.dkr.ecr.us-west-2.amazonaws.com/jrc_demo:latest" - }, - "KeyName":{ - "Type":"AWS::EC2::KeyPair::KeyName", - "Description":"Name of an existing EC2 KeyPair to enable SSH access to the ECS instances." - }, - "VpcId":{ - "Type":"AWS::EC2::VPC::Id", - "Description":"Select a VPC that allows instances to access the Internet.", - "Default": "vpc-9037f3f5" - }, - "SubnetID":{ - "Type":"List", - "Description":"Select at two subnets in your selected VPC.", - "Default": "subnet-4228de35,subnet-63f72706,subnet-b2c3d5f4" - }, - "DesiredCapacity":{ - "Type":"Number", - "Default":"1", - "Description":"Number of instances to launch in your ECS cluster." - }, - "MaxSize":{ - "Type":"Number", - "Default":"1", - "Description":"Maximum number of instances that can be launched in your ECS cluster." - }, - "InstanceType":{ - "Description":"EC2 instance type", - "Type":"String", - "Default":"t2.micro", - "AllowedValues":[ - "t2.micro", - "t2.small", - "t2.medium", - "t2.large", - "m3.medium", - "m3.large", - "m3.xlarge", - "m3.2xlarge", - "m4.large", - "m4.xlarge", - "m4.2xlarge", - "m4.4xlarge", - "m4.10xlarge", - "c4.large", - "c4.xlarge", - "c4.2xlarge", - "c4.4xlarge", - "c4.8xlarge", - "c3.large", - "c3.xlarge", - "c3.2xlarge", - "c3.4xlarge", - "c3.8xlarge", - "r3.large", - "r3.xlarge", - "r3.2xlarge", - "r3.4xlarge", - "r3.8xlarge", - "i2.xlarge", - "i2.2xlarge", - "i2.4xlarge", - "i2.8xlarge" - ], - "ConstraintDescription":"Please choose a valid instance type." - } - }, - "Mappings":{ - "AWSRegionToAMI":{ - "us-east-1":{ - "AMIID":"ami-eca289fb" - }, - "us-east-2":{ - "AMIID":"ami-446f3521" - }, - "us-west-1":{ - "AMIID":"ami-9fadf8ff" - }, - "us-west-2":{ - "AMIID":"ami-7abc111a" - }, - "eu-west-1":{ - "AMIID":"ami-a1491ad2" - }, - "eu-central-1":{ - "AMIID":"ami-54f5303b" - }, - "ap-northeast-1":{ - "AMIID":"ami-9cd57ffd" - }, - "ap-southeast-1":{ - "AMIID":"ami-a900a3ca" - }, - "ap-southeast-2":{ - "AMIID":"ami-5781be34" - } - } - }, - "Resources":{ - "ECSCluster":{ - "Type":"AWS::ECS::Cluster" - }, - "EcsSecurityGroup":{ - "Type":"AWS::EC2::SecurityGroup", - "Properties":{ - "GroupDescription":"ECS Security Group", - "VpcId":{ - "Ref":"VpcId" - } - } - }, - "EcsSecurityGroupHTTPinbound":{ - "Type":"AWS::EC2::SecurityGroupIngress", - "Properties":{ - "GroupId":{ - "Ref":"EcsSecurityGroup" - }, - "IpProtocol":"tcp", - "FromPort":"80", - "ToPort":"80", - "CidrIp":"0.0.0.0/0" - } - }, - "EcsSecurityGroupSSHinbound":{ - "Type":"AWS::EC2::SecurityGroupIngress", - "Properties":{ - "GroupId":{ - "Ref":"EcsSecurityGroup" - }, - "IpProtocol":"tcp", - "FromPort":"22", - "ToPort":"22", - "CidrIp":"0.0.0.0/0" - } - }, - "EcsSecurityGroupALBports":{ - "Type":"AWS::EC2::SecurityGroupIngress", - "Properties":{ - "GroupId":{ - "Ref":"EcsSecurityGroup" - }, - "IpProtocol":"tcp", - "FromPort":"31000", - "ToPort":"61000", - "SourceSecurityGroupId":{ - "Ref":"EcsSecurityGroup" - } - } - }, - "CloudwatchLogsGroup":{ - "Type":"AWS::Logs::LogGroup", - "Properties":{ - "LogGroupName":{ - "Fn::Join":[ - "-", - [ - "ECSLogGroup", - { - "Ref":"AWS::StackName" - } - ] - ] - }, - "RetentionInDays":14 - } - }, - "ECSTaskDefinition":{ - "Type":"AWS::ECS::TaskDefinition", - "Metadata": { - "Comment": {"Fn::Join": ["\n", [ - "Container images that are stored as AWS Repositories require additional", - "AMI privileges. See ECSServiceRole. ContainerDefinitions contains the list of", - "containers that you want to have. Docker standard containers like busybox or ", - "httpd do not need extra permissions to load. AWS repo'd containers do." - ]]} - }, - "Properties":{ - "Family":{ - "Fn::Join":[ - "", - [ - { - "Ref":"AWS::StackName" - }, - "-ardere-ecs-app" - ] - ] - }, - "ContainerDefinitions":[ - { - "Name": "ardere-demo-container", - "Cpu": 10, - "Essential": "true", - "Image": { - "Ref": "ImageURL" - }, - "Memory": "300", - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "CloudwatchLogsGroup" - }, - "awslogs-region":{ - "Ref": "AWS::Region" - }, - "awslogs-stream-prefix": "ardere-demo-container" - } - }, - "MountPoints":[ - { - "ContainerPath":"/usr/local/apache2/htdocs", - "SourceVolume":"my-vol" - } - ], - "PortMappings":[ - { - "ContainerPort":80 - } - ] - } - ], - "Volumes":[ - { - "Name":"my-vol" - } - ] - } - }, - "ECSALB":{ - "Type":"AWS::ElasticLoadBalancingV2::LoadBalancer", - "Metadata": { - "Comment": { - "Fn::Join": ["\n", [ - "ECR prefers using ALBs over ELBs." - ]] - } - }, - "Properties":{ - "Name":"ECSALB", - "Scheme":"internet-facing", - "LoadBalancerAttributes":[ - { - "Key":"idle_timeout.timeout_seconds", - "Value":"30" - } - ], - "Subnets":{ - "Ref":"SubnetID" - }, - "SecurityGroups":[ - { - "Ref":"EcsSecurityGroup" - } - ] - } - }, - "ALBListener":{ - "Type":"AWS::ElasticLoadBalancingV2::Listener", - "DependsOn":"ECSServiceRole", - "Properties":{ - "DefaultActions":[ - { - "Type":"forward", - "TargetGroupArn":{ - "Ref":"ECSTG" - } - } - ], - "LoadBalancerArn":{ - "Ref":"ECSALB" - }, - "Port":"80", - "Protocol":"HTTP" - } - }, - "ECSALBListenerRule":{ - "Type":"AWS::ElasticLoadBalancingV2::ListenerRule", - "DependsOn":"ALBListener", - "Properties":{ - "Actions":[ - { - "Type":"forward", - "TargetGroupArn":{ - "Ref":"ECSTG" - } - } - ], - "Conditions":[ - { - "Field":"path-pattern", - "Values":[ - "/" - ] - } - ], - "ListenerArn":{ - "Ref":"ALBListener" - }, - "Priority":1 - } - }, - "ECSTG":{ - "Type":"AWS::ElasticLoadBalancingV2::TargetGroup", - "DependsOn":"ECSALB", - "Properties":{ - "HealthCheckIntervalSeconds":10, - "HealthCheckPath":"/status/", - "HealthCheckProtocol":"HTTP", - "HealthCheckTimeoutSeconds":5, - "HealthyThresholdCount":2, - "Name":"ECSTG", - "Port":80, - "Protocol":"HTTP", - "UnhealthyThresholdCount":2, - "VpcId":{ - "Ref":"VpcId" - } - } - }, - "ECSAutoScalingGroup":{ - "Type":"AWS::AutoScaling::AutoScalingGroup", - "Properties":{ - "VPCZoneIdentifier":{ - "Ref":"SubnetID" - }, - "LaunchConfigurationName":{ - "Ref":"ContainerInstances" - }, - "MinSize":"1", - "MaxSize":{ - "Ref":"MaxSize" - }, - "DesiredCapacity":{ - "Ref":"DesiredCapacity" - } - }, - "CreationPolicy":{ - "ResourceSignal":{ - "Timeout":"PT15M" - } - }, - "UpdatePolicy":{ - "AutoScalingReplacingUpdate":{ - "WillReplace":"true" - } - } - }, - "ContainerInstances":{ - "Type":"AWS::AutoScaling::LaunchConfiguration", - "Metadata": { - "Comment": { - "Fn::Join": [ - "\n", - [ - "This is the ECS running the docker images. There is a shell available." - ] - ] - } - }, - "Properties":{ - "ImageId":{ - "Fn::FindInMap":[ - "AWSRegionToAMI", - { - "Ref":"AWS::Region" - }, - "AMIID" - ] - }, - "SecurityGroups":[ - { - "Ref":"EcsSecurityGroup" - } - ], - "InstanceType":{ - "Ref":"InstanceType" - }, - "IamInstanceProfile":{ - "Ref":"EC2InstanceProfile" - }, - "KeyName":{ - "Ref":"KeyName" - }, - "UserData":{ - "Fn::Base64":{ - "Fn::Join":[ - "", - [ - "#!/bin/bash -xe\n", - "echo ECS_CLUSTER=", - { - "Ref":"ECSCluster" - }, - " >> /etc/ecs/ecs.config\n", - "yum install -y aws-cfn-bootstrap\n", - "/opt/aws/bin/cfn-signal -e $? ", - " --stack ", - { - "Ref":"AWS::StackName" - }, - " --resource ECSAutoScalingGroup ", - " --region ", - { - "Ref":"AWS::Region" - }, - "\n" - ] - ] - } - } - } - }, - "ECSService":{ - "Type":"AWS::ECS::Service", - "DependsOn":"ALBListener", - "Properties":{ - "Cluster":{ - "Ref":"ECSCluster" - }, - "DesiredCount":"1", - "LoadBalancers":[ - { - "ContainerName":"ardere-demo-container", - "ContainerPort":"80", - "TargetGroupArn":{ - "Ref":"ECSTG" - } - } - ], - "Role":{ - "Ref":"ECSServiceRole" - }, - "TaskDefinition":{ - "Ref": "ECSTaskDefinition" - } - } - }, - "ECSServiceRole":{ - "Type":"AWS::IAM::Role", - "Properties":{ - "RoleName": {"Fn::Join": ["-", [ "Adere", "ECSRole", {"Ref": "AWS::Region"}]]}, - "AssumeRolePolicyDocument":{ - "Statement":[ - { - "Effect":"Allow", - "Principal":{ - "Service":[ - "ecs.amazonaws.com" - ] - }, - "Action":[ - "sts:AssumeRole" - ] - } - ] - }, - "Path":"/", - "Policies":[ - { - "PolicyName":"ecs-service", - "PolicyDocument":{ - "Statement":[ - { - "Effect":"Allow", - "Action":[ - "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", - "elasticloadbalancing:DeregisterTargets", - "elasticloadbalancing:Describe*", - "elasticloadbalancing:RegisterInstancesWithLoadBalancer", - "elasticloadbalancing:RegisterTargets", - "ec2:Describe*", - "ec2:AuthorizeSecurityGroupIngress", - "ecr:GetAuthorizationToken", - "ecr:" - ], - "Resource":"*" - } - ] - } - }, - { - "PolicyName": "ecr-service", - "PolicyDocument": { - "Statement":[ - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken" - ], - "Resource": "arn:aws:ecr:us-west-2:927034868273:repository/*" - } - ] - } - } - ] - } - }, - "ServiceScalingTarget":{ - "Type":"AWS::ApplicationAutoScaling::ScalableTarget", - "DependsOn": "ECSService", - "Properties":{ - "MaxCapacity":2, - "MinCapacity":1, - "ResourceId":{ - "Fn::Join":[ - "", - [ - "service/", - { - "Ref":"ECSCluster" - }, - "/", - { - "Fn::GetAtt":[ - "ECSService", - "Name" - ] - } - ] - ] - }, - "RoleARN":{ - "Fn::GetAtt":[ - "AutoscalingRole", - "Arn" - ] - }, - "ScalableDimension":"ecs:service:DesiredCount", - "ServiceNamespace":"ecs" - } - }, - "ServiceScalingPolicy":{ - "Type":"AWS::ApplicationAutoScaling::ScalingPolicy", - "Properties":{ - "PolicyName":"AStepPolicy", - "PolicyType":"StepScaling", - "ScalingTargetId":{ - "Ref":"ServiceScalingTarget" - }, - "StepScalingPolicyConfiguration":{ - "AdjustmentType":"PercentChangeInCapacity", - "Cooldown":60, - "MetricAggregationType":"Average", - "StepAdjustments":[ - { - "MetricIntervalLowerBound":0, - "ScalingAdjustment":200 - } - ] - } - } - }, - "ALB500sAlarmScaleUp":{ - "Type":"AWS::CloudWatch::Alarm", - "Properties":{ - "EvaluationPeriods":"1", - "Statistic":"Average", - "Threshold":"10", - "AlarmDescription":"Alarm if our ALB generates too many HTTP 500s.", - "Period":"60", - "AlarmActions":[ - { - "Ref":"ServiceScalingPolicy" - } - ], - "Namespace":"AWS/ApplicationELB", - "Dimensions":[ - { - "Name":"ECSService", - "Value":{ - "Ref": "ECSService" - } - } - ], - "ComparisonOperator":"GreaterThanThreshold", - "MetricName":"HTTPCode_ELB_5XX_Count" - } - }, - "EC2Role":{ - "Type":"AWS::IAM::Role", - "Metadata": { - "Comment": { - "Fn::Join": ["\n", [ - "The `ecr:*` permissions are REALLY REALLY IMPORTANT.", - "If not present, the build will hang. I have no idea", - "why they're not included in the original template, but they", - "are listed at https://github.com/awslabs/ecs-refarch-cloudformation/" - ] - ] - } - }, - "Properties":{ - "AssumeRolePolicyDocument":{ - "Statement":[ - { - "Effect":"Allow", - "Principal":{ - "Service":[ - "ec2.amazonaws.com" - ] - }, - "Action":[ - "sts:AssumeRole" - ] - } - ] - }, - "Path":"/", - "Policies":[ - { - "PolicyName":"ecs-service", - "PolicyDocument":{ - "Statement":[ - { - "Effect":"Allow", - "Action":[ - "ecs:CreateCluster", - "ecs:DeregisterContainerInstance", - "ecs:DiscoverPollEndpoint", - "ecs:Poll", - "ecs:RegisterContainerInstance", - "ecs:StartTelemetrySession", - "ecs:Submit*", - "logs:CreateLogStream", - "logs:PutLogEvents", - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer", - "ecr:GetAuthorizationToken" - ], - "Resource":"*" - } - ] - } - } - ] - } - }, - "AutoscalingRole":{ - "Type":"AWS::IAM::Role", - "Properties":{ - "AssumeRolePolicyDocument":{ - "Statement":[ - { - "Effect":"Allow", - "Principal":{ - "Service":[ - "application-autoscaling.amazonaws.com" - ] - }, - "Action":[ - "sts:AssumeRole" - ] - } - ] - }, - "Path":"/", - "Policies":[ - { - "PolicyName":"service-autoscaling", - "PolicyDocument":{ - "Statement":[ - { - "Effect":"Allow", - "Action":[ - "application-autoscaling:*", - "cloudwatch:DescribeAlarms", - "cloudwatch:PutMetricAlarm", - "ecs:DescribeServices", - "ecs:UpdateService" - ], - "Resource":"*" - } - ] - } - } - ] - } - }, - "EC2InstanceProfile":{ - "Type":"AWS::IAM::InstanceProfile", - "Properties":{ - "Path":"/", - "Roles":[ - { - "Ref":"EC2Role" - } - ] - } - } - }, - "Outputs":{ - "ecsservice":{ - "Value":{ - "Ref": "ECSService" - } - }, - "ecscluster":{ - "Value":{ - "Ref":"ECSCluster" - } - }, - "ECSALB":{ - "Description":"URL", - "Value":{ - "Fn::Join":[ - "", - [ "http://", - { - "Fn::GetAtt":[ - "ECSALB", - "DNSName" - ] - }, - "/" - ] - ] - } - }, - "taskdef":{ - "Value":{ - "Ref": "ECSTaskDefinition" - } - } - } -} diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..26bd6a9 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,6 @@ +futures==3.0.5 +typing==3.5.3.0 +nose==1.3.7 +mock==2.0.0 +coverage==4.3.4 +boto3==1.4.4 diff --git a/tests/fixtures.py b/tests/fixtures.py index 4c028bb..f51c662 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -8,12 +8,43 @@ "name": "TestCluster", "instance_count": 1, "instance_type": "t2.medium", - "run_max_time": 40, - "cpu_units": 2030, + "run_max_time": 140, + "cpu_units": 2048, "container_name": "bbangert/ap-loadtester:latest", - "additional_command_args": "./apenv/bin/aplt_testplan wss://autopush.stage.mozaws.net 'aplt.scenarios:notification_forever,1000,1,0'", - "docker_series": "push_tester" + "additional_command_args": "./apenv/bin/aplt_testplan wss://autopush.stage.mozaws.net 'aplt.scenarios:notification_forever,1000,1,0'" } ] } +""" + +future_hypothetical_test=""" +{ + "name": "TestCluster", + "instance_count": 1, + "instance_type": "t2.medium", + "run_max_time": 140, + "cpu_units": 2048, + "container_name": "bbangert/pushgo:1.5rc4", + "port_mapping": "8080:8090,8081:8081,3000:3000,8082:8082", + "load_balancer": { + "env_var": "TEST_CLUSTER", + "ping_path": "/status/health", + "ping_port": 8080, + "ping_protocol": "http", + "listeners": [ + { + "listen_protocol": "ssl", + "listen_port": 443, + "backend_protocol": "tcp", + "backend_port": 8080 + }, + { + "listen_protocol": "https", + "listen_port": 9000, + "backend_protocol": "http", + "backend_port": 8090 + } + ] + } +} """ \ No newline at end of file diff --git a/tests/test_aws.py b/tests/test_aws.py new file mode 100644 index 0000000..510a032 --- /dev/null +++ b/tests/test_aws.py @@ -0,0 +1,269 @@ +import json +import os +import time +import unittest + +import mock +from nose.tools import eq_, ok_ + +from tests import fixtures + + +class TestECSManager(unittest.TestCase): + def _make_FUT(self, plan=None): + from ardere.aws import ECSManager + self.boto_mock = mock.Mock() + ECSManager.boto = self.boto_mock + if not plan: + plan = json.loads(fixtures.sample_basic_test_plan) + return ECSManager(plan) + + def test_init(self): + ecs = self._make_FUT() + eq_(ecs._plan["plan_run_uuid"], ecs._plan_uuid) + + def test_ready_file(self): + ecs = self._make_FUT() + os.environ["s3_ready_bucket"] = "test_bucket" + ready_filename = ecs.s3_ready_file + ok_("test_bucket" in ready_filename) + ok_(ecs._plan_uuid in ready_filename) + + def test_query_active(self): + mock_paginator = mock.Mock() + mock_paginator.paginate.return_value = [ + {"Reservations": [ + { + "Instances": [ + { + "State": { + "Code": 16 + }, + "InstanceType": "t2.medium" + } + ] + } + ]} + ] + + ecs = self._make_FUT() + ecs._ec2_client.get_paginator.return_value = mock_paginator + instance_dct = ecs.query_active_instances() + eq_(len(instance_dct.values()), 1) + + def test_calculate_missing_instances(self): + ecs = self._make_FUT() + result = ecs.calculate_missing_instances( + desired={"t2.medium": 2}, current={"t2.medium": 1} + ) + eq_(result, {"t2.medium": 1}) + + def test_request_instances(self): + os.environ["ecs_profile"] = "arn:something:fantastic:::" + instances = { + "t2.medium": 10 + } + ecs = self._make_FUT() + ecs._ec2_client.run_instances.return_value = { + "Instances": [{"InstanceId": 12345}] + } + ecs.request_instances(instances) + ecs._ec2_client.create_tags.assert_called_with( + Resources=[12345], Tags=[ + {'Value': 'ardere', 'Key': 'Owner'}, + {'Value': u'ardere-test', 'Key': 'ECSCluster'} + ] + ) + + def test_create_service(self): + os.environ["s3_ready_bucket"] = "test_bucket" + ecs = self._make_FUT() + + step = ecs._plan["steps"][0] + + # Setup mocks + ecs._ecs_client.register_task_definition.return_value = { + "taskDefinition": { + "taskDefinitionArn": "arn:of:some:task::" + } + } + ecs._ecs_client.create_service.return_value = { + "service": {"serviceArn": "arn:of:some:service::"} + } + + ecs.create_service(step) + + eq_(step["serviceArn"], "arn:of:some:service::") + ecs._ecs_client.register_task_definition.assert_called() + + def test_create_services(self): + ecs = self._make_FUT() + ecs.create_service = mock.Mock() + ecs.create_services(ecs._plan["steps"]) + ecs.create_service.assert_called() + + def test_service_ready_true(self): + ecs = self._make_FUT() + step = ecs._plan["steps"][0] + + ecs._ecs_client.describe_services.return_value = { + "services": [{ + "deployments": [{ + "desiredCount": 2, + "runningCount": 2 + }] + }] + } + + result = ecs.service_ready(step) + eq_(result, True) + + def test_service_not_known_yet(self): + ecs = self._make_FUT() + step = ecs._plan["steps"][0] + + ecs._ecs_client.describe_services.return_value = { + "services": [] + } + + result = ecs.service_ready(step) + eq_(result, False) + + def test_all_services_ready(self): + ecs = self._make_FUT() + ecs.service_ready = mock.Mock() + + ecs.all_services_ready(ecs._plan["steps"]) + ecs.service_ready.assert_called() + + def test_stop_finished_service_stopped(self): + ecs = self._make_FUT() + ecs._ecs_client.update_service = mock.Mock() + step = ecs._plan["steps"][0] + step["service_status"] = "STARTED" + past = time.time() - 400 + ecs.stop_finished_service(past, step) + ecs._ecs_client.update_service.assert_called() + eq_(step["service_status"], "STOPPED") + + def test_stop_finished_service_stop_already_stopped(self): + ecs = self._make_FUT() + ecs._ecs_client.update_service = mock.Mock() + step = ecs._plan["steps"][0] + step["service_status"] = "STOPPED" + past = time.time() - 400 + ecs.stop_finished_service(past, step) + ecs._ecs_client.update_service.assert_not_called() + eq_(step["service_status"], "STOPPED") + + def test_stop_finished_service_still_running(self): + ecs = self._make_FUT() + ecs._ecs_client.update_service = mock.Mock() + step = ecs._plan["steps"][0] + step["service_status"] = "STARTED" + past = time.time() - 100 + ecs.stop_finished_service(past, step) + ecs._ecs_client.update_service.assert_not_called() + eq_(step["service_status"], "STARTED") + + def test_stop_finished_services(self): + ecs = self._make_FUT() + ecs.stop_finished_service = mock.Mock() + + past = time.time() - 100 + ecs.stop_finished_services(past, ecs._plan["steps"]) + ecs.stop_finished_service.assert_called() + + def test_shutdown_plan(self): + mock_paginator = mock.Mock() + mock_paginator.paginate.return_value = [ + {"serviceArns": ["arn:123:::", "arn:456:::"]} + ] + + ecs = self._make_FUT() + ecs._ecs_client.get_paginator.return_value = mock_paginator + ecs._ecs_client.describe_task_definition.return_value = { + "taskDefinition": {"taskDefinitionArn": "arn:task:::"} + } + + ecs.shutdown_plan(ecs._plan["steps"]) + ecs._ecs_client.deregister_task_definition.assert_called() + ecs._ecs_client.delete_service.assert_called() + + def test_shutdown_plan_update_error(self): + from botocore.exceptions import ClientError + + mock_paginator = mock.Mock() + mock_paginator.paginate.return_value = [ + {"serviceArns": ["arn:123:::", "arn:456:::"]} + ] + + ecs = self._make_FUT() + ecs._ecs_client.get_paginator.return_value = mock_paginator + ecs._ecs_client.describe_task_definition.return_value = { + "taskDefinition": {"taskDefinitionArn": "arn:task:::"} + } + ecs._ecs_client.update_service.side_effect = ClientError( + {"Error": {}}, "some_op" + ) + + ecs.shutdown_plan(ecs._plan["steps"]) + ecs._ecs_client.delete_service.assert_not_called() + + def test_shutdown_plan_describe_error(self): + from botocore.exceptions import ClientError + + mock_paginator = mock.Mock() + mock_paginator.paginate.return_value = [ + {"serviceArns": ["arn:123:::", "arn:456:::"]} + ] + + ecs = self._make_FUT() + ecs._plan["steps"] = ecs._plan["steps"][:1] + ecs._ecs_client.get_paginator.return_value = mock_paginator + ecs._ecs_client.describe_task_definition.side_effect = ClientError( + {"Error": {}}, "some_op" + ) + + ecs.shutdown_plan(ecs._plan["steps"]) + ecs._ecs_client.deregister_task_definition.assert_not_called() + + def test_shutdown_plan_delete_error(self): + from botocore.exceptions import ClientError + + mock_paginator = mock.Mock() + mock_paginator.paginate.return_value = [ + {"serviceArns": ["arn:123:::", "arn:456:::"]} + ] + + ecs = self._make_FUT() + ecs._ecs_client.get_paginator.return_value = mock_paginator + ecs._ecs_client.describe_task_definition.return_value = { + "taskDefinition": {"taskDefinitionArn": "arn:task:::"} + } + ecs._ecs_client.delete_service.side_effect = ClientError( + {"Error": {}}, "some_op" + ) + + ecs.shutdown_plan(ecs._plan["steps"]) + ecs._ecs_client.delete_service.assert_called() + + def test_shutdown_plan_deregister_error(self): + from botocore.exceptions import ClientError + + mock_paginator = mock.Mock() + mock_paginator.paginate.return_value = [ + {"serviceArns": ["arn:123:::", "arn:456:::"]} + ] + + ecs = self._make_FUT() + ecs._ecs_client.get_paginator.return_value = mock_paginator + ecs._ecs_client.describe_task_definition.return_value = { + "taskDefinition": {"taskDefinitionArn": "arn:task:::"} + } + ecs._ecs_client.deregister_task_definition.side_effect = ClientError( + {"Error": {}}, "some_op" + ) + + ecs.shutdown_plan(ecs._plan["steps"]) + ecs._ecs_client.delete_service.assert_called() diff --git a/tests/test_handler.py b/tests/test_handler.py new file mode 100644 index 0000000..f0d4f44 --- /dev/null +++ b/tests/test_handler.py @@ -0,0 +1,146 @@ +import json +import os +import time +import unittest +import uuid + +import mock +from botocore.exceptions import ClientError +from nose.tools import eq_, assert_raises + +from tests import fixtures + +class TestHandler(unittest.TestCase): + + def setUp(self): + self.plan = json.loads(fixtures.sample_basic_test_plan) + self.mock_ecs = mock.Mock() + self._patcher = mock.patch("ardere.handler.ECSManager") + mock_manager = self._patcher.start() + mock_manager.return_value = self.mock_ecs + + def tearDown(self): + self._patcher.stop() + + def test_build_instance_map(self): + from ardere.handler import _build_instance_map + + result = _build_instance_map(self.plan) + eq_(len(result), 1) + eq_(result, {"t2.medium": 1}) + + def test_find_test_plan_duration(self): + from ardere.handler import _find_test_plan_duration + + result = _find_test_plan_duration(self.plan) + eq_(result, 140) + + def test_populate_missing_instances(self): + from ardere.handler import populate_missing_instances + + populate_missing_instances(self.plan, {}) + self.mock_ecs.query_active_instances.assert_called() + self.mock_ecs.request_instances.assert_called() + + def test_create_ecs_services(self): + from ardere.handler import create_ecs_services + + create_ecs_services(self.plan, {}) + self.mock_ecs.create_services.assert_called_with(self.plan["steps"]) + + def test_wait_for_cluster_ready_not_ready(self): + from ardere.handler import wait_for_cluster_ready + from ardere.exceptions import ServicesStartingException + + self.mock_ecs.all_services_ready.return_value = False + assert_raises(ServicesStartingException, wait_for_cluster_ready, + self.plan, {}) + + def test_wait_for_cluster_ready_all_ready(self): + from ardere.handler import wait_for_cluster_ready + from ardere.exceptions import ServicesStartingException + + self.mock_ecs.all_services_ready.return_value = True + wait_for_cluster_ready(self.plan, {}) + self.mock_ecs.all_services_ready.assert_called() + + @mock.patch("ardere.handler.boto3") + def test_signal_cluster_start(self, mock_boto): + from ardere.handler import signal_cluster_start + + self.plan["plan_run_uuid"] = str(uuid.uuid4()) + + signal_cluster_start(self.plan, {}) + mock_boto.client.assert_called() + + @mock.patch("ardere.handler.boto3") + def test_check_for_cluster_done_not_done(self, mock_boto): + from ardere.handler import check_for_cluster_done + os.environ["s3_ready_bucket"] = "test_bucket" + mock_file = mock.Mock() + mock_file.get.return_value = {"Body": mock_file} + mock_file.read.return_value = "{}".format(int(time.time())-100).encode( + 'utf-8') + mock_s3_obj = mock.Mock() + mock_s3_obj.Object.return_value = mock_file + mock_boto.resource.return_value = mock_s3_obj + + self.plan["plan_run_uuid"] = str(uuid.uuid4()) + check_for_cluster_done(self.plan, {}) + + @mock.patch("ardere.handler.boto3") + def test_check_for_cluster_done_shutdown(self, mock_boto): + from ardere.handler import check_for_cluster_done + from ardere.exceptions import ShutdownPlanException + os.environ["s3_ready_bucket"] = "test_bucket" + mock_file = mock.Mock() + mock_file.get.return_value = {"Body": mock_file} + mock_file.read.return_value = "{}".format(int(time.time())-400).encode( + 'utf-8') + mock_s3_obj = mock.Mock() + mock_s3_obj.Object.return_value = mock_file + mock_boto.resource.return_value = mock_s3_obj + + self.plan["plan_run_uuid"] = str(uuid.uuid4()) + assert_raises(ShutdownPlanException, check_for_cluster_done, + self.plan, {}) + + @mock.patch("ardere.handler.boto3") + def test_check_for_cluster_done_object_error(self, mock_boto): + from ardere.handler import check_for_cluster_done + from ardere.exceptions import ShutdownPlanException + os.environ["s3_ready_bucket"] = "test_bucket" + mock_file = mock.Mock() + mock_file.get.return_value = {"Body": mock_file} + mock_file.read.return_value = "{}".format(int(time.time())-400).encode( + 'utf-8') + mock_s3_obj = mock.Mock() + mock_s3_obj.Object.side_effect = ClientError( + {"Error": {}}, None + ) + mock_boto.resource.return_value = mock_s3_obj + + self.plan["plan_run_uuid"] = str(uuid.uuid4()) + assert_raises(ShutdownPlanException, check_for_cluster_done, + self.plan, {}) + + @mock.patch("ardere.handler.boto3") + def test_cleanup_cluster(self, mock_boto): + from ardere.handler import cleanup_cluster + self.plan["plan_run_uuid"] = str(uuid.uuid4()) + + cleanup_cluster(self.plan, {}) + mock_boto.resource.assert_called() + + @mock.patch("ardere.handler.boto3") + def test_cleanup_cluster_error(self, mock_boto): + from ardere.handler import cleanup_cluster + self.plan["plan_run_uuid"] = str(uuid.uuid4()) + + mock_s3 = mock.Mock() + mock_boto.resource.return_value = mock_s3 + mock_s3.Object.side_effect = ClientError( + {"Error": {}}, None + ) + cleanup_cluster(self.plan, {}) + mock_s3.Object.assert_called()