Merge pull request #61 from richford/region

Track region in config file

Makefile

@@ -15,4 +15,4 @@ test:
 devtest:
     # Unit testing with the -x option, aborts testing after first failure
     # Useful for development when tests are long
-	py.test -x --pyargs cloudknot --cov-report term-missing --cov=cloudknot --fulltrace
+	py.test -x --pyargs cloudknot --cov-report term-missing --cov=cloudknot

cloudknot/aws/base_classes.py

@@ -11,10 +11,16 @@
 
 from ..config import get_config_file
 
-__all__ = ["ResourceDoesNotExistException", "ResourceClobberedException",
-           "ResourceExistsException", "CannotDeleteResourceException",
-           "NamedObject", "ObjectWithArn", "ObjectWithUsernameAndMemory",
-           "clients", "wait_for_compute_environment", "wait_for_job_queue"]
+__all__ = [
+    "ResourceDoesNotExistException", "ResourceClobberedException",
+    "ResourceExistsException", "CannotDeleteResourceException",
+    "RegionException",
+    "NamedObject", "ObjectWithArn", "ObjectWithUsernameAndMemory",
+    "clients", "refresh_clients",
+    "wait_for_compute_environment", "wait_for_job_queue",
+    "get_region", "set_region",
+    "get_profile", "set_profile", "list_profiles",
+]
 
 mod_logger = logging.getLogger(__name__)
 
@@ -356,7 +362,7 @@ def __init__(self, message, resource_id):
         resource_id : string
             The resource ID (e.g. ARN, VPC-ID) of the requested resource
         """
-        super(ResourceDoesNotExistException, self).__init__(message)
+        super(ResourceClobberedException, self).__init__(message)
         self.resource_id = resource_id
 
 
@@ -377,6 +383,28 @@ def __init__(self, message, resource_id):
         self.resource_id = resource_id
 
 
+# noinspection PyPropertyAccess,PyAttributeOutsideInit
+class RegionException(Exception):
+    """Exception indicating that an AWS resource's region does not match
+    the current region"""
+    def __init__(self, resource_region):
+        """Initialize the Exception
+
+        Parameters
+        ----------
+        resource_region : string
+            The resource region
+        """
+        super(RegionException, self).__init__(
+            "This resource's region ({resource:s}) does not match the "
+            "current region ({current:s})".format(
+                resource=resource_region, current=get_region()
+            )
+        )
+        self.current_region = get_region()
+        self.resource_region = resource_region
+
+
 # noinspection PyPropertyAccess,PyAttributeOutsideInit
 class NamedObject(object):
     """Base class for building objects with name property"""
@@ -390,9 +418,11 @@ def __init__(self, name):
         """
         self._name = str(name)
         self._clobbered = False
+        self._region = get_region()
 
     name = property(operator.attrgetter('_name'))
     clobbered = property(operator.attrgetter('_clobbered'))
+    region = property(operator.attrgetter('_region'))
 
 
 # noinspection PyPropertyAccess,PyAttributeOutsideInit
@@ -412,9 +442,7 @@ def __init__(self, name):
         super(ObjectWithArn, self).__init__(name=name)
         self._arn = None
 
-    @property
-    def arn(self):
-        return self._arn
+    arn = property(operator.attrgetter('_arn'))
 
 
 # noinspection PyPropertyAccess,PyAttributeOutsideInit
@@ -432,11 +460,11 @@ def __init__(self, name, memory=32000, username='cloudknot-user'):
             Name of the object
 
         memory : int
-            memory (MiB) to be used for this job definition
+            memory (MiB) to be used for this resource
             Default: 32000
 
         username : string
-            username for be used for this job definition
+            username for be used for this resource
             Default: cloudknot-user
         """
         super(ObjectWithUsernameAndMemory, self).__init__(name=name)

cloudknot/aws/batch.py

@@ -8,10 +8,10 @@
 from collections import namedtuple
 
 from .base_classes import NamedObject, ObjectWithArn, \
-    ObjectWithUsernameAndMemory, clients, \
+    ObjectWithUsernameAndMemory, clients, RegionException, \
     ResourceExistsException, ResourceDoesNotExistException, \
     ResourceClobberedException, CannotDeleteResourceException, \
-    wait_for_job_queue
+    wait_for_job_queue, get_region
 from .ec2 import Vpc, SecurityGroup
 from .ecr import DockerRepo
 from .iam import IamRole
@@ -109,8 +109,9 @@ def __init__(self, arn=None, name=None, job_role=None, docker_image=None,
             self._arn = resource.arn
 
             # Add to config file
+            self._section_name = 'job-definitions ' + self.region
             cloudknot.config.add_resource(
-                'job-definitions', self.name, self.arn
+                self._section_name, self.name, self.arn
             )
 
             mod_logger.info(
@@ -295,7 +296,8 @@ def _create(self):
         arn = response['jobDefinitionArn']
 
         # Add this job def to the list of job definitions in the config file
-        cloudknot.config.add_resource('job-definitions', self.name, arn)
+        self._section_name = 'job-definitions ' + self.region
+        cloudknot.config.add_resource(self._section_name, self.name, arn)
 
         mod_logger.info('Created AWS batch job definition {name:s}'.format(
             name=self.name
@@ -310,10 +312,13 @@ def clobber(self):
         -------
         None
         """
+        if self.region != get_region():
+            raise RegionException(resource_region=self.region)
+
         clients['batch'].deregister_job_definition(jobDefinition=self.arn)
 
         # Remove this job def from the list of job defs in the config file
-        cloudknot.config.remove_resource('job-definitions', self.name)
+        cloudknot.config.remove_resource(self._section_name, self.name)
 
         # Set the clobbered parameter to True,
         # preventing subsequent method calls
@@ -475,8 +480,9 @@ def __init__(self, arn=None, name=None, batch_service_role=None,
             self._bid_percentage = resource.bid_percentage
             self._arn = resource.arn
 
+            self._section_name = 'compute-environments ' + self.region
             cloudknot.config.add_resource(
-                'compute-environments', self.name, self.arn
+                self._section_name, self.name, self.arn
             )
 
             mod_logger.info(
@@ -857,7 +863,8 @@ def _create(self):
         arn = response['computeEnvironmentArn']
 
         # Add this compute env to the list of compute envs in the config file
-        cloudknot.config.add_resource('compute-environments', self.name, arn)
+        self._section_name = 'compute-environments ' + self.region
+        cloudknot.config.add_resource(self._section_name, self.name, arn)
 
         mod_logger.info('Created compute environment {name:s}'.format(
             name=self.name
@@ -872,6 +879,9 @@ def clobber(self):
         -------
         None
         """
+        if self.region != get_region():
+            raise RegionException(resource_region=self.region)
+
         retry = tenacity.Retrying(
             wait=tenacity.wait_exponential(max=32),
             stop=tenacity.stop_after_delay(60),
@@ -925,9 +935,8 @@ def clobber(self):
                         resource_id=associated_queues
                     )
 
-        # Remove this compute env from the list of compute envs
-        # in config file
-        cloudknot.config.remove_resource('compute-environments', self.name)
+        # Remove this compute env from the list of compute envs in config file
+        cloudknot.config.remove_resource(self._section_name, self.name)
 
         # Set the clobbered parameter to True,
         # preventing subsequent method calls
@@ -1002,7 +1011,10 @@ def __init__(self, arn=None, name=None, compute_environments=None,
             self._priority = resource.priority
             self._arn = resource.arn
 
-            cloudknot.config.add_resource('job-queues', self.name, self.arn)
+            self._section_name = 'job-queues ' + self.region
+            cloudknot.config.add_resource(
+                self._section_name, self.name, self.arn
+            )
 
             mod_logger.info('Retrieved pre-existing job queue {name:s}'.format(
                 name=self.name
@@ -1148,7 +1160,8 @@ def _create(self):
         wait_for_job_queue(name=self.name, max_wait_time=180)
 
         # Add this job queue to the list of job queues in the config file
-        cloudknot.config.add_resource('job-queues', self.name, arn)
+        self._section_name = 'job-queues ' + self.region
+        cloudknot.config.add_resource(self._section_name, self.name, arn)
 
         mod_logger.info('Created job queue {name:s}'.format(name=self.name))
 
@@ -1174,6 +1187,9 @@ def get_jobs(self, status='ALL'):
                 self.arn
             )
 
+        if self.region != get_region():
+            raise RegionException(resource_region=self.region)
+
         # Validate input
         allowed_statuses = ['ALL', 'SUBMITTED', 'PENDING', 'RUNNABLE',
                             'STARTING', 'RUNNING', 'SUCCEEDED', 'FAILED']
@@ -1199,6 +1215,9 @@ def clobber(self):
         -------
         None
         """
+        if self.region != get_region():
+            raise RegionException(resource_region=self.region)
+
         # First, disable submissions to the queue
         retry = tenacity.Retrying(
             wait=tenacity.wait_exponential(max=32),
@@ -1233,7 +1252,7 @@ def clobber(self):
         retry.call(clients['batch'].delete_job_queue, jobQueue=self.arn)
 
         # Remove this job queue from the list of job queues in config file
-        cloudknot.config.remove_resource('job-queues', self.name)
+        cloudknot.config.remove_resource(self._section_name, self.name)
 
         # Set the clobbered parameter to True,
         # preventing subsequent method calls
@@ -1305,7 +1324,10 @@ def __init__(self, job_id=None, name=None, job_queue=None,
             self._environment_variables = job.environment_variables
             self._job_id = job.job_id
 
-            cloudknot.config.add_resource('batch-jobs', self.job_id, self.name)
+            self._section_name = 'batch-jobs ' + self.region
+            cloudknot.config.add_resource(
+                self._section_name, self.job_id, self.name
+            )
 
             mod_logger.info('Retrieved pre-existing batch job {id:s}'.format(
                 id=self.job_id
@@ -1425,7 +1447,10 @@ def _create(self):  # pragma: nocover
         job_id = response['jobId']
 
         # Remove this job from the list of jobs in the config file
-        cloudknot.config.add_resource('batch-jobs', self.job_id, self.name)
+        self._section_name = 'batch-jobs ' + self.region
+        cloudknot.config.add_resource(
+            self._section_name, self.job_id, self.name
+        )
 
         mod_logger.info(
             'Submitted batch job {name:s} with jobID '
@@ -1450,6 +1475,9 @@ def status(self):
                 self.job_id
             )
 
+        if self.region != get_region():
+            raise RegionException(resource_region=self.region)
+
         # Query the job_id
         response = clients['batch'].describe_jobs(jobs=[self.job_id])
         job = response.get('jobs')[0]
@@ -1481,6 +1509,9 @@ def terminate(self, reason):
                 self.job_id
             )
 
+        if self.region != get_region():
+            raise RegionException(resource_region=self.region)
+
         # Require the user to supply a reason for job termination
         if not isinstance(reason, six.string_types):
             raise ValueError('reason must be a string.')
@@ -1512,4 +1543,4 @@ def clobber(self):
         self._clobbered = True
 
         # Remove this job from the list of jobs in the config file
-        cloudknot.config.remove_resource('batch-jobs', self.job_id)
+        cloudknot.config.remove_resource(self._section_name, self.job_id)

cloudknot/aws/ec2.py

@@ -78,7 +78,10 @@ def __init__(self, vpc_id=None, name=None, ipv4_cidr=None,
             self._instance_tenancy = resource.instance_tenancy
             self._subnet_ids = resource.subnet_ids
 
-            cloudknot.config.add_resource('vpc', self.vpc_id, self.name)
+            self._section_name = 'vpc ' + self.region
+            cloudknot.config.add_resource(
+                self._section_name, self.vpc_id, self.name
+            )
 
             mod_logger.info('Retrieved pre-existing VPC {id:s}'.format(
                 id=self.vpc_id
@@ -284,7 +287,8 @@ def _create(self):
         )
 
         # Add this VPC to the list of VPCs in the config file
-        cloudknot.config.add_resource('vpc', vpc_id, self.name)
+        self._section_name = 'vpc ' + self.region
+        cloudknot.config.add_resource(self._section_name, vpc_id, self.name)
 
         return vpc_id
 
@@ -379,7 +383,7 @@ def clobber(self):
             clients['ec2'].delete_vpc(VpcId=self.vpc_id)
 
             # Remove this VPC from the list of VPCs in the config file
-            cloudknot.config.remove_resource('vpc', self.vpc_id)
+            cloudknot.config.remove_resource(self._section_name, self.vpc_id)
 
             # Set the clobbered parameter to True,
             # preventing subsequent method calls
@@ -480,8 +484,9 @@ def __init__(self, security_group_id=None, name=None, vpc=None,
                     resource_id=self.security_group_id
                 )
 
+            self._section_name = 'security-groups ' + self.region
             cloudknot.config.add_resource(
-                'security-groups', self.security_group_id, self.name
+                self._section_name, self.security_group_id, self.name
             )
 
             mod_logger.info(
@@ -646,9 +651,9 @@ def _create(self):
             ]
         )
 
-        # Add this security group to the list of security groups in the
-        # config file
-        cloudknot.config.add_resource('security-groups', group_id, self.name)
+        # Add this security group to the config file
+        self._section_name = 'security-groups ' + self.region
+        cloudknot.config.add_resource(self._section_name, group_id, self.name)
 
         return group_id
 
@@ -695,7 +700,7 @@ def has_security_group(instance, sg_id):
 
         # Remove this VPC from the list of VPCs in the config file
         cloudknot.config.remove_resource(
-            'security-groups', self.security_group_id
+            self._section_name, self.security_group_id
         )
 
         # Set the clobbered parameter to True,

cloudknot/aws/ecr.py

@@ -35,8 +35,9 @@ def __init__(self, name):
         self._repo_registry_id = repo_info.registry_id
 
         # Add to config file
+        self._section_name = 'docker-repos ' + self.region
         cloudknot.config.add_resource(
-            'docker-repos', self.name, self.repo_uri
+            self._section_name, self.name, self.repo_uri
         )
 
     # Declare read only properties
@@ -102,7 +103,7 @@ def clobber(self):
             pass
 
         # Remove from the config file
-        cloudknot.config.remove_resource('docker-repos', self.name)
+        cloudknot.config.remove_resource(self._section_name, self.name)
 
         # Set the clobbered parameter to True,
         # preventing subsequent method calls

cloudknot/aws/iam.py

@@ -392,7 +392,7 @@ def is_deleting(res):
                         clients['batch'].describe_compute_environments,
                         computeEnvironments=[ce['computeEnvironmentArn']]
                     )
-                except tenacity.RetryError:
+                except tenacity.RetryError:  # pragma: nocover
                     raise CannotDeleteResourceException(
                         'Could not delete this batch service role because it '
                         'it is taking too long for a dependent compute '