Merge pull request #250 from richford/config-prune

Add functions to clean up the cloudknot config file
nrdg · Sep 11, 2020 · e48b6ab · e48b6ab
2 parents a5ab64f + e2bde0e
commit e48b6ab
Show file tree

Hide file tree

Showing 17 changed files with 618 additions and 210 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -3,8 +3,8 @@ branch = True
 source = cloudknot/*
 include = cloudknot/*
 omit = */setup.py
+    cloudknot/commands/*.py
     cloudknot/due.py
-    cloudknot/config.py
     cloudknot/version.py
     cloudknot/_meta.py
     cloudknot/data/*/*/*.py

diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
@@ -19,6 +19,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install software
         run: |
+          python -m pip install coveralls --use-feature=2020-resolver
           python -m pip install --upgrade pip --use-feature=2020-resolver
           python -m pip install .[dev] --use-feature=2020-resolver
       - name: Configure
@@ -34,9 +35,19 @@ jobs:
       - name: Test
         run: |
           make test
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+      - name: Coveralls Parallel
+        run: |
+          coveralls
+        env:
+          COVERALLS_PARALLEL: true
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  finish:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@master
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: .coverage
-          name: codecov-cloudknot
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          parallel-finished: true
diff --git a/Makefile b/Makefile
@@ -7,12 +7,12 @@ lint: flake
 
 test:
     # Unit testing using pytest
-	py.test --pyargs cloudknot --cov-report term-missing --cov=cloudknot
+	pytest --pyargs cloudknot --cov-report term-missing --cov=cloudknot
 
 devtest:
     # Unit testing with the -x option, aborts testing after first failure
     # Useful for development when tests are long
-	py.test -x --pyargs cloudknot --cov-report term-missing --cov=cloudknot
+	pytest -x --pyargs cloudknot --cov-report term-missing --cov=cloudknot
 
 clean: clean-build clean-pyc ## remove all build, test, coverage and Python artifacts
 

diff --git a/README.md b/README.md
@@ -1,9 +1,8 @@
-![Build Status](https://github.com/nrdg/cloudknot/workflows/build/badge.svg)
-[![Codacy Badge](https://app.codacy.com/project/badge/Grade/4a5d0c767bfd4f0eae820c24df1ce2a8)](https://www.codacy.com/gh/nrdg/cloudknot?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=nrdg/cloudknot&amp;utm_campaign=Badge_Grade)
-[![Codacy Badge](https://app.codacy.com/project/badge/Coverage/d6fa3a18646f4a8089c7c897819d0342)](https://www.codacy.com/manual/richford/cloudknot?utm_source=github.com&utm_medium=referral&utm_content=nrdg/cloudknot&utm_campaign=Badge_Coverage)
-
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Build Status](https://github.com/nrdg/cloudknot/workflows/build/badge.svg)](https://github.com/nrdg/cloudknot/workflows/build/badge.svg)
+[![Coverage Status](https://coveralls.io/repos/github/nrdg/cloudknot/badge.svg?branch=master)](https://coveralls.io/github/nrdg/cloudknot?branch=master)
+[![Codacy Badge](https://app.codacy.com/project/badge/Grade/4a5d0c767bfd4f0eae820c24df1ce2a8)](https://www.codacy.com/gh/nrdg/cloudknot?utm_source=github.com&utm_medium=referral&utm_content=nrdg/cloudknot&utm_campaign=Badge_Grade)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![DOI](https://zenodo.org/badge/102051437.svg)](https://zenodo.org/badge/latestdoi/102051437)
 
 # cloudknot

diff --git a/cloudknot/__init__.py b/cloudknot/__init__.py
@@ -22,7 +22,7 @@
 
 try:
     __version__ = version(__name__)
-except PackageNotFoundError:
+except PackageNotFoundError:  # pragma: nocover
     # package is not installed
     pass
 
@@ -59,7 +59,7 @@
     pre_existing = e.errno == errno.EEXIST and os.path.isdir(logdir)
     if pre_existing:
         pass
-    else:
+    else:  # pragma: nocover
         raise e
 
 handler = logging.FileHandler(logpath, mode="w")

diff --git a/cloudknot/aws/base_classes.py b/cloudknot/aws/base_classes.py
@@ -33,14 +33,14 @@ def get_tags(name, additional_tags=None):
     if additional_tags is not None:
         if isinstance(additional_tags, list):
             if not all(
-                [set(item.keys) == set(["Key", "Value"]) for item in additional_tags]
+                [set(item.keys()) == set(["Key", "Value"]) for item in additional_tags]
             ):
                 raise ValueError(
                     "If additional_tags is a list, it must be a list of "
                     "dictionaries of the form {'Key': key_val, 'Value': "
                     "value_val}."
                 )
-            tag_list = additional_tags
+            tag_list += additional_tags
         elif isinstance(additional_tags, dict):
             if "Key" in additional_tags.keys() or "Value" in additional_tags.keys():
                 raise ValueError(
@@ -578,7 +578,10 @@ def set_region(region="us-east-1"):
         # throughout the package
         max_pool = clients["iam"].meta.config.max_pool_connections
         boto_config = botocore.config.Config(max_pool_connections=max_pool)
-        session = boto3.Session(profile_name=get_profile(fallback=None))
+        profile_name = get_profile(fallback=None)
+        session = boto3.Session(
+            profile_name=profile_name if profile_name != "from-env" else None
+        )
         clients["batch"] = session.client(
             "batch", region_name=region, config=boto_config
         )
@@ -591,6 +594,8 @@ def set_region(region="us-east-1"):
         clients["iam"] = session.client("iam", region_name=region, config=boto_config)
         clients["s3"] = session.client("s3", region_name=region, config=boto_config)
 
+    mod_logger.debug("Set region to {region:s}".format(region=region))
+
 
 @registered
 def list_profiles():
@@ -724,7 +729,7 @@ def set_profile(profile_name):
     """
     profile_info = list_profiles()
 
-    if profile_name not in profile_info.profile_names:
+    if not (profile_name in profile_info.profile_names or profile_name == "from-env"):
         raise CloudknotInputError(
             "The profile you specified does not exist in either the AWS "
             "config file at {conf:s} or the AWS shared credentials file at "
@@ -750,7 +755,9 @@ def set_profile(profile_name):
         # throughout the package
         max_pool = clients["iam"].meta.config.max_pool_connections
         boto_config = botocore.config.Config(max_pool_connections=max_pool)
-        session = boto3.Session(profile_name=profile_name)
+        session = boto3.Session(
+            profile_name=profile_name if profile_name != "from-env" else None
+        )
         clients["batch"] = session.client(
             "batch", region_name=get_region(), config=boto_config
         )
@@ -773,6 +780,8 @@ def set_profile(profile_name):
             "s3", region_name=get_region(), config=boto_config
         )
 
+    mod_logger.debug("Set profile to {profile:s}".format(profile=profile_name))
+
 
 #: module-level dictionary of boto3 clients for IAM, EC2, Batch, ECR, ECS, S3.
 clients = {

diff --git a/cloudknot/aws/ecr.py b/cloudknot/aws/ecr.py
@@ -22,6 +22,22 @@ def registered(fn):
 mod_logger = logging.getLogger(__name__)
 
 
+def _get_repo_info_from_uri(repo_uri):
+    # Get all repositories
+    repositories = clients["ecr"].describe_repositories(maxResults=500)["repositories"]
+
+    _repo_uri = repo_uri.split(":")[0]
+    # Filter by matching on repo_uri
+    matching_repo = [
+        repo for repo in repositories if repo["repositoryUri"] == _repo_uri
+    ][0]
+
+    return {
+        "registry_id": matching_repo["registryId"],
+        "repo_name": matching_repo["repositoryName"],
+    }
+
+
 # noinspection PyPropertyAccess,PyAttributeOutsideInit
 @registered
 class DockerRepo(NamedObject):

diff --git a/cloudknot/cloudknot.py b/cloudknot/cloudknot.py
@@ -14,7 +14,7 @@
 from concurrent.futures import ThreadPoolExecutor
 
 from . import aws
-from .config import get_config_file, rlock
+from .config import get_config_file, rlock, is_valid_stack
 from . import dockerimage
 
 __all__ = []
@@ -129,54 +129,13 @@ def __init__(
 
             self._stack_id = config.get(self._pars_name, "stack-id")
 
-            try:
-                response = aws.clients["cloudformation"].describe_stacks(
-                    StackName=self._stack_id
-                )
-            except aws.clients["cloudformation"].exceptions.ClientError as e:
-                error_code = e.response.get("Error").get("Message")
-                no_stack_code = "Stack with id {0:s} does not exist" "".format(
-                    self._stack_id
-                )
-                if error_code == no_stack_code:
-                    # Remove this section from the config file
-                    with rlock:
-                        config.read(get_config_file())
-                        config.remove_section(self._pars_name)
-                        with open(get_config_file(), "w") as f:
-                            config.write(f)
-                    raise aws.ResourceDoesNotExistException(
-                        "Cloudknot found this PARS in its config file, but "
-                        "the PARS stack that you requested does not exist on "
-                        "AWS. Cloudknot has deleted this PARS from the config "
-                        "file, so you may be able to create a new one simply "
-                        "by re-running your previous command.",
-                        self._stack_id,
-                    )
-                else:  # pragma: nocover
-                    raise e
-
-            no_stack = len(response.get("Stacks")) == 0 or response.get("Stacks")[0][
-                "StackStatus"
-            ] in [
-                "CREATE_FAILED",
-                "ROLLBACK_COMPLETE",
-                "ROLLBACK_IN_PROGRESS",
-                "ROLLBACK_FAILED",
-                "DELETE_IN_PROGRESS",
-                "DELETE_FAILED",
-                "DELETE_COMPLETE",
-                "UPDATE_ROLLBACK_FAILED",
-            ]
-
-            if no_stack:
+            if not is_valid_stack(self._stack_id):
                 # Remove this section from the config file
                 with rlock:
                     config.read(get_config_file())
                     config.remove_section(self._pars_name)
                     with open(get_config_file(), "w") as f:
                         config.write(f)
-
                 raise aws.ResourceDoesNotExistException(
                     "Cloudknot found this PARS in its config file, but "
                     "the PARS stack that you requested does not exist on "
@@ -186,6 +145,9 @@ def __init__(
                     self._stack_id,
                 )
 
+            response = aws.clients["cloudformation"].describe_stacks(
+                StackName=self._stack_id
+            )
             outs = response.get("Stacks")[0]["Outputs"]
 
             self._batch_service_role = _stack_out("BatchServiceRole", outs)
@@ -1005,61 +967,24 @@ def __init__(
 
             self._stack_id = config.get(self._knot_name, "stack-id")
 
-            try:
-                response = aws.clients["cloudformation"].describe_stacks(
-                    StackName=self._stack_id
-                )
-            except aws.clients["cloudformation"].exceptions.ClientError as e:
-                error_code = e.response.get("Error").get("Message")
-                no_stack_code = "Stack with id {0:s} does not exist" "".format(
-                    self._stack_id
-                )
-                if error_code == no_stack_code:
-                    # Remove this section from the config file
-                    with rlock:
-                        config.read(get_config_file())
-                        config.remove_section(self._knot_name)
-                        with open(get_config_file(), "w") as f:
-                            config.write(f)
-                    raise aws.ResourceDoesNotExistException(
-                        "The Knot cloudformation stack that you requested "
-                        "does not exist. Cloudknot has deleted this Knot from "
-                        "the config file, so you may be able to create a new "
-                        "one simply by re-running your previous command.",
-                        self._stack_id,
-                    )
-                else:
-                    raise e
-
-            no_stack = len(response.get("Stacks")) == 0 or response.get("Stacks")[0][
-                "StackStatus"
-            ] in [
-                "CREATE_FAILED",
-                "ROLLBACK_COMPLETE",
-                "ROLLBACK_IN_PROGRESS",
-                "ROLLBACK_FAILED",
-                "DELETE_IN_PROGRESS",
-                "DELETE_FAILED",
-                "DELETE_COMPLETE",
-                "UPDATE_ROLLBACK_FAILED",
-            ]
-
-            if no_stack:
+            if not is_valid_stack(self._stack_id):
                 # Remove this section from the config file
                 with rlock:
                     config.read(get_config_file())
                     config.remove_section(self._knot_name)
                     with open(get_config_file(), "w") as f:
                         config.write(f)
-
                 raise aws.ResourceDoesNotExistException(
-                    "The Knot cloudformation stack that you requested does "
-                    "not exist. Cloudknot has deleted this Knot from the "
-                    "config file, so you may be able to create a new one "
-                    "simply by re-running your previous command.",
+                    "The Knot cloudformation stack that you requested "
+                    "does not exist. Cloudknot has deleted this Knot from "
+                    "the config file, so you may be able to create a new "
+                    "one simply by re-running your previous command.",
                     self._stack_id,
                 )
 
+            response = aws.clients["cloudformation"].describe_stacks(
+                StackName=self._stack_id
+            )
             outs = response.get("Stacks")[0]["Outputs"]
 
             job_def_arn = _stack_out("JobDefinition", outs)