Merge pull request #24 from MITLibraries/etd-436-smoke-tests

Add infrastructure permissions check

Makefile

@@ -32,6 +32,12 @@ promote: ## Promote the current staging build to production
 	docker push $(ECR_REGISTRY)/dspacesubmissionservice-prod:latest
 	docker push $(ECR_REGISTRY)/dspacesubmissionservice-prod:$(DATETIME)
 
+check-permissions-stage: ## Check infrastructure permissions on the staging deplpyment
+	aws ecs run-task --cluster dspacesubmissionservice-stage --task-definition dspacesubmissionservice-stage --network-configuration "awsvpcConfiguration={subnets=[subnet-0744a5c9beeb49a20],securityGroups=[sg-06b90b77a06e5870a],assignPublicIp=DISABLED}" --launch-type FARGATE --region us-east-1 --overrides '{"containerOverrides": [{"name": "DSS","command": ["check-permissions"]}]}'
+
+check-permissions-prod: ## Check infrastructure permissions on the prod deplpyment
+	aws ecs run-task --cluster dspacesubmissionservice-prod --task-definition dspacesubmissionservice-prod --network-configuration "awsvpcConfiguration={subnets=[subnet-0744a5c9beeb49a20],securityGroups=[sg-0b29d571e70c05101],assignPublicIp=DISABLED}" --launch-type FARGATE --region us-east-1 --overrides '{"containerOverrides": [{"name": "DSS","command": ["check-permissions"]}]}'
+
 run-stage:  ## Runs the task in stage - see readme for more info
 	aws ecs run-task --cluster dspacesubmissionservice-stage --task-definition dspacesubmissionservice-stage --network-configuration "awsvpcConfiguration={subnets=[subnet-0744a5c9beeb49a20],securityGroups=[sg-06b90b77a06e5870a],assignPublicIp=DISABLED}" --launch-type FARGATE --region us-east-1
 

README.md

@@ -34,12 +34,16 @@ Set env variables in `.env` file as needed:
   external libraries whose debug logs may have more information
 - DSS_LOG_LEVEL: level for logging, defaults to INFO. Can be useful to set to DEBUG for
   more detailed logging
+- DSS_OUTPUT_QUEUES: comma-separated string list of valid output queues, defaults to
+  "output". Update if using a different name for the output queue(s) in development
+- DSS_S3_BUCKET_NAMES: comma-separated string list of any S3 buckets needed to retrieve
+  files referenced in input messages (e.g. content files and JSON metadata files), only
+  needed if doing a permissions check
 - SKIP_PROCESSING: skips the publishing process for messages, defaults to "true". Can
   be useful for working on just the SQS components of the application. Set to "false"
   if messages should be processed and published
 - SQS_ENDPOINT_URL: needed if using Moto for local development (see section below)
-- DSS_OUTPUT_QUEUES": comma-separated string list of valid output queues, defaults to
-  "output". Update if using a different name for the output queue(s) in development
+
 
 ### Using Moto for local SQS queues
 

conftest.py

@@ -5,7 +5,7 @@
 import pytest
 import requests_mock
 from dspace import DSpaceClient
-from moto import mock_sqs, mock_ssm
+from moto import mock_iam, mock_s3, mock_sqs, mock_ssm
 from requests import exceptions
 
 
@@ -15,8 +15,40 @@ def aws_credentials():
     os.environ["AWS_ACCESS_KEY_ID"] = "testing"
     os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
     os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
-    os.environ["AWS_SECURITY_TOKEN"] = "testing"
-    os.environ["AWS_SESSION_TOKEN"] = "testing"
+
+
+@pytest.fixture()
+def test_aws_user(aws_credentials):
+    with mock_iam():
+        user_name = "test-user"
+        policy_document = {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Action": ["s3:ListBucket", "sqs:GetQueueUrl"],
+                    "Resource": "*",
+                },
+                {
+                    "Effect": "Deny",
+                    "Action": [
+                        "s3:GetObject",
+                        "sqs:ReceiveMessage",
+                        "sqs:SendMessage",
+                        "ssm:GetParameter",
+                    ],
+                    "Resource": "*",
+                },
+            ],
+        }
+        client = boto3.client("iam", region_name="us-east-1")
+        client.create_user(UserName=user_name)
+        client.put_user_policy(
+            UserName=user_name,
+            PolicyName="policy1",
+            PolicyDocument=json.dumps(policy_document),
+        )
+        yield client.create_access_key(UserName="test-user")["AccessKey"]
 
 
 @pytest.fixture(scope="function")
@@ -122,7 +154,7 @@ def mocked_ssm(aws_credentials):
             Type="String",
         )
         ssm.put_parameter(
-            Name="/test/example/SQS_dss_input_queue",
+            Name="/test/example/dss_input_queue",
             Value="empty_input_queue",
             Type="String",
         )
@@ -136,19 +168,40 @@ def mocked_ssm(aws_credentials):
             Value="info",
             Type="String",
         )
-        ssm.put_parameter(
-            Name="/test/example/sentry_dsn",
-            Value="http://12345.6789.sentry",
-            Type="String",
-        )
         ssm.put_parameter(
             Name="/test/example/dss_output_queues",
             Value="test_output_1,test_output_2",
             Type="StringList",
         )
+        ssm.put_parameter(
+            Name="/test/example/dss_s3_bucket_arns",
+            Value="arn:aws:s3:::test-bucket-01,arn:aws:s3:::test-bucket-02",
+            Type="StringList",
+        )
+        ssm.put_parameter(
+            Name="/test/example/secure",
+            Value="true",
+            Type="SecureString",
+        )
+        ssm.put_parameter(
+            Name="/test/example/sentry_dsn",
+            Value="http://12345.6789.sentry",
+            Type="String",
+        )
         yield ssm
 
 
+@pytest.fixture()
+def mocked_s3(aws_credentials):
+    with mock_s3():
+        s3 = boto3.client("s3")
+        s3.create_bucket(
+            Bucket="test-bucket",
+        )
+        s3.put_object(Bucket="test-bucket", Key="object1", Body="I am an object.")
+        yield s3
+
+
 @pytest.fixture(scope="function")
 def test_client(mocked_dspace):
     client = DSpaceClient("mock://dspace.edu/rest/")

submitter/cli.py

@@ -4,7 +4,15 @@
 
 from submitter import CONFIG
 from submitter.message import generate_submission_messages_from_file
-from submitter.sqs import create, message_loop, write_message_to_queue
+from submitter.s3 import check_s3_permissions
+from submitter.sqs import (
+    check_read_permissions,
+    check_write_permissions,
+    create,
+    message_loop,
+    write_message_to_queue,
+)
+from submitter.ssm import SSM
 
 logger = logging.getLogger(__name__)
 
@@ -61,3 +69,33 @@ def create_queue(name):
     """Create queue with NAME supplied as argument"""
     queue = create(name)
     logger.info(queue.url)
+
+
+@main.command()
+def check_permissions():
+    """Confirm DSS infrastructure has deployed properly with correct permissions to all
+    expected resources given the current env configuration.
+
+    Note: Only useful in stage and prod envs, as this command requires SSM access which
+        does not get configured in dev.
+
+    Note: Checking SQS write permissions does write a message to each configured output
+        queue. The test message gets deleted as part of the process, however if there
+        are already more than 10 messages in the output queue that delete may not
+        happen and the test message will remain in the queue. It is best to only run
+        this command when the configured output queues are empty.
+    """
+
+    # Confirm we can retrieve an encrypted ssm parameter
+    ssm = SSM()
+    logger.info(ssm.check_permissions(CONFIG.SSM_PATH))
+
+    # Confirm we can read from and write to all expected SQS queues
+    logger.info(check_read_permissions(CONFIG.INPUT_QUEUE))
+    for queue in CONFIG.VALID_RESULT_QUEUES:
+        logger.info(check_write_permissions(queue))
+
+    # Confirm we can list and get objects from all expected s3 buckets
+    logger.info(check_s3_permissions(CONFIG.S3_BUCKETS))
+
+    logger.info(f"All permissions confirmed for env '{CONFIG.ENV}'")

submitter/config.py

@@ -39,14 +39,20 @@ def load_config_variables(self, env: str):
                 ssm.get_parameter_value(self.SSM_PATH + "dspace_timeout")
             )
             self.INPUT_QUEUE = ssm.get_parameter_value(
-                self.SSM_PATH + "SQS_dss_input_queue"
+                self.SSM_PATH + "dss_input_queue"
             )
             self.LOG_FILTER = ssm.get_parameter_value(
                 self.SSM_PATH + "dss_log_filter"
             ).lower()
             self.LOG_LEVEL = ssm.get_parameter_value(
                 self.SSM_PATH + "dss_log_level"
             ).upper()
+
+            bucket_arns = ssm.get_parameter_value(
+                self.SSM_PATH + "dss_s3_bucket_arns"
+            ).split(",")
+            self.S3_BUCKETS = [bucket.split(":")[-1] for bucket in bucket_arns]
+
             self.SENTRY_DSN = ssm.get_parameter_value(self.SSM_PATH + "sentry_dsn")
             self.SKIP_PROCESSING = "false"
             self.SQS_ENDPOINT_URL = "https://sqs.us-east-1.amazonaws.com/"
@@ -61,6 +67,7 @@ def load_config_variables(self, env: str):
             self.INPUT_QUEUE = "test_queue_with_messages"
             self.LOG_FILTER = "true"
             self.LOG_LEVEL = os.getenv("DSS_LOG_LEVEL", "INFO").upper()
+            self.S3_BUCKETS = ["test-bucket"]
             self.SENTRY_DSN = None
             self.SKIP_PROCESSING = "false"
             self.SQS_ENDPOINT_URL = "https://sqs.us-east-1.amazonaws.com/"
@@ -73,9 +80,10 @@ def load_config_variables(self, env: str):
             self.INPUT_QUEUE = os.getenv("DSS_INPUT_QUEUE")
             self.LOG_FILTER = os.getenv("DSS_LOG_FILTER", "true").lower()
             self.LOG_LEVEL = os.getenv("DSS_LOG_LEVEL", "INFO").upper()
+            self.S3_BUCKETS = os.getenv("DSS_S3_BUCKET_NAMES", "no-bucket").split(",")
             self.SENTRY_DSN = os.getenv("DSS_SENTRY_DSN")
-            self.SKIP_PROCESSING = os.environ.get("SKIP_PROCESSING", "false").lower()
-            self.SQS_ENDPOINT_URL = os.environ.get("SQS_ENDPOINT_URL")
+            self.SKIP_PROCESSING = os.getenv("SKIP_PROCESSING", "false").lower()
+            self.SQS_ENDPOINT_URL = os.getenv("SQS_ENDPOINT_URL")
             self.VALID_RESULT_QUEUES = os.getenv("DSS_OUTPUT_QUEUES", "output").split(
                 ","
             )

submitter/s3.py

@@ -0,0 +1,29 @@
+import logging
+from typing import List
+
+import boto3
+
+logger = logging.getLogger(__name__)
+
+
+def check_s3_permissions(buckets: List[str]) -> str:
+    """Checks S3 ListObjectV2 and GetObject permissions for all buckets provided in the
+    passed list. If either command is not allowed for any of the provided buckets,
+    raises an Access Denied bocotore client error.
+    """
+    s3 = boto3.client("s3")
+    bucket_names = []
+    for bucket in buckets:
+        response = s3.list_objects_v2(Bucket=bucket, MaxKeys=1)
+        logger.debug(f"Successfully listed objects in bucket '{bucket}'")
+        for object in response["Contents"]:
+            s3.get_object(Bucket=bucket, Key=object["Key"])
+            bucket_names.append(bucket)
+            logger.debug(
+                f"Successfully retrieved object '{object['Key']}' from bucket "
+                f"'{bucket}'"
+            )
+    return (
+        "S3 list objects and get object permissions confirmed for buckets: "
+        f"{bucket_names}"
+    )

submitter/sqs.py

@@ -110,3 +110,40 @@ def verify_sent_message(
         json.dumps(sent_message_body).encode("utf-8")
     ).hexdigest()
     return body_md5 == sqs_send_message_response["MD5OfMessageBody"]
+
+
+def check_read_permissions(queue_name):
+    messages = retrieve_messages_from_queue(queue_name, 20)
+    logger.debug(
+        f"Able to access queue '{queue_name}', retrieved {len(messages)} messages"
+    )
+    return f"Read permissions confirmed for queue '{queue_name}'"
+
+
+def check_write_permissions(queue_name):
+    response = write_message_to_queue(
+        {"PackageID": {"DataType": "String", "StringValue": "SmokeTest"}},
+        {"TestBody": "Testing write permissions"},
+        queue_name,
+    )
+    message_id = response["MessageId"]
+
+    if verify_sent_message({"TestBody": "Testing write permissions"}, response):
+        logger.debug(
+            f"Test message successfully written to queue '{queue_name}' with message "
+            f"ID '{message_id}'"
+        )
+    else:
+        logger.error(
+            f"Unable to verify that message with ID '{message_id}' was "
+            f"successfully written to queue '{queue_name}'"
+        )
+
+    messages = retrieve_messages_from_queue(queue_name, 3, 3)
+    for message in messages:
+        if message.message_id == message_id:
+            message.delete()
+            logger.debug(
+                f"Test message with ID '{message_id}' deleted from queue '{queue_name}'"
+            )
+    return f"Write permissions confirmed for queue '{queue_name}'"

submitter/ssm.py

@@ -1,4 +1,9 @@
+import logging
+
 from boto3 import client
+from botocore.exceptions import ClientError
+
+logger = logging.getLogger(__name__)
 
 
 class SSM:
@@ -10,8 +15,26 @@ def __init__(self):
 
     def get_parameter_value(self, parameter_key):
         """Get parameter value based on the specified key."""
-        parameter_object = self.client.get_parameter(
-            Name=parameter_key, WithDecryption=True
-        )
+        try:
+            parameter_object = self.client.get_parameter(
+                Name=parameter_key, WithDecryption=True
+            )
+        except ClientError as e:
+            if "ParameterNotFound" in str(e):
+                raise Exception(f"Parameter '{parameter_key}' not found") from e
+            raise e
         parameter_value = parameter_object["Parameter"]["Value"]
         return parameter_value
+
+    def check_permissions(self, ssm_path: str) -> str:
+        """Check whether we can retrieve an encrypted ssm parameter.
+        Raises an exception if we can't retrieve the parameter at all OR if the
+        parameter is retrieved but the value can't be decrypted.
+        """
+        decrypted_parameter = self.get_parameter_value(ssm_path + "secure")
+        if decrypted_parameter != "true":
+            raise (
+                "Was not able to successfully retrieve encrypted SSM parameter "
+                f"{decrypted_parameter}"
+            )
+        return f"SSM permissions confirmed for path '{ssm_path}'"

tests/test_config.py

@@ -32,6 +32,7 @@ def test_prod_stage_config_success(mocked_ssm):
     assert config.SENTRY_DSN == "http://12345.6789.sentry"
     assert config.SKIP_PROCESSING == "false"
     assert config.SQS_ENDPOINT_URL == "https://sqs.us-east-1.amazonaws.com/"
+    assert config.S3_BUCKETS == ["test-bucket-01", "test-bucket-02"]
     assert config.VALID_RESULT_QUEUES == ["test_output_1", "test_output_2"]
 
 
@@ -47,6 +48,8 @@ def test_dev_config_success():
     os.environ.pop("SENTRY_DSN", None)
     os.environ["SKIP_PROCESSING"] = "True"
     os.environ.pop("SQS_ENDPOINT_URL", None)
+    os.environ.pop("DSS_S3_BUCKET_NAMES", None)
+    os.environ["DSS_S3_BUCKET_NAMES"] = "test-bucket"
     os.environ.pop("DSS_OUTPUT_QUEUES", None)
     config = Config()
     assert config.DSPACE_API_URL is None
@@ -59,4 +62,5 @@ def test_dev_config_success():
     assert config.SENTRY_DSN is None
     assert config.SKIP_PROCESSING == "true"
     assert config.SQS_ENDPOINT_URL is None
+    assert config.S3_BUCKETS == ["test-bucket"]
     assert config.VALID_RESULT_QUEUES == ["output"]

tests/test_s3.py

@@ -0,0 +1,29 @@
+import os
+
+import boto3
+import pytest
+from botocore.exceptions import ClientError
+from moto.core import set_initial_no_auth_action_count
+
+from submitter.s3 import check_s3_permissions
+
+
+def test_check_s3_permissions_success(mocked_s3):
+    result = check_s3_permissions(["test-bucket"])
+    assert (
+        result == "S3 list objects and get object permissions confirmed for buckets: "
+        "['test-bucket']"
+    )
+
+
+@set_initial_no_auth_action_count(0)
+def test_check_s3_permissions_raises_error(mocked_s3, test_aws_user):
+    os.environ["AWS_ACCESS_KEY_ID"] = test_aws_user["AccessKeyId"]
+    os.environ["AWS_SECRET_ACCESS_KEY"] = test_aws_user["SecretAccessKey"]
+    boto3.setup_default_session()
+    with pytest.raises(ClientError) as e:
+        check_s3_permissions(["test-bucket"])
+    assert (
+        "An error occurred (AccessDenied) when calling the GetObject operation: Access "
+        "Denied" in str(e.value)
+    )