Merge pull request #17 from MITLibraries/etd-495-improve-dspace-error…

…-messages

Etd 495 improve dspace error messages

README.md

@@ -20,6 +20,25 @@ pipenv run submitter --help
 The [Click documentation](https://click.palletsprojects.com/en/8.0.x/quickstart/)
 will be helpful to understand how to create and run commands.
 
+Set env variables in `.env` file as needed:
+- WORKSPACE: required, "dev" is a good value for local development
+- DSPACE_API_URL: only needed if publishing to DSpace, use DSpace test instance for
+  development
+- DSPACE_USER: only needed if publishing to DSpace
+- DSPACE_PASSWORD: only needed if publishing to DSpace
+- DSPACE_TIMEOUT: only needed if publishing to DSpace, defaults to 120 seconds
+- DSS_INPUT_QUEUE: input message queue to use for development (see section below on
+  using Moto for local SQS queues)
+- DSS_LOG_FILTER: filters out logs from external libraries, defaults to "true".
+  Can be useful to set this to "false" if there are errors that seem to involve
+  external libraries whose debug logs may have more information
+- DSS_LOG_LEVEL: level for logging, defaults to INFO. Can be useful to set to DEBUG for
+  more detailed logging
+- SKIP_PROCESSING: skips the publishing process for messages, defaults to "true". Can
+  be useful for working on just the SQS components of the application. Set to "false"
+  if messages should be processed and published
+- SQS_ENDPOINT_URL: needed if using Moto for local development (see section below)
+
 ### Using Moto for local SQS queues
 
 It is often desireable to use [Moto](https://github.com/spulec/moto) for local development using the [Standalone Server Mode(https://github.com/spulec/moto#stand-alone-server-mode)] rather than using true AWS SQS queues.
@@ -51,8 +70,8 @@ file.
 
 `pipenv run submitter load-sample-data -i=YOUR_INPUT_QUEUE -o=YOUR_OUTPUT_QUEUE` will
 load some sample data into the SQS input queue. If you want to load data for
-integration testing with DSpace test, add an additional option to the command: `-f
-"tests/fixtures/integration-test-submission-messages"`.
+integration testing with DSpace test, add an additional option to the command:
+`-f tests/fixtures/integration-test-submission-messages.json`.
 
 Warning: please do not run this against the production system or a bunch of junk records
 will load into dspace
@@ -72,7 +91,7 @@ docker run submitter:latest --
 
 note: the application requires being run in an environment with Roles based access to the AWS resources. in addition, the environment must have WORKSPACE and SSM_PATH variables set according to stage and prod conventions.
 
-## Makefile Info 
+## Makefile Info
 ### Run-Stage
 Run-stage is outputted by the terraform used to create the infrastructure and copy/pasted here for convenience.
 Calling run-stage will execute the latest version of the container in the stage environment using the MITVPC.

conftest.py

@@ -5,7 +5,8 @@
 import pytest
 import requests_mock
 from dspace import DSpaceClient
-from moto import mock_sqs
+from moto import mock_sqs, mock_ssm
+from requests import exceptions
 
 
 @pytest.fixture(scope="function")
@@ -29,19 +30,37 @@ def mocked_dspace():
             "mock://dspace.edu/rest/handle/0000/collection01",
             json={"uuid": "collection01"},
         )
+        m.get(
+            "mock://dspace.edu/rest/handle/0000/collection02",
+            json={"uuid": "collection02"},
+        )
+        m.get(
+            "mock://dspace.edu/rest/handle/0000/collection03",
+            exc=exceptions.ConnectTimeout,
+        )
+        m.get(
+            "mock://dspace.edu/rest/handle/0000/not-a-collection",
+            status_code=404,
+        )
         m.post(
             "mock://dspace.edu/rest/collections/collection01/items",
-            json=item_post_response,
+            json=item_post_response_01,
+        )
+        m.post(
+            "mock://dspace.edu/rest/collections/collection02/items",
+            json=item_post_response_02,
         )
         m.post(
             "mock://dspace.edu/rest/items/item01/bitstreams",
             json=bitstream_post_response,
         )
         m.post(
-            "mock://dspace.edu/rest/collections/not-a-collection/items", status_code=404
+            "mock://dspace.edu/rest/items/item02/bitstreams",
+            status_code=500,
         )
         m.delete("mock://dspace.edu/rest/bitstreams/bitstream01", status_code=200)
         m.delete("mock://dspace.edu/rest/items/item01", status_code=200)
+        m.delete("mock://dspace.edu/rest/items/item02", status_code=200)
         yield m
 
 
@@ -73,6 +92,48 @@ def mocked_sqs(aws_credentials):
         yield sqs
 
 
+@pytest.fixture(scope="function")
+def mocked_ssm(aws_credentials):
+    with mock_ssm():
+        ssm = boto3.client("ssm")
+        ssm.put_parameter(
+            Name="/test/example/dspace_api_url",
+            Value="mock://dspace.edu/rest/",
+            Type="String",
+        )
+        ssm.put_parameter(
+            Name="/test/example/dspace_user",
+            Value="test",
+            Type="String",
+        )
+        ssm.put_parameter(
+            Name="/test/example/dspace_password",
+            Value="test",
+            Type="SecureString",
+        )
+        ssm.put_parameter(
+            Name="/test/example/dspace_timeout",
+            Value="3.0",
+            Type="String",
+        )
+        ssm.put_parameter(
+            Name="/test/example/SQS_dss_input_queue",
+            Value="empty_input_queue",
+            Type="String",
+        )
+        ssm.put_parameter(
+            Name="/test/example/dss_log_filter",
+            Value="False",
+            Type="String",
+        )
+        ssm.put_parameter(
+            Name="/test/example/dss_log_level",
+            Value="info",
+            Type="String",
+        )
+        yield ssm
+
+
 @pytest.fixture(scope="function")
 def test_client(mocked_dspace):
     client = DSpaceClient("mock://dspace.edu/rest/")
@@ -176,7 +237,31 @@ def input_message_item_post_error(mocked_sqs):
 
 
 @pytest.fixture
-def input_message_bitstream_post_error(mocked_sqs):
+def input_message_item_post_dspace_timeout(mocked_sqs):
+    queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue")
+    queue.send_message(
+        MessageAttributes=test_attributes,
+        MessageBody=json.dumps(
+            {
+                "SubmissionSystem": "DSpace@MIT",
+                "CollectionHandle": "0000/collection03",
+                "MetadataLocation": "tests/fixtures/test-item-metadata.json",
+                "Files": [
+                    {
+                        "BitstreamName": "test-file-01.pdf",
+                        "FileLocation": "tests/fixtures/test-file-01.pdf",
+                        "BitstreamDescription": "A test bitstream",
+                    }
+                ],
+            }
+        ),
+    )
+    message = queue.receive_messages(MessageAttributeNames=["All"])[0]
+    yield message
+
+
+@pytest.fixture
+def input_message_bitstream_file_open_error(mocked_sqs):
     queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue")
     queue.send_message(
         MessageAttributes=test_attributes,
@@ -204,35 +289,57 @@ def input_message_bitstream_post_error(mocked_sqs):
     yield message
 
 
+@pytest.fixture
+def input_message_bitstream_dspace_post_error(mocked_sqs):
+    queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue")
+    queue.send_message(
+        MessageAttributes=test_attributes,
+        MessageBody=json.dumps(
+            {
+                "SubmissionSystem": "DSpace@MIT",
+                "CollectionHandle": "0000/collection02",
+                "MetadataLocation": "tests/fixtures/test-item-metadata.json",
+                "Files": [
+                    {
+                        "BitstreamName": "test-file-01.pdf",
+                        "FileLocation": "tests/fixtures/test-file-01.pdf",
+                        "BitstreamDescription": "A test bitstream",
+                    },
+                ],
+            }
+        ),
+    )
+    message = queue.receive_messages(MessageAttributeNames=["All"])[0]
+    yield message
+
+
 @pytest.fixture
 def raw_attributes():
     yield test_attributes
 
 
 @pytest.fixture
 def raw_body():
-    yield json.dumps(
-        {
-            "SubmissionSystem": "DSpace@MIT",
-            "CollectionHandle": "0000/collection01",
-            "MetadataLocation": "tests/fixtures/test-item-metadata.json",
-            "Files": [
-                {
-                    "BitstreamName": "test-file-01.pdf",
-                    "FileLocation": "tests/fixtures/test-file-01.pdf",
-                    "BitstreamDescription": "A test bitstream",
-                },
-                {
-                    "BitstreamName": "No file",
-                    "FileLocation": "tests/fixtures/nothing-here",
-                    "BitstreamDescription": "No file",
-                },
-            ],
-        }
-    )
+    yield {
+        "SubmissionSystem": "DSpace@MIT",
+        "CollectionHandle": "0000/collection01",
+        "MetadataLocation": "tests/fixtures/test-item-metadata.json",
+        "Files": [
+            {
+                "BitstreamName": "test-file-01.pdf",
+                "FileLocation": "tests/fixtures/test-file-01.pdf",
+                "BitstreamDescription": "A test bitstream",
+            },
+            {
+                "BitstreamName": "No file",
+                "FileLocation": "tests/fixtures/nothing-here",
+                "BitstreamDescription": "No file",
+            },
+        ],
+    }
 
 
-item_post_response = {
+item_post_response_01 = {
     "uuid": "item01",
     "name": "Test Thesis",
     "handle": "0000/item01",
@@ -255,6 +362,29 @@ def raw_body():
     "withdrawn": "false",
 }
 
+item_post_response_02 = {
+    "uuid": "item02",
+    "name": "Test Thesis",
+    "handle": "0000/item02",
+    "type": "item",
+    "link": "/rest/items/item02",
+    "expand": [
+        "metadata",
+        "parentCollection",
+        "parentCollectionList",
+        "parentCommunityList",
+        "bitstreams",
+        "all",
+    ],
+    "lastModified": "2015-01-12 15:44:12.978",
+    "parentCollection": None,
+    "parentCollectionList": None,
+    "parentCommunityList": None,
+    "bitstreams": None,
+    "archived": "true",
+    "withdrawn": "false",
+}
+
 bitstream_post_response = {
     "uuid": "bitstream01",
     "name": "test-file-01.pdf",

submitter/__init__.py

@@ -3,4 +3,14 @@
 """
 import logging
 
-logging.basicConfig(level=logging.INFO)
+from submitter.config import Config
+
+CONFIG = Config()
+logging.basicConfig(level=getattr(logging, CONFIG.LOG_LEVEL))
+if CONFIG.LOG_FILTER == "true":
+    for handler in logging.root.handlers:
+        handler.addFilter(logging.Filter(__name__))
+logger = logging.getLogger(__name__)
+logger.info(
+    "Logging configured with level=%s, filter=%s", CONFIG.LOG_LEVEL, CONFIG.LOG_FILTER
+)

submitter/cli.py

@@ -2,7 +2,7 @@
 
 import click
 
-from submitter import config
+from submitter import CONFIG
 from submitter.message import generate_submission_messages_from_file
 from submitter.sqs import create, message_loop, write_message_to_queue
 
@@ -16,7 +16,7 @@ def main():
 
 @main.command()
 @click.option(
-    "--queue", default=config.INPUT_QUEUE, help="Name of queue to process messages from"
+    "--queue", default=CONFIG.INPUT_QUEUE, help="Name of queue to process messages from"
 )
 @click.option("--wait", default=20, help="seconds to wait for long polling. max 20")
 def start(queue, wait):
@@ -29,7 +29,7 @@ def start(queue, wait):
 @click.option(
     "-i",
     "--input-queue",
-    default=config.INPUT_QUEUE,
+    default=CONFIG.INPUT_QUEUE,
     help="Name of queue to load sample messages to",
 )
 @click.option(
@@ -46,7 +46,7 @@ def start(queue, wait):
     help="Path to json file of sample messages to load",
 )
 def load_sample_data(input_queue, output_queue, filepath):
-    logger.info(f"Loading sample data from file {filepath} into queue {input_queue}")
+    logger.info(f"Loading sample data from file '{filepath}' into queue {input_queue}")
     count = 0
     messages = generate_submission_messages_from_file(filepath, output_queue)
     for message in messages: