From 1ee21b58194df8bbf93a824e084dd1710469ed06 Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Wed, 15 Sep 2021 14:15:27 -0400 Subject: [PATCH 1/3] Add DSpace submit functionality Why these changes are being introduced: The core functionality of this app is to create new items with associated bitstreams in DSpace based on the messages retrieved from an input SQS queue. This commit adds the functionality to do so, along with some necessary functionality around configuration and secrets handling. How this addresses that need: * Adds a submission module with a Submission class that can be instantiated from an SQS message, with methods to facilitate creating a DSpace item with bitstreams and generating both success and error result messages * Updates the sqs module process function to use the new submission module * Adds an smm module with an SSM class that handles retrieving secrets from AWS Systems Manager Parameter Store * Adds a config module that configures the application based on env (set by the WORKSPACE env variable) * Adds and updates relevant tests, including adding a conftest.py and fixtures * Updates Dockerfile and setup.py to reflect new dependencies Additional changes in this commit: * Adds logging throughout the app with logging configuration set by the root logger * Adds pytest-cov and updates make test command to create a coverage report * Updates .gitignore Side effects of this change: * For the app to work in staging and production environments, the WORKSPACE and SSM_PATH env variables must be set and the application must have an IAM role with access to SSM. This has been handled in a separate ticket, just noting it here (also added this info to the README). Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/ETD-398 * https://mitlibraries.atlassian.net/browse/ETD-407 --- .gitignore | 2 + Dockerfile | 2 + Makefile | 2 +- Pipfile | 5 + Pipfile.lock | 100 +++++-- README.md | 2 +- conftest.py | 259 +++++++++++++++++++ pyproject.toml | 7 +- setup.py | 6 + submitter/__init__.py | 4 +- submitter/cli.py | 29 ++- submitter/config.py | 29 +++ submitter/sqs.py | 88 +++---- submitter/ssm.py | 17 ++ submitter/submission.py | 136 ++++++++++ tests/fixtures/test-file-01.pdf | Bin 0 -> 35721 bytes tests/fixtures/test-item-metadata-error.json | 7 + tests/fixtures/test-item-metadata.json | 12 + tests/test_cli.py | 57 +--- tests/test_submission.py | 128 +++++++++ 20 files changed, 759 insertions(+), 133 deletions(-) create mode 100644 conftest.py create mode 100644 submitter/config.py create mode 100644 submitter/ssm.py create mode 100644 submitter/submission.py create mode 100644 tests/fixtures/test-file-01.pdf create mode 100644 tests/fixtures/test-item-metadata-error.json create mode 100644 tests/fixtures/test-item-metadata.json create mode 100644 tests/test_submission.py diff --git a/.gitignore b/.gitignore index b6e4761..7f6b817 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,5 @@ dmypy.json # Pyre type checker .pyre/ + +.DS_Store diff --git a/Dockerfile b/Dockerfile index 4baa9bd..bcf8bfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ ENV PIP_NO_CACHE_DIR yes WORKDIR /app RUN pip install --no-cache-dir --upgrade pip pipenv +RUN apt-get update && apt-get upgrade -y && apt-get install -y git + COPY Pipfile* / RUN pipenv install --system --clear --deploy diff --git a/Makefile b/Makefile index 58ab82e..f84acba 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ isort: ## isort your imports, so you don't have to pipenv run isort . --diff test: ## runs pytest - pipenv run pytest + pipenv run pytest --cov=submitter coveralls: test pipenv run coveralls diff --git a/Pipfile b/Pipfile index c602c6f..d79b17a 100644 --- a/Pipfile +++ b/Pipfile @@ -6,6 +6,8 @@ name = "pypi" [packages] click = "*" boto3 = "*" +smart-open = "*" +dspace-python-client = {ref = "0.1.0", git = "https://github.com/mitlibraries/dspace-python-client.git"} [dev-packages] flake8 = "*" @@ -15,6 +17,9 @@ bandit = "*" moto = {extras = ["s3", "sqs"], version = "*"} pytest = "*" coveralls = "*" +requests-mock = "*" +pytest-cov = "*" +pytest-env = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 16b19ad..9372353 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4886d0e7b0613ebefce99153e350cb6a32dcaf8fcb73d743599db3faad690916" + "sha256": "6ae732f1c1cc7845eab2c51f30d7a092ea2c119be63c44c12f4f7050f542f14f" }, "pipfile-spec": 6, "requires": { @@ -18,19 +18,19 @@ "default": { "boto3": { "hashes": [ - "sha256:a299d0c6b5a30dc2e823944286ec782aec415d83965a51f97fc9a779a04ff194", - "sha256:f4b17a2b6e04e5ec6f494e643d05b06dd60c88943f33d6f9650dd9e7f89a7022" + "sha256:63b9846c26e0905f4e9e39d6b59f152330c53a926d693439161c43dcf9779365", + "sha256:a9232185d8e7e2fd2b166c0ebee5d7b1f787fdb3093f33bbf5aa932c08f0ccac" ], "index": "pypi", - "version": "==1.18.32" + "version": "==1.18.42" }, "botocore": { "hashes": [ - "sha256:5803bf852304a301de41dccc3c0431053354144f3aefc7571dbe240a4288d3c5", - "sha256:95ff61534b2a423d0e70067c39615e4e70c119773d2180d7254bf4025c54396d" + "sha256:0952d1200968365b440045efe8e45bbae38cf603fee12bcfc3d7b5f963cbfa18", + "sha256:6de4fec4ee10987e4dea96f289553c2f45109fcaafcb74a5baee1221926e1306" ], "markers": "python_version >= '3.6'", - "version": "==1.21.32" + "version": "==1.21.42" }, "click": { "hashes": [ @@ -40,6 +40,10 @@ "index": "pypi", "version": "==8.0.1" }, + "dspace-python-client": { + "git": "https://github.com/mitlibraries/dspace-python-client.git", + "ref": "dd7a8ab508f1a0e7722b47851d6702d6a8ca9301" + }, "jmespath": { "hashes": [ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", @@ -51,6 +55,8 @@ "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:104a4ff9f1ece23d8a31582156ea3ae928afe7121fac9fed3e967a1e2d6cf6ed", + "sha256:1efd93a2e222eb7360b5396108fdfa04e9753637d24143b8026dfb48ffbc755b", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", @@ -72,12 +78,20 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, + "smart-open": { + "hashes": [ + "sha256:71d14489da58b60ce12fc3ecb823facc59a8b23cd1b58edb97175640350d3a62", + "sha256:75abf758717a92a8f53aa96953f0c245c8cedf8e1e4184903db3659b419d4c17" + ], + "index": "pypi", + "version": "==5.2.1" + }, "urllib3": { "hashes": [ "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==1.26.6" } }, @@ -100,7 +114,9 @@ "bandit": { "hashes": [ "sha256:216be4d044209fa06cf2a3e51b319769a51be8318140659719aa7a115c35ed07", - "sha256:8a4c7415254d75df8ff3c3b15cfe9042ecee628a1e40b44c15a98890fbfc2608" + "sha256:8a4c7415254d75df8ff3c3b15cfe9042ecee628a1e40b44c15a98890fbfc2608", + "sha256:8eb574cd365cd5b900cc21592a7f8e1bc2d856838f366d7351c004a8dce4c7ec", + "sha256:9cc799df25fdc3c555566bb60979552c5ff4a0ebfdec847e92925f5debe5f2b8" ], "index": "pypi", "version": "==1.7.0" @@ -115,19 +131,19 @@ }, "boto3": { "hashes": [ - "sha256:a299d0c6b5a30dc2e823944286ec782aec415d83965a51f97fc9a779a04ff194", - "sha256:f4b17a2b6e04e5ec6f494e643d05b06dd60c88943f33d6f9650dd9e7f89a7022" + "sha256:63b9846c26e0905f4e9e39d6b59f152330c53a926d693439161c43dcf9779365", + "sha256:a9232185d8e7e2fd2b166c0ebee5d7b1f787fdb3093f33bbf5aa932c08f0ccac" ], "index": "pypi", - "version": "==1.18.32" + "version": "==1.18.42" }, "botocore": { "hashes": [ - "sha256:5803bf852304a301de41dccc3c0431053354144f3aefc7571dbe240a4288d3c5", - "sha256:95ff61534b2a423d0e70067c39615e4e70c119773d2180d7254bf4025c54396d" + "sha256:0952d1200968365b440045efe8e45bbae38cf603fee12bcfc3d7b5f963cbfa18", + "sha256:6de4fec4ee10987e4dea96f289553c2f45109fcaafcb74a5baee1221926e1306" ], "markers": "python_version >= '3.6'", - "version": "==1.21.32" + "version": "==1.21.42" }, "certifi": { "hashes": [ @@ -188,11 +204,11 @@ }, "charset-normalizer": { "hashes": [ - "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", - "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3" + "sha256:7098e7e862f6370a2a8d1a6398cd359815c45d12626267652c3f13dec58e2367", + "sha256:fa471a601dfea0f492e4f4fca035cd82155e65dc45c9b83bf4322dfab63755dd" ], "markers": "python_version >= '3'", - "version": "==2.0.4" + "version": "==2.0.5" }, "click": { "hashes": [ @@ -210,6 +226,7 @@ "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a", "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03", "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529", + "sha256:146ecef2215d4d828e18cc835b081d82aac994d4ccfd5bef0a0a5010a812f564", "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a", "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a", "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2", @@ -223,6 +240,7 @@ "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793", "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb", "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905", + "sha256:3ef25667b6e598f01e2f6990483f0d1d9637d2a3afd3c619acb0d8634c3416d2", "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821", "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b", "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81", @@ -251,6 +269,7 @@ "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b", "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6", "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb", + "sha256:d9e6f4f41c3969fb6cc5aa47bf58649f2f0103b882f312198f7e62af537b7cfa", "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160", "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c", "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079", @@ -293,7 +312,8 @@ }, "docopt": { "hashes": [ - "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491", + "sha256:b2de412b0b73a5f16110cb1becdfbabceb3fd80811441183245938ff135ef9c1" ], "version": "==0.6.2" }, @@ -332,6 +352,8 @@ "iniconfig": { "hashes": [ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:714dfcaf550420126ed7e66bb1cbc893752af388742828f294c21f847d78b88a", + "sha256:8cd395a2a89caee27b08d7879c1515fa9d93e0c477a5b38c9893787fc5e51896", "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" ], "version": "==1.1.1" @@ -429,11 +451,11 @@ }, "more-itertools": { "hashes": [ - "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d", - "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a" + "sha256:70401259e46e216056367a0a6034ee3d3f95e0bf59d3aa6a4eb77837171ed996", + "sha256:8c746e0d09871661520da4f1241ba6b908dc903839733c8203b552cffaf173bd" ], "markers": "python_version >= '3.5'", - "version": "==8.8.0" + "version": "==8.9.0" }, "moto": { "extras": [ @@ -533,9 +555,26 @@ "index": "pypi", "version": "==6.2.5" }, + "pytest-cov": { + "hashes": [ + "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a", + "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7" + ], + "index": "pypi", + "version": "==2.12.1" + }, + "pytest-env": { + "hashes": [ + "sha256:7e94956aef7f2764f3c147d216ce066bf6c42948bb9e293169b1b1c880a580c2" + ], + "index": "pypi", + "version": "==0.6.2" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:104a4ff9f1ece23d8a31582156ea3ae928afe7121fac9fed3e967a1e2d6cf6ed", + "sha256:1efd93a2e222eb7360b5396108fdfa04e9753637d24143b8026dfb48ffbc755b", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", @@ -580,7 +619,6 @@ "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==5.4.1" }, "regex": { @@ -637,13 +675,21 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==2.26.0" }, + "requests-mock": { + "hashes": [ + "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970", + "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba" + ], + "index": "pypi", + "version": "==1.9.3" + }, "responses": { "hashes": [ - "sha256:9476775d856d3c24ae660bbebe29fb6d789d4ad16acd723efbfb6ee20990b899", - "sha256:d8d0f655710c46fd3513b9202a7f0dcedd02ca0f8cf4976f27fa8ab5b81e656d" + "sha256:57bab4e9d4d65f31ea5caf9de62095032c4d81f591a8fac2f5858f7777b8567b", + "sha256:93f774a762ee0e27c0d9d7e06227aeda9ff9f5f69392f72bb6c6b73f8763563e" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==0.13.4" + "version": "==0.14.0" }, "s3transfer": { "hashes": [ @@ -698,7 +744,7 @@ "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==1.26.6" }, "werkzeug": { diff --git a/README.md b/README.md index 92ec3e7..72d2963 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,4 @@ make dist docker run submitter:latest -- ``` -note: the application requires being run in an environment with Roles based access to the AWS resources. +note: the application requires being run in an environment with Roles based access to the AWS resources. in addition, the environment must have WORKSPACE and SSM_PATH variables set according to stage and prod conventions. diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..f49badd --- /dev/null +++ b/conftest.py @@ -0,0 +1,259 @@ +import json +import os + +import boto3 +import pytest +import requests_mock +from dspace import DSpaceClient +from moto import mock_sqs + + +@pytest.fixture(scope="function") +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + + +@pytest.fixture(scope="function") +def mocked_dspace(): + with requests_mock.Mocker() as m: + m.post( + "mock://dspace.edu/rest/login", + cookies={"JSESSIONID": "sessioncookie"}, + ) + m.get( + "mock://dspace.edu/rest/handle/0000/collection01", + json={"uuid": "collection01"}, + ) + m.post( + "mock://dspace.edu/rest/collections/collection01/items", + json=item_post_response, + ) + m.post( + "mock://dspace.edu/rest/items/item01/bitstreams", + json=bitstream_post_response, + ) + m.post( + "mock://dspace.edu/rest/collections/not-a-collection/items", status_code=404 + ) + m.delete("mock://dspace.edu/rest/bitstreams/bitstream01", status_code=200) + m.delete("mock://dspace.edu/rest/items/item01", status_code=200) + yield m + + +@pytest.fixture(scope="function") +def mocked_sqs(aws_credentials): + with mock_sqs(): + sqs = boto3.resource("sqs") + sqs.create_queue(QueueName="empty_input_queue") + sqs.create_queue(QueueName="empty_result_queue") + queue = sqs.create_queue(QueueName="input_queue_with_messages") + for i in range(11): + queue.send_message( + MessageAttributes=test_attributes, + MessageBody=json.dumps( + { + "SubmissionSystem": "DSpace@MIT", + "CollectionHandle": "0000/collection01", + "MetadataLocation": "tests/fixtures/test-item-metadata.json", + "Files": [ + { + "BitstreamName": "test-file-01.pdf", + "FileLocation": "tests/fixtures/test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + } + ], + } + ), + ) + yield sqs + + +@pytest.fixture(scope="function") +def test_client(mocked_dspace): + client = DSpaceClient("mock://dspace.edu/rest/") + client.login("test", "test") + yield client + + +@pytest.fixture +def input_message_good(mocked_sqs): + queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue") + queue.send_message( + MessageAttributes=test_attributes, + MessageBody=json.dumps( + { + "SubmissionSystem": "DSpace@MIT", + "CollectionHandle": "0000/collection01", + "MetadataLocation": "tests/fixtures/test-item-metadata.json", + "Files": [ + { + "BitstreamName": "test-file-01.pdf", + "FileLocation": "tests/fixtures/test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + } + ], + } + ), + ) + message = queue.receive_messages(MessageAttributeNames=["All"])[0] + yield message + + +@pytest.fixture +def input_message_item_create_error(mocked_sqs): + queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue") + queue.send_message( + MessageAttributes=test_attributes, + MessageBody=json.dumps( + { + "SubmissionSystem": "DSpace@MIT", + "CollectionHandle": "0000/collection01", + "MetadataLocation": "tests/fixtures/test-item-metadata-error.json", + "Files": [ + { + "BitstreamName": "test-file-01.pdf", + "FileLocation": "tests/fixtures/test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + } + ], + } + ), + ) + message = queue.receive_messages(MessageAttributeNames=["All"])[0] + yield message + + +@pytest.fixture +def input_message_bitstream_create_error(mocked_sqs): + queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue") + queue.send_message( + MessageAttributes=test_attributes, + MessageBody=json.dumps( + { + "SubmissionSystem": "DSpace@MIT", + "CollectionHandle": "0000/collection01", + "MetadataLocation": "tests/fixtures/test-item-metadata.json", + "Files": [ + { + "BitstreamName": "test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + } + ], + } + ), + ) + message = queue.receive_messages(MessageAttributeNames=["All"])[0] + yield message + + +@pytest.fixture +def input_message_item_post_error(mocked_sqs): + queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue") + queue.send_message( + MessageAttributes=test_attributes, + MessageBody=json.dumps( + { + "SubmissionSystem": "DSpace@MIT", + "CollectionHandle": "0000/not-a-collection", + "MetadataLocation": "tests/fixtures/test-item-metadata.json", + "Files": [ + { + "BitstreamName": "test-file-01.pdf", + "FileLocation": "tests/fixtures/test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + } + ], + } + ), + ) + message = queue.receive_messages(MessageAttributeNames=["All"])[0] + yield message + + +@pytest.fixture +def input_message_bitstream_post_error(mocked_sqs): + queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue") + queue.send_message( + MessageAttributes=test_attributes, + MessageBody=json.dumps( + { + "SubmissionSystem": "DSpace@MIT", + "CollectionHandle": "0000/collection01", + "MetadataLocation": "tests/fixtures/test-item-metadata.json", + "Files": [ + { + "BitstreamName": "test-file-01.pdf", + "FileLocation": "tests/fixtures/test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + }, + { + "BitstreamName": "No file", + "FileLocation": "tests/fixtures/nothing-here", + "BitstreamDescription": "No file", + }, + ], + } + ), + ) + message = queue.receive_messages(MessageAttributeNames=["All"])[0] + yield message + + +item_post_response = { + "uuid": "item01", + "name": "Test Thesis", + "handle": "0000/item01", + "type": "item", + "link": "/rest/items/item01", + "expand": [ + "metadata", + "parentCollection", + "parentCollectionList", + "parentCommunityList", + "bitstreams", + "all", + ], + "lastModified": "2015-01-12 15:44:12.978", + "parentCollection": None, + "parentCollectionList": None, + "parentCommunityList": None, + "bitstreams": None, + "archived": "true", + "withdrawn": "false", +} + +bitstream_post_response = { + "uuid": "bitstream01", + "name": "test-file-01.pdf", + "handle": None, + "type": "bitstream", + "link": "/rest/bitstreams/bitstream01", + "expand": ["parent", "policies", "all"], + "bundleName": "ORIGINAL", + "description": "A test bitstream", + "format": "Adobe PDF", + "mimeType": "application/pdf", + "sizeBytes": 129112, + "parentObject": None, + "retrieveLink": "/bitstreams/bitstream01/retrieve", + "checkSum": { + "value": "62778292a3a6dccbe2662a2bfca3b86e", + "checkSumAlgorithm": "MD5", + }, + "sequenceId": 1, + "policies": None, +} + +test_attributes = { + "PackageID": {"DataType": "String", "StringValue": "etdtest01"}, + "SubmissionSource": {"DataType": "String", "StringValue": "etd"}, + "OutputQueue": { + "DataType": "String", + "StringValue": "empty_result_queue", + }, +} diff --git a/pyproject.toml b/pyproject.toml index b0471b7..27c83b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,8 @@ [build-system] requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta:__legacy__" \ No newline at end of file +build-backend = "setuptools.build_meta:__legacy__" + +[tool.pytest.ini_options] +env = [ + "WORKSPACE=test" +] diff --git a/setup.py b/setup.py index db5884c..b8146a5 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,12 @@ include_package_data=True, install_requires=[ "Click", + "boto3", + "smart-open", + ( + "dspace-python-client @ git+https://github.com/mitlibraries/" + "dspace-python-client@0.1.0#egg=dspace" + ), ], entry_points={"console_scripts": ["submitter=submitter.cli:main"]}, ) diff --git a/submitter/__init__.py b/submitter/__init__.py index 074be5d..f7bcfd4 100644 --- a/submitter/__init__.py +++ b/submitter/__init__.py @@ -1,6 +1,6 @@ """ DSpace Submission Service """ +import logging -INPUT_QUEUE = "queue1-stage" -OUTPUT_QUEUE = "queue2-stage" +logging.basicConfig(level=logging.INFO) diff --git a/submitter/cli.py b/submitter/cli.py index 0047ac2..80cb3a3 100644 --- a/submitter/cli.py +++ b/submitter/cli.py @@ -1,9 +1,13 @@ +import logging + import click -from submitter import INPUT_QUEUE, OUTPUT_QUEUE +from submitter import config from submitter.sample_data import sample_data from submitter.sqs import message_loop +logger = logging.getLogger(__name__) + @click.group() def main(): @@ -12,21 +16,22 @@ def main(): @main.command() @click.option( - "--input-queue", default=INPUT_QUEUE, help="queue name to use as input queue" -) -@click.option( - "--output-queue", default=OUTPUT_QUEUE, help="queue name to use as output queue" + "--queue", default=config.INPUT_QUEUE, help="Name of queue to process messages from" ) @click.option("--wait", default=20, help="seconds to wait for long polling. max 20") -def start(input_queue, output_queue, wait): - click.echo("Processing starting") - message_loop(input_queue, output_queue, wait) - click.echo("Processing complete") +def start(queue, wait): + logger.info("Starting processing messages from queue %s", queue) + message_loop(queue, wait) + logger.info("Completed processing messages from queue %s", queue) @main.command() -@click.option("--queue", default=INPUT_QUEUE, help="queue name to use as input queue") +@click.option( + "--queue", + default=config.INPUT_QUEUE, + help="Name of queue to load sample messages to", +) def sample_data_loader(queue): - click.echo("sample this!") + logger.info("sample this!") sample_data(queue) - click.echo("sample data (probably) loaded into input queue") + logger.info("sample data (probably) loaded into input queue") diff --git a/submitter/config.py b/submitter/config.py new file mode 100644 index 0000000..b12ae66 --- /dev/null +++ b/submitter/config.py @@ -0,0 +1,29 @@ +import logging +import os + +from submitter.ssm import SSM + +logger = logging.getLogger(__name__) + +env = os.getenv("WORKSPACE") +ssm_path = os.getenv("SSM_PATH") + +logger.info("Configuring dspace-submission-service for current env: %s", env) + +ssm = SSM() + +if env == "stage" or env == "prod": + DSPACE_API_URL = ssm.get_parameter_value(ssm_path + "dspace_api_url") + DSPACE_USER = ssm.get_parameter_value(ssm_path + "dspace_user") + DSPACE_PASSWORD = ssm.get_parameter_value(ssm_path + "dspace_password") + INPUT_QUEUE = ssm.get_parameter_value(ssm_path + "SQS_dss_input_queue") +elif env == "test": + DSPACE_API_URL = "mock://dspace.edu/rest/" + DSPACE_USER = "test" + DSPACE_PASSWORD = "test" # nosec + INPUT_QUEUE = "test_queue_with_messages" +else: + DSPACE_API_URL = os.getenv("DSPACE_API_URL") + DSPACE_USER = os.getenv("DSPACE_USER") + DSPACE_PASSWORD = os.getenv("DSPACE_PASSWORD") + INPUT_QUEUE = os.getenv("DSS_INPUT_QUEUE") diff --git a/submitter/sqs.py b/submitter/sqs.py index 7a10d41..b2ea842 100644 --- a/submitter/sqs.py +++ b/submitter/sqs.py @@ -1,76 +1,72 @@ +import json +import logging + import boto3 -import click +from dspace.client import DSpaceClient +from submitter import config +from submitter.submission import Submission -def message_loop(input_queue, output_queue, wait): - msgs = retrieve(input_queue, wait) +logger = logging.getLogger(__name__) - if len(msgs) > 0: - click.echo(len(msgs)) - process(msgs, output_queue) - message_loop(input_queue, output_queue, wait) - else: - click.echo("No messages received") +def message_loop(queue, wait): + logger.info("Message loop started") + msgs = retrieve(queue, wait) -def process(msgs, output_queue): - for message in msgs: - click.echo(message.message_attributes) - click.echo(message.body) - print("Do all the dspace submission stuff here") + if len(msgs) > 0: + process(msgs) + message_loop(queue, wait) + else: + logger.info("No messages available in queue %s", queue) - # faking it with always succeeding for now... creating of this status - # dict is likely better moved to our upcoming submission class but this - # was convenient for initial testing - status = { - "PackageSource": { - "DataType": "String", - "StringValue": message.message_attributes.get("PackageSource").get( - "StringValue" - ), - }, - "PackageID": { - "DataType": "String", - "StringValue": message.message_attributes.get("PackageID").get( - "StringValue" - ), - }, - "status": {"DataType": "String", "StringValue": "success"}, - "handle": { - "DataType": "String", - "StringValue": "https://example.com/handle/this", - }, - } - # write result to output - write(status, output_queue) +def process(msgs): + client = DSpaceClient(config.DSPACE_API_URL) + client.login(config.DSPACE_USER, config.DSPACE_PASSWORD) - # cleanup (probs better to confirm the write to the output was good - # before cleanup but for now yolo it) + for message in msgs: + message_id = message.message_attributes["PackageID"]["StringValue"] + message_source = message.message_attributes["SubmissionSource"]["StringValue"] + logger.info("Processing message %s from source %s", message_id, message_source) + try: + submission = Submission.from_message(message) + except Exception as e: + # TODO: handle and test submit message errors + raise e + submission.submit(client) + write( + submission.result_attributes, + json.dumps(submission.result_message), + submission.result_queue, + ) + # TODO: probs better to confirm the write to the output was good + # before cleanup but for now yolo it message.delete() + logger.info("Deleted message %s from input queue", message_id) def retrieve(input_queue, wait): sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName=input_queue) - click.echo("Polling for messages") + logger.info("Polling queue %s for messages", input_queue) msgs = queue.receive_messages( MaxNumberOfMessages=10, WaitTimeSeconds=wait, MessageAttributeNames=["All"], AttributeNames=["All"], ) + logger.info("%d messages received", len(msgs)) return msgs -def write(status, output_queue): +def write(attributes, body, output_queue): sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName=output_queue) - - # Send message to SQS result queue queue.send_message( - MessageAttributes=status, - MessageBody=("testing"), + MessageAttributes=attributes, + MessageBody=body, ) + logger.info("Wrote message to %s with message body: %s", output_queue, body) diff --git a/submitter/ssm.py b/submitter/ssm.py new file mode 100644 index 0000000..e860e8f --- /dev/null +++ b/submitter/ssm.py @@ -0,0 +1,17 @@ +from boto3 import client + + +class SSM: + """An SSM class that provides a generic boto3 SSM client with specific SSM + functionality necessary for dspace submission service""" + + def __init__(self): + self.client = client("ssm", region_name="us-east-1") + + def get_parameter_value(self, parameter_key): + """Get parameter value based on the specified key.""" + parameter_object = self.client.get_parameter( + Name=parameter_key, WithDecryption=True + ) + parameter_value = parameter_object["Parameter"]["Value"] + return parameter_value diff --git a/submitter/submission.py b/submitter/submission.py new file mode 100644 index 0000000..2a4d044 --- /dev/null +++ b/submitter/submission.py @@ -0,0 +1,136 @@ +import json +import logging +import traceback + +import dspace +import smart_open + +logger = logging.getLogger(__name__) + + +class Submission: + def __init__( + self, + destination, + collection_handle, + metadata_location, + files, + attributes, + result_queue, + ): + self.destination = destination + self.collection_handle = collection_handle + self.metadata_location = metadata_location + self.files = files + self.result_attributes = attributes + self.result_message = None + self.result_queue = result_queue + + @classmethod + def from_message(cls, message): + body = json.loads(message.body) + return cls( + destination=body["SubmissionSystem"], + collection_handle=body["CollectionHandle"], + metadata_location=body["MetadataLocation"], + files=body["Files"], + attributes={ + "PackageID": message.message_attributes["PackageID"], + "SubmissionSource": message.message_attributes["SubmissionSource"], + }, + result_queue=message.message_attributes["OutputQueue"]["StringValue"], + ) + + def get_metadata_entries_from_file(self): + with smart_open.open(self.metadata_location) as f: + metadata = json.load(f) + for entry in metadata["metadata"]: + yield entry + + def result_error_message(self, error, info): + self.result_message = { + "ResultType": "error", + "ErrorInfo": info, + "ExceptionMessage": str(error), + "ExceptionTraceback": traceback.format_exc(), + } + + def result_success_message(self, item): + self.result_message = { + "ResultType": "success", + "ItemHandle": item.handle, + "lastModified": item.lastModified, + "Bitstreams": [], + } + + for bitstream in item.bitstreams: + self.result_message["Bitstreams"].append( + { + "BitstreamName": bitstream.name, + "BitstreamUUID": bitstream.uuid, + "BitstreamChecksum": bitstream.checkSum, + } + ) + + def submit(self, client): + # Create item instance and add metadata + try: + item = dspace.item.Item() + for entry in self.get_metadata_entries_from_file(): + metadata_entry = dspace.item.MetadataEntry.from_dict(entry) + item.metadata.append(metadata_entry) + except Exception as e: + self.result_error_message( + e, "Error occurred while creating item metadata from file" + ) + return + + # Add bitstreams to item from files + logger.info("Adding bitstreams to item") + try: + for file in self.files: + bitstream = dspace.bitstream.Bitstream( + file_path=file["FileLocation"], + name=file["BitstreamName"], + description=file.get("BitstreamDescription"), + ) + item.bitstreams.append(bitstream) + except Exception as e: + self.result_error_message( + e, "Error occurred while adding bitstreams to item from files" + ) + return + + # Post item to DSpace + try: + item.post(client, collection_handle=self.collection_handle) + except Exception as e: + self.result_error_message(e, "Error occurred while posting item to DSpace") + return + logger.info("Posted item to Dspace with handle %s", item.handle) + + # Post all bitstreams to item + try: + for bitstream in item.bitstreams: + bitstream.post(client, item_uuid=item.uuid) + except Exception as e: + handle = item.handle + for bitstream in item.bitstreams: + if bitstream.uuid is not None: + bitstream.delete(client) + item.delete(client) + self.result_error_message( + e, + ( + f"Error occurred while posting bitstreams to item in DSpace. Item " + f"with handle {handle} and any successfully posted bitstreams " + f"have been deleted" + ), + ) + return + logger.info( + "Posted %d bitstreams to item with handle %s", + len(item.bitstreams), + item.handle, + ) + self.result_success_message(item) diff --git a/tests/fixtures/test-file-01.pdf b/tests/fixtures/test-file-01.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a3e5a49335352f0631d8e90b917086b022522914 GIT binary patch literal 35721 zcmb@t1yr0%w_kMHd&iwyc z({Ha;RlA-owd>hc-D`D|D~O2EG0}4%z>+t7ybxfSh!}}%4J{C0d3YF<-Ryu2!Uj$T zR<@=L3I?V?M1HSaSoJgl(N|J{&Sgn;Sc75wZNy(jofKdVfU!Mek!JAC8>VRioJ5@eLMS-c8Yuytv=}}T z$)F5$bz+dT{>YTj-;MC!jpU!OK2~mQZXjgqs`a7DNW{*>OvJ&(tV8s%3>&A9fEumSo@2~8O{~0hSF(^A2*f`q#3EaqyL0E}F1n6RJ1O$i+eb`ktFm(KY{Jjr< zFV2yO@jvI`kKupvAZBjm1ax2!v-((-2++vZ7|0+4v@vxuBVyuW`lICN$z4`dOIEMe z`i38rKfl*S=)0u9%353HcQ0|=f_!D(x`(S}{srA)#4Gd{d;VF=KScVoum2?Cul)Ym zat0-5L#IDfSN>q=Us5>(Yv2bB7=#QQfqx-{ghfO}MQMc%tjrA^%o#*&KJsR6W6Gdr zZX;;pX#PLPf50L@M`<+|=yvDM1(0zXBp+V`lm&TGzh^OdPB%fB94U8_CMZ$e{i&tv@!D zoPgG#711kn21ERn9`OjYdN!tHaNm%gTmHxQ?M@i7x$;{S)O4i)S!Pe2%#ED4W z#N?x>h*TVb4j<>5@JAVZY~p{KSF?36{%6A6)<)!`f~Z8enHiax7&)0Znb?_`S=ebA z87V*BA23;4P{XGQkSAV$$3CN)aeGh=L9xC#L&RDB8+z4+ZaZ-B~$COPn4AuS6OLP9A%*T2xc7>UM&7Txa+Mxq&xb z6-pb^{(5)7lU=jXU3-trdQ*}Ca)L8s^RbZ5tXJ+WT5?OJNGP`oIo9dBeOC=2&>v65 zIWee}LT3i^W#HPUDs3=mIkzafC`-;8L>V@>^?!Rv{8f&BkwlA#nfXsC{&x>E+n<~1 z|Leof^ym0tW&6XyKM%`ah47Cy+g}v;XY{{sSvmgEW#!@|VqyGakCBL(N(e{Bx?hrl06Q zyT*&YWj=OT51mo84pA%_qq!u>3Ngk^B8b+;2!+NAV6x8!C^Mo{gTZ3snW2v?(JoG+ zjhKMKsNPtaGAzpXDV^V-_thzqJ$7Edx(IbIjSqOVdAvPKuX}Agj@j;|rPykI_BTjp zGU*A9Qb~xQuUrINY+*%AEKekFCjOYB#60BdV2`~VCNerwi7VVg`uR2m`i#B3pR24g z)_p`CTsPclVkHPl!#JB>yIN;Cgy)a565;v^ek+ZOS?Qt1n^`03-Cy2}AX|2P&9}zq zS3`%`PtnUZud?8MUw_VcRF~aB9BHS1E`@cAMVhhn)9#UGf2k5F<}JCVe@HgmvUJ~@ ziBK%g}iTBWS3wbC4uM*DJ3Yh5U_|O_gbeYGyR$zbgC25C22?fQ;oBRR~ z?Lz}4vj-ucQ$E)o%M3D^QF8a7!g$v*j?WjD>rLTc_5shrx!FFzIeKP2+X=wLs4O|` z2YccKJ7Xdv#Dq#r*^$FzpUi~JQ`s%JA`r>&Ro`hugyc5{i7K$2E9R>D$q^h!XEaug;vqqs7RzBipmgw<#Byw@1#mCHVoO4f+P=T93@y z``{5=_|9^#&feL9==p-PuO|2uHTL}Kg+B_u$sV)U%66)Fc#qGjEtS6;zqSaLm#$nR z)-84Br#mo@9F-ecV)1SjzfrTKUyW>e4@P+f6-zgr7f&Q#A0O>i)i`N|O@a6>whH)?SC#Mpr7vJDI9%M89ny@!>H&iR`f@GF5GkWkq8# znVkdqvF@`NcjZ)9OZW|q64=>;_`8`4q8F-<7K4l{%!6b*s@Rb<;D%iTHZf`e>d^}M|yW$vE8xJ3aR)kS|}DH=W`lDNeWq@nH)v% z-3a7bfLuUBLq2gJOoJM4f}(n;Jaw+_(TZ5BTeW(u%}1;=f*41bMaHj}sfc=F8w zW!N=}S}B}ama6VXZ{6@n_=p&3`C%pR<3j2AF(rp}=A-_1PH(VLKM5jYP` znm0`YZ;({^348+<4ko z)4TE%78-xQuo9(VO8HdI&EDF}?fxigI%;}c-eEn?s{oXqU<5i&@4??MZ5>_?wROy1 zylZm7YnZld1O3SnQ0;OmoX1LAv=vtYwPivrRvU{oUZTrt2;$CI;wppSYUcOJF-1~v z*G|zlq_uC!68xf1UrC$CSiVOdtPCh^T?b^+-nlJ5L|hTw^c(6GP}a-Ops`kr%%iH3 zwduxR#B<_>^AhXsQo|%ugDiKebV>O~3xh;zwUh06w%(U1ov&fb(qT?!N)u-_LU-xz%?6lhVY zF&)qY)m3Ekm8m-O)$2!;rU5E42T(o(UcLIt8xMd|Ji#x;*1$yo$7e9Rz4}{-P6+8Wi#h_9Y2h z*1tBX`+{w6J{d1wwAAOj=H-dIeqZe~#?C|yMK0Cil_=H_H4;sv3B!zZcB5@K=AR-{ z%a?sBk13JXVdvyt5t(?LDpagDQ?Qkn)Ui*3aw$_LG}q5>B(y0a=_k3;k7o>B;8Hay z(NJ%`pe`133~f5N30O3ipH$l~yD>m~q=XjT=vB8oZ?Y<_D)$sq@K}k}08G(|K0?OB zGBMNwg{Zsnzl=E#EtD1vm>rJqEE?4g3|VEFsYy#)(<;^VD!Vi|$R31`SHa5;SKI{d z?LM{TW_Ufux(`$NeTWEPI z`IBmEZWlo;w8&vh$ms*y$_-gd0gDH!g#1lya}yI8+@wE8`jhup!}QfRr4;85hUaB= zAiufUftcxvx&|td@sJQ(q<#;O4F7g|#WG*Md&C-?SZTg3bDfeOU6;xp=*jM`(0E70 zZzIcQ%{_cMVn*OeamA+OBuBO!fS)$l9KA_oph=-Yiw1rYJ@+j8hTSPzLxT<2gza%8 zG&JH|G1*i-vk=WtV*Q3^YZH0d>SE3&8!Ez13V2BU>Sfma=@@{LDNcZde=4XiW|tP2 z<9&*Tk{4)z$&JSpY?m4DYSH(irL?pNP@#4~N{na@v;Dl)5&I?oK`gfqN}6h3jf30x zE)w%iXp@1gQ}7`&`H?4O8yGIzR4^PIiA_r}sGUBaN>h4M@9d-u(LT|9r{Z3dXadUe zJttWk-{z_OG6auQ0rEas!BMus3>pqXRKsZgXR1`Wv<9(If1sg)uuMLbNW2BG*IhUV zceKGGi(<2Xeo7msbQA*#VZ3T~zKpK7|0yUCeo48APm71DvzGi9hro(w^}JCX`=>C$ zmBm>iH*JY6g9KIyF*r@#ZDCc}gy6kF<#HhR7ek$$lAg?}k;)n5<1l09DLS?ht4M^t zog5)gIO}mVk0`zobq&eL1ppoisB0sgu~>L*D?BhCL=0METSV)k9aI)0CpyGpcx4fN zgT^W&cS=)M%s9AQW4qM8MbV9grvpr5k&Io?F+XZx&=Q@uNq@D5yxw_rPwVv5aPe#J zA>|&S<7u}LFoAA>TZqN+`HqR;xWVEq5i{;K7;e%vr97jYE1?|Mp2oK?U}i6^NCZEj zspb(BN^UFog9?vadS{|`%ScVcu4gDvh z`B2dr?QNLx9%J#q0u{Ofx~(uxB?6qr!8oCChgOwP=U+^23b6-r#cs7lrPE4*Wz-Bv z?(OvDV%0^T@0Il$IAJN2d?+R+D{1EF48vGfm1%T{Nf8U5j<~b1C?zCL1X!PPBU=V* z4Js2c^`s@VnhHLN-7cxSY(s2%NLE(0$Z6*s_aU^%(Q1Y5S>%ygI3_$)Cm#Qv??(j> zuCUE~C^kpIx>-_b#)k2b778~tgKPY~#L4Tl#5PlEryPnll-GV_fd#-0uqROo4WnYD zkcOB>MXu0DUg2IzcAm}iVi+&bk5i@9WGBZ?{UTg{OV&HoOiW$?l9GGvIUJF_TBIDy zi`a8KQ0@>qof!i_iD>&=@xrM%ORk5F;V=GGnB=(Zx;}P)@(Wh~jrc1k{;Z->LQy=W zI;bP9yx0SQYIa*6*l^nKg~HLN?o*g0suzNEKacn2o89s10?1@{D|CBN&`Nc7Y zrRYleR7r-l=yCbURDBTCa`Cw`VV8i2IKy&uqkO7_T#+zC>aculx;`rE7XZ2x6CeyN zADg((jygz^NjVIRIx5L91l_*?o2*ZO8cJeHFU%XTBOOaAsz?1K5lh==54cGr5f9S^ z7)!^Jix#5uQM*fRCUlbqYK46U7)!*`2TF&rQUeO?q+&@#YtbbEocR**B!aN5=w>+p3rR^oc`~mne zTVJU=61MQDI}*2Oso$vkk^m=3TX58TNrqYIS(29}=#x(wB+owo;ct=vC5}spwTwHR54v`K}saN%^jdVOaUD zx?%hIuBu_C`L3E_{L+{4=<-sR0qD`vm&xcpKeiO9KgSw2p(9CO#-JlfUHYQmOWS9m zx8|G085W`6OVtR6odT@lNWO@cp%YV|#ctV9pZ(a9qi&S44@UQi-*Tb0i8BmGZ<4YP zN9T~Xk3;8>s?i9O1*{VF-2${J`x*e+q+$pd zG9g_Mb5BTr&ab`h^Rw;U&h35k>aAz@HF)G1De3>=#D9cyYUj4HdDYmnYaKkYf|%4q zKyUk1d)?({+v!EFwNO2^X|P9x)Al(2Su6y19KbZd10I}B>BuS9dGj;oq^48_F?ZyF za_-z_gkrYJW(3KkRIXBps=~U16dkFi)OcJEPM)B22qqd`ZcLzvtO#Y`jI;n;$Q~LS zDjV`#@JzU%b9^azDRHW}vaGTwYmymaLYi!eh6qcj39_G#BMo1CvG0LvF0qJzNv)s=cu69t$ZP16BJfefO}UamkuX;~cRR!#Z5R1A zx3n&_ws>5IgiukwbXj^Q$H6A|vj~pJp$L6$z1^~i0!=JSIy+3fP`u1I83{BOS}1DB zNxUg(7^NwGY{HYS(m5!p-phHi?(z!dV}xZ08CfL7`??m#;CHU1v5fRA!n{03*&OX>#6!CUBt zX=)?%O^ldO@km(8OQ56^wOzhG>1v(SJt_Sh`8DvsI=6{xM^>s`@W!1J$bHaYH_SR& zCgL&p2s6D?h#;l1c6!bYKqq$TFq0EEml1emL#r(+T^v-i8P_E2|2M$$%=5GT2m~GB&-QEXv;eh?CMB6AxzDQs44o< zQaeLV?TRERt`{he5Qfm_w1tP#leEN`kxoIPyYUrhhK!SPCz*jyJ=p=+4$|$Wc_x=c zOatSN1Wi#hP~>2JQ+p&ABOVVo^NF*aMJ!7maB05bPL*utYff6wbcC6aPf?4|h9siV z<&`D@F5^NI1L7Su z#WQs!z`76B2znM5P_*5 zMF4@9ASFNoXX>FE{2q)i3Sbo4A5+JAq7F?1^OfBCRox>`pjGDif`;2C@J(!c|mmCPy&6j#R0KLjf1)Jp)Is@7+b zo_n6b3d@ouiz+(ozJT(Ec(`CViu6b};e)>u2j*wfPSF zKzY?6;?92odBIhU^fuuuAA66K&kxE6^@jLDSNZEzQ9JKh_&50Ps;=jYQ^-!}Hn=wb z8@(=Da9dDYFk5wbhfbT*b8m!~(uSe-fz(rPNLQREa?8qR(ch52-FEn)H|pjC&rGwk zg5Fqel;$DNC@Zel86Lb;!t@y3@ z(sfnrb)0E+ae#B=j7I(*+lyOG%eM9H`1}*`Npsd2;iv`ISqE|>p$nsHw+q-+&_$?l zhC2Jr{zT+nv?gv9tDpTRNB{a!2hkn-J(oq|vsFWv7bb&U7LFNWdQAhDox8rZ{@Q#4 z=(VPe?y;s;<6V7&)FqGr8;YM&W96p< zRZi6}dO6FDRA-w9_gk6k7<=Ov#wsJ`hez#18@3H%PPxG=>cux1)HQ) zTHsKlv3zYR z*;67NrFBtT!d`4sxe*Zo$OT z&sbmtJseO5A!u$ajYUi?F)%go?wM_km{nK;c$N;NP27cgrDS~8efDlmy{SHhOhYA{ z*xvrSh2g`4OGvmMrPM1fo_-X4?Bi+PHfOEUD;}&Od#AOD@_GzR(N@H@bxBJQ$Z`9j zahV1eF7>+C@hbIRkhLPhe>l`Uvj5v?ymq&FnFEmE87Q#_TlJvZh;M$`AS7M4n#BqW zKBfbuMV~R^(=zMRdb!aG`u;4|ImA*b%#}7rV+v{tW(qRab=bwNPgf1aiNJ}Wj!+J! z3RVO{>$}^<{M%R`rW#uv;tNEoZ=5e=muEJL8OV_zcQ$4*=pN{t00tvgGWaOe$Cph% z9077nenR?W zc3_NP*og4pL12(wmtDyEjEtYKKf!}Mfxr5C5+NPH-GDmyZTqeZAWp&!Le=`pbV1)h zwE05Yf%E#p6A{>fnhVhUh5+~)6T#SlgbIM^Lt6>(AVaHz!s$a9`eAoLV1t5ZLp$Yh zUzC8sc7gU=aCx>7aDsUv;X@&ULi>Sted}uK;^J;7E)1XaLv#xIqMtt$z9Iw9R{=F|$@zVvUJs zg(==r!HK~gd)mPG_m)&^6Rkx;jS$}Mw^7~q-S?-x{R*!JZ{oV*uPJ`y%=7leMtaYf z*|LNSeU8+CQ0@su`QVD}Ya`FBCcR|?OIsGQYW0e86*rvqs)yJpVNQqGGi;LSVWx*>GOda$FK3$$a zBXq_q_0oKCeG(CbIK$ z*3#%Fgi$pVFHf0txcQ5!OVm8~jm}L4`Sgs+Mb=V5FLkUvdaNn&4~v|8A`1q` zz8Jn`QYoFv+xOz8f;(P+ukB6TgSy`r6K1g-^^|4do?wZt&LVLDb9gk zHPZIpOWFvpY{MUVGBR~e_+0o+28FFSTvEj^$$62w@9lb@llSs&@=oL>!l|Tqe`WR} z-m-cG@s8-~N0Z*R-94fM7;2N2XiuDN0^LKszE6;zh`t7YiHhuwe!n+JuQGC6Oy6+$ z_I3Wo{7b0y8Ruk8?Y1v>;@mZ?Hqm*n*77)gm;6b1l$ZlY_}ps(*Wz82?VQH%bW+ zGck>azg%+nwXo1tOg1@I8{}CKF7!X!FtFP6p0-^Kwu)y?@L!#E1mrwEyS{xr?|kk! z9oat>zJ~7T+<$$0)%F%obv8u{L%ch+!Y#*TXD_)EQvArb@_||eUYbH^9V5r-_|laD$`8npK5n8e&Em%%1}tdcefErtQ*jrEG2~MU$8z>r$B`kGfJ3wreJi)oN2(E$X zGmUxW>1(gA#JX)p*fu713XBdeXqAd+ z<ojCb*CnyZ%&J@)z=oU6%inU$YYPMYQ+*I$AIoClZWQQ zYK>E|!`NxCZN}DF!PmMZkEbz0Jzk%IrR~n9Xe;jkdl%CI;s!9@^S*J^R~t5pM#U;r z(srqc8ycqRvR;WQ)D`q+L`$gSSObypCUS~G+-2AC#?0LkP*rv86vxNtrC`85lzw*u zMc5>PY{)6~_a+YWsqyaa@#X^ODSGZC5r7=Ql4*kwmL*%G>GhhJWx3ppcU({0-pL{3 zt(>A1+qqHbp^MV9dfr$<{6Xyi{^Ot-+tKaA{h+8sfH;OIO zY^4lk@Cd6bs1?uA-*pOL3|#Y~cQJ?G?nYS#6LRQ?J;-8v`DGY%dB|TM+A>Y@bQNVy z;;846_Rp3kj99qZO2#uoncAF$t|Q)oqedZ^J3-dcs1-0{rw~~ zs^yXDOakRbP5dems(UdGbu4|U=OBt~pqe3SDf zC&KcBgq)Ln3SVG4x1o>iK3711r_6E5d-BL(lw#dGr|}qyC_M;7(MjTI3s(+p%E?|* zq{iR*m2y${#Lo5@s4frK$rLFUQBExm%pDvi9Y$=@-g>rkjb&uCFs>(F4m{l4Hs4u*>TyFA=9TY#SxDEK`C8 ztMN#BY(n5$iJ4FS40@}J7Fl|eO+%O_sx``Adf}0-n0py&89h|-qSs6q)jh3O+vrMG zr-3NbcZSEchzKO^;%MD*A}@Co6XYsX86k1ZU$D5QUsp>VEiFoMlcdf?G%p?OiEpKlm;h_2mxRO~4p|%Zs zetH&Hh7Uoy}^Q9>qt$JpN`;Pz;jEi5H2 zgLlWZGDE8cJx~6&)D}lYt8pQ1m$77KUG+J67K*%wGqY{-jYG`2-jq%PAhKHlnIWWPsRgWWC5rv`zz~MXro!l8FxBp-Q^|F zspSn&5uL=-dxH=o+;?B3pOP~4R!+B_#OgK&H$(~JA@Y`rg_jp~`|u*L9Zo{}@5&yn zp&r7~rccCD?1!Y}i}X zTcmv^ZRDp2?q&;FUHH3=o3>7${AEKf4-h%b71t6YHjCscje$O%p3fE?z4|B_)!54n z>;|WtH7*zKi2Va)wx;gOO=(sop)lrV9)Y3UmVm4~6IW8}eV{s}@0#Wk1S@$Gz>m8FYb?BIcvBl~2)PDrU9js#Bc`(bw!<=Avqp7A#OM6%#e zqvbnCc+{&WT4q)c6oD6CR2MhCrN7qgb)RyVo%!j-TxgFCBVz00vgP|@o{f^`uT>`V zS9jBFrZ9^Trd0gO(fZF^)HP!1XXcqAXcK9TW`=}$T;o4`o7Kf@E~CR{{FutQQNw}e?uyu&yj&%P76v-(Jj^B z59dD`)2>@(F3g`ve^KNSk#;FB&*7{U7L_x$W>2)T-yW<@j}`Wy8>#tZqvAZF4+6$| z*w%J(4en!5*wR{B`276-R8?y6SSRI_5B(Mt>aHu=+>KVro1Q-wBPuL*30MY^%_dIr z{{iicgV_-{H-c%G^BQ-vbbCT*YeJ!sw_tdUtL|_Y=adcQ-Un-ONBx7%)tWup4?zo3 zJ_8B2lS(#!j$LoIc?B~)E-8cY#@*!mLeP4rk1V>^l6X*M~vWUk|$?f$+~{nC@S;?;EOSug9NF)+mu4R zzE!wr#fgfTbo+b#q*7i!lM2pfQOXRAcRi8L?LDQ^#qqn(6MUoRNrS+r&mKhnFjGx z=`G+bQ}heORid{boHZTJv`3=?uA&!mc?VJ`PHkt|=k&2eQ@YCYt$oM6)yvEtyldxO4UW4#^xsW)?4)&m;A236ZN4pyzBsS+^GEFw7Q)&SKL^e5G+5yMctOvtmfQeJUhUXq6OpD?*k-lX^33R^`$6+`QVXekHE|^J{$WR~YHpXQ zws#*fqosk1#qEmEp|m`#il_bKR}tsuOKeh~ix;6#t=ctnKQ5T<*k|G>|F#x-d9~bd z0QDCQHm9F;REa~mj1(h>oQ8}Zeh(_2p$1S0h-N|D*0UiW52`KTuEyA=o1|X&!k6o{ ziFV!M)~{d(9W-ihcfWwvim>CIm$wTDIN!BoT|ApFJYT+H*p>vF#l-LwGwt+I^cM~c z47>TIEIwd6@K&!N%64%^1hUe=#Wu8&WWT-q6xIdt{$6B|Wxwz&)TYW0lLzS=4hj)%aO&#=}n5 z>p92NhPS^c$dT6NreM~g;kyZe4xWqYH=aiNmp=Jh87DXPiZh<-?o!nrWa0MaJY<@a zGyL=;B#hcJJbXJD0S<9f$0cxjqqQfGg)#M`uepH@I=nIBE5y;6CU<^}gML*8pWV1M zVG>A#Ipc&CO`^6Zp?5?PnP~1&FDFMXoy9XVM4WFLZw6D@|mkxpI}S1 zdj%~mrvQAG{KhO^VqbgVb7LW!{j4wAR^ht|yQ{u%)aNFeLRxlWHt^E^?l-#-9K=%J zvB=M{PenVmQ=jHh-%Bel(rBLvs8wp0w_dsp#c#HJ@Y>^rP!%c^`%w1c=uk!vFlcy5@#hs@rs&ySks z#yE$oW-x||nKU{iEPL2P9mkn69xg@uu_r;g2Pw^qQKRFwu9K`8mvtaroU`mZPE}^H z%9z1_vAd$RT=;IhupYN^+$pl==F~Uu!Y}y)WuuD#Z_A`LWo{WBH&3pjRwMHCo6gVU zNwssmCC7{Fynbl8UTDg+m^1ia2iqunh?7vCg%|lJ96`36F?S{L0XxwhOm>LWkiTOf z7W~*H$Sj4iDmyYc?A#3*^S`j&E%y`{PkkN4sQDHvwrOZZ;ixhDIR$4&_W&<}9CThL zNc!Uh_rN-gqQRqH7N`q}EMFcHs42)NWmIK-{4_QQEMFpU4ye^VEptft&UBzFdb9}J ze)G&36%>2s<+s)GCZL84`HMSGbc9M7*3v|FL{8O~1FD_yUAa01`DP3B;DgXYXC!;}@t-&>d!bH*?4H<*ibHIB}Qo zMkzLd6JjY7K5{f`tq^it$|$@88FSL~Vul=%c6!<_IcUSFx(cd-bE1q z@lhqvtpry$I$rt<7LZR~QVmxWBz)H5$`1E+AgzoZe2;k<-A_zqCw9sT5 z#0Y1oU7$448YdVb~0{ySynlomNX?-X`z8A4iNgAY(j6 zHm5q!JFew(J+_!;r9vrZ+4`~Z?w6g=9vYHi?f#I@;1fp8y6r7*hymM6Ej8Sr+C)g{ zYV(4_ZZYMAE1J>3b`R0BsN5)n7Ni81TN}El#3$I>b?uVYWSlSlIdEG*|-r+vU!NP%FxQb{l zuU5Vu{q7co+Hi7W-D&OVI^08Q(bJO$qM=FX`voXb5X83F9hFcKQ*ei8atI(5cO2`| z2-sGSR@4i?^}Z}fR*Z9nQH{us-RAigh!7kh+6k8ZnF|C9Gx6qbj!=F8wC$U zB%M`l_p7b#z9n}_V8I5y0Bh4`PB85o$7Q@~;OVJNRfdAdHafT~-%E>gJ6ez*mLsDh zmeTM@FzH0A-Kz+6>JkL=^m3>k%txYF=}I1(qIqlir0Twu#naA7u>y&)VvW2%4Gb$X zsb#8_>!iuJ^3X)AG@piwtngugT^%%%flaFnHmxE0t**6J{48Kqmw;N93N+l-$H+}! zVumgQ9SbC}yPZC`Sl3-d02OP8x1ok9&Jxm+EIxuU(!s&wfP5H3L|6v6$R`sf27{2B z&V>f$<2dwQkSO2R>)vl6Ai++VNWhnEHn7UwjczaKg}zv*&p4r(7{~tdT`VcCM!UW? zy~+9&%Lv2gZ!OvJ4tA-i%Zu~L_hcu(g)tR0Cfd|hju)QbsLgFfp(B~PvHs$g+|-s| zA?YC(zI8t&G)9YHvD%1loQNHFSaNx0#SYpfXWGB6gDjiR)mgHuU7Z{%Y%RobdBI4w zA$+IoA!6>!UKx+H=1K6PN2K&@E0UOi4V$zgGZ!fbF%k_p@7csKf?4+vk&jwstgznE zK%^+&4r^UCYQ^u>DTs})vELLg$Tg(A_x8@K*{IXs|2heqT9i z27Y0+vrC>0(~f<`j;iGc^YOn`@@VzKc1ztS(Q2}QpsSU+Q@Q&}C?Z_0S|@K5jNXit zZy5VET7VXMlDM)3uX18)D6l#}cKz8xdE4Mr7>$0f z_gxu$zaq|SQf9e%GIBOnG+5%~yF2JFR;A4dLby28ksd+5LLlsTvhBsC03Gdu?ua1U z4~*(6?eYWZlh#1dh1I*LLGkJI-*V6m8*!s9-_#>1@+#-R+D|M1Xw){2qrdA_LDH0)ZC?shX@WwesA0cI;@S-2{Ljha~sn`+fQ~R@@6o0xPz_eXITFS0tEu;#p{C2sn zK5!}OYk+i30M*s^twR|McQm4~meiI!Ls%eNr~G@+ENgg_|lkZ+O3K;aX6cxMv^ z#nrYI)^)ZkY^<cqo&)xPqLaUMNet!Y$sQjAwbyjIF zdGvSv1$`93i<91UVJ4Ycr%5))&r6A{qHZDP7`$_ssLIZ(p@NA~$&IPovXjv9RAu=J z`@XXQ=o~0li#v#0-oabVYOwVFx;uq>&qyg4i-KNMpaxU|s%?Bo1EwQGx4hq4@aTP1lUsupxC)N>kGa2DcR>e z`_h+U?i=^vVl!J?Ryf#}LbLE6m;Kw*`SFsi8KSYDC}Ej4Du32B@>+cMJXq?XJ@ z5o=Yd%hLL~%x*7(ue6h7U#L0iBg2MlkO5I5Cj?LnQl&Jg6rRKt_*|;x3k4;B4gOVx zCtj`hsfW9+X(A#3Kb$i**>6&ZM!J6630_SZ){(8^#|`};8XZL;HY?5}5KtI1LRXo+ zlxSrJ^96lC!Bl#@;1W{DB9){#qfM(yfoiAlL4PJ;zY?Pb8qb4-TdGo-rru1~d29~# z5w|x#`ge7vA|(_zlq$k1$HfWA47K1SwZSZPyJMjya?h9YU>+I}RKsH>D42|HTL zFQW`q>2^{OhDNROqJC)<8Jon&Q7@3$v^|lY-OJlceHnaec>I*3LoGmEvnV4NJvCl~ zSk;4U1|L|*_SvW0OIWKXs)M`X*Jz0|Pw>3n1^YU-1UD%Ho34-b3OA8M+|mF(i2Qdw zA{%dTRy5mRzx8Q*+t6fC&eIXZBERuRyo_UJ$Tn3u*3J~q;C!V#%z6Gbxh5)qXqV%5 zyx!d!)D#kW&+@gWn$XQGf6 zrZIQR{v?G)gl{v3hh>1K-3(4diKw+7qv9f1dywJU`Ef+6d-J%paFp+d$y_EbzU0sI zc^qtP^HpH@H&M%$xI15Ixtt*xA7LN1xGDyc+cI0rrat&K^5)xL&ZGnLbKSp11(W=- zhPdXg-4mqLT;xSL>N~BC;(9)OUcbs!d7vDcHFsDwoT%d@4wd)WE%>orJ_J24MC)y5 zqjr6}HQnKRnh=C@#Xp{)nuItu9NzsAUPGz zR0=D0YgEEf2+Q;W7IM*9($|^JExeFSJ{a<5bCyIldvc&grfZD6e?fvGTrXi{8;rE`^lL2b5otnh z_vy3r)~90?rn5JfCGD~PQhl9N=vg?&I84rb=HkTUtzyJ3uP5@9fO%cPN(Fu+RO4h< zD3Gon#8+@;#~^&~nDGa4OmPpMxpBg}UZbZ`hcSuDN(eRMBv`Z!?C7#)+$^m5$VDs+ zLTln=cHG#3OUb0T>J7xIsG8RZeK9Z-`dd$~P-v!`DA3bwkKY6zaX1(TEW`D0(tZk( z5!^+~ukq;qK=(;{gEr&tAlu9pJx2~lh~8RlE|lI5fWS38?y{??TUKzm_h`4LWhl%_>glbo5|NxKlPk&Ae5Ws32etnX-*EnHa6GcP^K6{`&OG#bx? zR-=fHp0G$os#uLl*ml~IHML+-5P3JSqfQNLf#bxb*ftZS%0TXvXEu_P&D#f}Af*NI zo4reKt=)s)MQUXb<3uoC%RHn0#KYS2`Ff|ghI{$_+g6U@w_ZNoWu+5Q?C9#WZ(n%t|LNsfvT@EaNC{P6-<7{_>;Tb1L?6bI&B<62a-$ zuc}K#>jF!q3PpO?EBp3I*!3KoiDiM%Wh&`Eez4|R(andLibf8`03)f614w-sIQ@5r zW;$SLyW0=jm)nhuJR0JSu*Z;{3KZV9A7hN@YQH9i=4|PEFsVfJwT%s#)Vg8V#S4kz zmVngl&`7~%fBeE;!qE_?yW5{^{|4f0m~QkIGPT*H7%^42OkhwERY;U2vcb3c1ZMou zSE~PUuh|D2HYK^-q-MP?pg+TiY3a1~_Y!1PXojqE=vX=+XG_{RrfQ(KfX&?*7We8B zLi1AC3nBj(qt=r$E`$Wy7<-3?|X4h5;4e~ zVYgx8TJf;w$u*A?AnHv_Rg7Cd=uytZbFeE<^&aL8te&rlIz4jQ6}`?u{=zXM z;eCIoF%Kc-w*CKT?<<4rSdulxELmVNTFh|7Y%w!4Gc!vTGh58ej21I9Gg?d*v$VQ* z=FZ+Xb7ObsMeOdM>F797ot2rL^>y{>tcLmub`8fJXHC8b7yHkQkk!9I2#cA5pBG@B z(rq;iez7^s2f`j49|T7{*m=h)n)__2v&%*?CrOpYL$t+{4=y{CTKB{0@{MFMZ5*6- zesp}Ot@o@hctHIa0*jj&l>eS1k_-i*VZL<_%fHb{UuaTw;%!J&%3>*|vg`yZ?mb_* zjA>Z*d(}yCWL5bAXq0-E6PKnJ8c0m;ovlwqBj>6)T89k}45R077Qk`}Ra$4ab#>BTV$}sk zKi5tOQq~+VH1>2W)z~J7Fxsc)nzA#tvf$ktw}N)_`_nCa`4G^3G6!i*1@3rtm6}3DW_aaoXir^7Wu0pC+Q7Q#1x99~|{%=-? zY@cCMZDp%7SfBE@G8!qRVX7+{#PLe+#S|CxFLU=(QT(sOsqH)0**w+|&%^^cXLt(a zD{lwor8eTph+dm+KWEFkf7f0{8G?u;hCgi_3RBkxf3K*JVw+|Mn3mOFt50Sk`hnVN)Wwk{2ZfW27{@JJW?%Q?t9nc+#_zu|a z-c1A>gOT_MFBS3q;*$(?1J?_Rg?>@CIbBpIa?!8w@g~-ZadsrSxcZ2=ernNcJY@^! z(mX&#uVK&_o-&^sUSs~#Y~pvd?wGMn3wUw)Jm0*DuVfD(`m1=SfE5)@|%w)gqc*jx-LA zl45;m&D|4e0tZRFaV@oJpN|zIE~n|Gn8kq+bf{sKV z26c0Y-P>Ui*l8sm2iPY}lrOTk19zl$F09yg72jS`&kg$&HJPgk4-}U(S1z0EGuAFD z7{t!nr!0VE<%W9(l6My!A}*$Qqy7wWI%sp+;Al7lso}W*IbPYl$bVxND#XgF-%FmV zzj_#bTD;BA^g*t1u>Lf#^dg1vC|i2hyd_1VH!8au4`N-Wf;UL}gT>6bWM8HJ z7OL<7BuXZeV%IBf!`!s0O1o0QYzd8Vq};eILixb1^BHNBVj^9xCP!%4gc37bz^o5433&nE9sV1#>Nq1u|yrxI*|`!e&L`*i%?n@60ERy zcXi#SeWk6@r8`YOJS=&V`k8JQhcrGio_Wu+pL}LZxOkP4R*EC49*Kb*CAi%3uFB&X z`8Gh7NBEuBvnOdSkm^y1ZukFDJV3Tn}FDD&RY z>$1sF_>AS-so5Os83{yk@WK7Z`v}}M*%zTCPk6o|<2B0;lEoBgR^~xolR$B@ z+OILL;h4e4OfdXxCnEtk0-C@e?=0$g9+}X4mkfX|n@(jb;bL2(`5wO}GtUemn>t%8 zRqijWkm*t1)GC#h%(*g)m2#?uxihiS{Ma-zee$&xvF9QrD^%V|55n0MDMiK(%9=_P z8{L1BB~~Vem>gL>Af)x<@^G;Ow{=PGPn0#QyLL8#xYn&nE<3SBOYh^F>O|vaR4tq+ zI+vvJ3?JhdZ*&hswU1#qa=ZvD=WO}Ynk?nYs`f~$#Hb*`n|g4A^A44nOApge^*=A2 zgsCb~TJ-Ie)=(rgVv%nbRlbESecJHF2@m5eH2ioz;E8vZr6Nn;=#LZrN^(X1*$t-O z>MndncoL@fj%;Oh=g>yShOh(q?F%uF*cmDBAWr3e0@2#&Ex)#wq=7d3)dR-mvA#QA zReq5ndiwVGPk+FKKE0|FAbjR$ovIfpIK%T{o5L{ut!Ro`-)5pjUCHAC8v+o*DMZ>4 z%?-V-FM{efNWrfl>gOM?G%Nc?>S8y8;{b_td-V}SmETuw5S62(Ua&1Vmm43;u7A0> z3*8E6V=Kl3d&FxAmN1r5+g{2E?as7pG_l>8V(@i}vTopbw&4YM=ATr0z}^?$pWCe2 zY;v9Bu|`;KvBuDx)jAtlX=3x*U%XwypIy2>2VK72+rypLqCsi5E?slZN`k}6U{WqJvEAM?wy?a?FMebR@J?{>`wW; zjdqH($Br4tlT8g)0ZJq(L&WQ(%pNW#8A}v{aQL!RP7lEzbqmfDs)*~39AiX{>L_+J z4tGwKZyu=>pTQ{QvI6nNT$!Fb0V#8zwt?+n&{(oJ&V%s#7H30)sDxBGQY4MVsuax< z0|W16>)tb@jCZSTZz)~Y$cDlV$*zgE$S>L!v(p1s@_|T=h4ZFo z85dxG_7>ejmNf!R+NxBCDp@3B0x#6eF!KLc=*hTO3Q}ab=K;$+RXjGK)7fLfIxpo! zc3wT}3(y|s+P!M)eb#(i_L;(pRKH5Yi-+A{6!1<{?S28vDAF z(j}WjXz~W;D;B@I@C#lr?GZOZAK8=#PKcDr=SEEUwnfG}%;mG0G#qi8?=gE*OIx#D z5(kENn84baH(Cg&7bDhG z+>Kb2FtbN%mKc90^;r61g3+*sd>InwD+f`b%h@sLj{TIy1QsWL0}cWYRnjaZ^ut1S z*h^L?bfT}~s(?v_-+ikrTdyW{zV+vUK0w@E3=WlEnRCyzB7wnxx|2BQ>C5Qym~e37 zD|F7(B}ddEWk?g0-(I=#&${MF@NjevK$X1DB-R*}xG zZpjIq@C&naBvP}qqwAwX^V4?Jk&alkgEzEAgDl=7$7Aho28R0dIcVb^qu6&kspCk< z;u0F|7pG<$&7O-+WoU>8xuq{U{U?a*NwkXAzetKOyrvXk&$4oF-UPM#Y?s|^|cbedN>EX3MWr+a~0%&G0Si$et2&DL)vXH&7XE*hpKHgv~{QFv3C|e$O z&i9#=s@^0e(d>`)=-w}idiKNif3v>gP34W=ymM~1%iQMUd&?J#+CRX3ocHFJNP+iw z=0^wl0C2w>9Z{o}QUuWQoA=m}+R|%kHvacC#YBJgeeIHRAniNgl5*nre0d#kmTDi$ zg!$~mS5ZofdVt#_Ps7(Oi;>n2D0UxyN|bL6Tr>${UFz>blnPzuuYrn%WykbitrWzJ z)aKpL>Q|`c)ey#B4hm{PY#Wh2Tq>&^mwg>;RG=*LUB06zDc7S|%@Q}FF}-?ISP{Vmb`p18ZQ9 ztp@)GQ)F+Mo)wX;1F-TEU?J5N=t)Vxyy^kUrB&in1Kehmq&8i|J5RFHd-^va;6&?g zmz_W#NZkxhz11rlSE~DTFice#m8jIOxNonx)xi|1i?BW}kT1A%gS573icR= zqTfk{`ou*+1ZO1uzFC1%>R%AHP#(4%t4}*QEkgK(;kH(`-#1XTl)p{3$AA60ws`OL z_T0XC&n3MO<8iQY&3WLm0*9g;bWg!hrFH2|q5Z{(90;X`y89xZk7!pNdD5ar#wa^{ z+M~fa`H(^web1(4v$|u0 zKP--=SgECaNeuPyX5iJ#N17O(z6Xg4-_3eYZ<*2$6n}ok5~^$&z)~Ol+>1pMA&i}W z(gm|3kZ0aT+{54p_$oI@Yp{ppPf8d;$Fg;8=OotM!EeG-p`yi80AxeTqgjR|LtjISF>{#a%C5L7U4P9ruY%1F9jf#9lrU_F)DX;b zw$T;LiTDT}HcCZRM%=bJ#AWSYBVBYnm9KX6;zU19^QCB#TCqn|DVZ#N%*c2jOIO zK6ERnV3y;hXf1Uga(!J)z-K{kGW6tT_z7*!x&O)Ci8&P55ZVgXt3kIsAvKli6ZCnz zxTc6;R>SuwZn=!&+!NoCU(y;vKGK7a{EKAJ=LP@)tDL>;1bfX_Ui>^qR23{P1*-Mr zFM#;6Fh$J^Lx@Z{rKW*fdGqRS*FZhPAJ&Q|V&(;PaKvJ_h*~+mYXvh3?;W(_Txlvw?VCzf3Yq%M$`LT@PAh4c{c445R5Li_ z890G?Y9a66GBLoJwJTuJyqJQb^jxRuamFGN>MK6Gx;1C={~RRKW{8$MFx19^4DwQ6mZcSsF@??pL=+dh}xx@&+na4C)3l) z50XyM-j$eD=YO6W-Lj*ZKz55n7-*}+me83fH484PU^-)RGH6K*g&WJ;;l0VL!Oq`p zl}N=b357s@iK4e~!vv=4BW)MBXT43Xd_lJKt=V-dQRo*UkPD=e?p0YAUn7dp8n}5I zTOz|MrEqdFv#ZGVN-(T$-Oi+XiV$IDUa+wHPFdr~5zfV7meDL5X*tBddxY?14i=gtbAWoBqv06Ux8O zfaw_Wgy<{U3Y9Kg9|lMan!k#$!SXgeHUuwtkdvth;#gldhS#}&89^jEaO#+L@u9q@wO$w5SqT5s80aCAFs&VA*TKm6QV-^F;NO%wj2gt z&NABrCisayy*EK)$^r|>Dk8{%nFg3DXuufjl6aNC3mknq9}oXiV0YO<>DMtn@`b|0 zPJ$1URH8?(ElcWi#Mo*^3=J&*WfTlHfrX%LxeBHFBEM?p5T!modQXI4A20Lf@PUhZ z+G)~{J6!0|lZ%zPXRu6dkb3)smmzn;hgbS2NPKF2)gezu3903V&G7uwshV|4W zN<}1j1?nH!u3}FULISB-KBi&5dTGSCyu>iQTq32G$cPdJfPq4}q>jlHGL%@M!%*WU zFCG$;Caz3jST#$?aPeluw#K>=@+LGd4MK{M9^@be1Y;x!klEXwG$pZqr~q8ukOsVb@y_c2@{TBN+6WY3pl91N|Z8F{(_i6%OBkiOUB!LyYUf zE)|1OD*;l7CA$602r`%yAk&P@TF)J*Ys25ARvgL+|k)RP|r&P&Z`$`Dm$w}vbZ|v}U{h z!Uy{N1>N!I^TS+@b`g?TK1?hr{Iai>I!X#fS2LDWxRP#%Jn8&Z?@a$Wp%v#Z!ex;> z`6GEJ4kvWFs)z`ZF^)%<~^Ci|Qzy7+-26N}0%`FV{OGda)Fi_^>bm;Dr3WY73WL+i84#MKliBBX6u`}`D#Cu0hR=j1~r`&7*2+opzK zl@VCyzB_Mek9OreAg4gP`AUH=Hdru`C?9x^iy_;e%)!5IJZ_hr@k?Y ze`0#)CvdAT2*s)26ofG`o?$IS7@jziXy4~*OglJDUlBdd5^+EMBEkF++Z>jGkphf{xq?!@ig<#5GY3h7mdPS=PbKaa-o1&pF*r%@et1@wV6W z0oXGeuIjY@Yy~y3)bQ&iK)_^3wNs$;_G*A9ivED&*=_v- zKfnOiDHw!O1&h0rOFU0rBG#)^>PT*Ms87q7&wxu2q_Cnm_bD#l37}%AUkuT0;e;p& zu$7ZKc;p0;?FfV!N*;75@#tYf*^`M*0r|95odV1Mj+SoqdcK7q%Z~2A4 zdwIED`dn{=pS4$gqVM48dUyi*_?M4(ntwSFh-mr55OM116?1vZH?Rna`TmS&XdCGR z3Op#{AxxW><8Ss*1`M%{z^ibnga5LwXOn1SB-Dkaf0tC*)HrZb*5TM539mjW`7(3c z*y!rjpHykHhEc(q_aV=~Ef3CuyHWW}-IfXdo=ShZ-A;qJ9=HvJU=Mor3Pnm3W2+m0 zG!X%&tH=*$6hW*QaVf}QfW;746O%ll3O!~;jK=6q(#UX^c59@V3U8+S9hHvqZY_PyZP_`@}6II_75ZK6o4**+SU`LZ*)s?hxb~2%* ziMs5-((lNVfyj_;AwVafx-h}n;(<)?w#tFht$s5}_~w9Viw81+UFRF{Lybod+&*og zUr#x0YY8Zdamy7%8fvQ#I10*E7-WL2D+QQM0>}h+>l8TM;F|-I?K#kdMAst}8_Ty@ zI9tu*ooe;ut)(3saY#n;t|Le`e_%3H+gczpblWT71Cg#}1U8Z10|ZKE?PICVqq~IP zY6g+^=(?;3H^9^L+XZ9u>^g#G695tcW-|aXf!Lx3PKW%j2X=&0Z|8qN(8gsXOjO2MLuy(&TI@ohOe2OC(o-Vvs| z=M-pf_dcTKIbCpJS9dr6wJB61X1B3cQBpM&i$(=qj%AfbH3k~BI%#0d)Ui+rDKe%z z2m)h!3T8J09X0ifyaEk0LY1OUsSa?}c$#r&3HoJIMtM3rmU20bT8u

8vI!!xS6Z5)UJdFoXjoh7ZJg1GKZZ$^LB6Qr2ehcMZ~Fa<*BZ3nG%R=>M%EGi2;H0$^&we%&$9ya-1W4mX8Sa zenP4kOI=yM2wsFU7)Lf>IrKbNbca18$(q+#!Z(tBx*&w%Lbl6w+gtWfv`rjj*!b-8 znht&%qhWYdHX6OKNg}ot637zBzT-XYKXL3S+emw0SEG{|WCd0yv0xv#Sh@w~nEK$O z`?tPWW03}nAabGDziBzgcmTeDd|BX~M2ObCia4Ds2o65lwu8cJqD4iR zndKmawY0$8QA6QMLw-hn=D+8^U(Tr$kW-c3k=*%d>xu>zjli2qoF0@+Jk>kmS6yq^ zGfe^(Xd;U4$wGiJ{TWBsXrdbMsd%ph9hX%&_r^9-*J$K>W-k$2q&E%ww+sVMUl5#8 z+&xhqW7wh&UmA{@c{fDKbS~yF5a%jwB_>rMNUJ_G&@eS5Nh)V}S=^CwYD6Pt&7P5C z&^$Owk`O>@tdxPqDt5Gz3M?e$#zovJPM#CxBRf8CK5ZoD8`H-d z|Jl*WveZtC5hv2}foIVaE5)gjLu7@O;H{%ASCAtof?-agB}#dg(QO}5wxJnVl+Ic$g-^nQbT(BB7HTA0mY$A+0Wy$X zWLnSuET9rM1^EY{|JRR%#J4qcFNC>}IIf|uk^Dz|5JqM{@eJ`JnZ`kR=Whsl1v&OQ zwS8-#A|T(>6Sq6UKNSTkM$KwA8a=N~tu-wKfY;Fvnt@1qLGo>l=TR^3<6@;S;ErK_ z4;eq}B2^WXCL`^pjL3OXz$0ahqDsGwim-k_4ZX-zdbn3RY@*?w3O6?+jF|6JYo5FK znLYF7f_fxHHx?@6xi_+;k!*l3u~&U3$8OAvn#ur8Iy-@`NZnAb4{;9negsf%{$5)- zok$Pa4yaBwwxe#uWuzTgjWUmbz~QuR6HTL3C_GdYx6vp?O0BQ13rQ6JOj*#knqn`= zsM%Tw@9j0zO0wrSk+!_Nm>Hcwk)O%H5z8+n3=S?ov8nFilFy-#(Z=}*`Jmvhi=hk1 zuWfoEN>!87bj&m{LBqiOc_!SDBQh~Ukb>n^M!+Wg6-J-1#q}AA!|h2P_mzv&-Q(G{ zDM!=H9|xO>Jm}3DB0-=YK$1*vlDvn=0+)OMaM-_%p0zGPeKf`#iQh&LV!Wf@{bNR%xXF-Rb4t7Ori5tgM)Wof zW9P;G4ynAf8s}_>!}bHppoX@nrib{B{oO%dC(NdM6uaxs%-&T47MZM1#W$AtF|lh6 zJx$f`$4BgK(4YnyDy(kPBMauKjOk=7lyp$0|+?9D?Is zGbfXBKGtW|KgN8@;8DZTnyY(CyuVjh%j6KS4*h~sD34wA2q=+`&rzf6Z8EFZX0{g8 z?voD4{@IIcPLKxbW@c^8hgeQuN{4EWqcFB+o#iDQ2ArST8C#xr8q;B=N)853IL>x^ zH4P^essmJ&LECV1RI}^js^eAWvG$OpKVDkq!oD0aBgK*tLcBgxpmsT?a*oMK3ZwOk zqRm_C-mu@YT&&mrbn8r;oNU{lr7&{O-Y>Ef;hrS!|4;o&#*SLa)9 z+bWZLer5FML{@A3=bUexN8=uoM%oUpDwWI3(mqQa*Vn_FkkMSPab9Fb9`1(|R|(Kb zO`fiZ4s&*s5d*Kn&KetQ_N&;-)$kN^%{iQt;z^RnajFwLGQY&z-vM1K1)nhQyj>KY z!-fxJvT)Rxw2eLXyDo{=U?_v7zM^{Q%J+}pOO{||;NET#RqF2*mS1LvMZSxe2Xf9Q zO1q=aUL#Xr5=nG#761?C(#g5+>Mw-Cd=uyE9*w3oZV(W^k3KYEv$;#Tey_#alUko{ z8Pi|T_fnocZVa7I<%+(qQryVR;Q-i-);ft3u*CN2(@R?3juCl= z-FO?ecV`HFu1Tmp6cWFGwZ3y%)6%ZnhYZ8te5yFQA#)CYt%4{vP}0J;K~$CRnc=Q0 z1}P)TLuj9;HBGd$UzvD>WRD*ro3MDvie7Wk8a!mZo!euNo-p!GzVsH**S=rxY_XIa z*=+UEzdVgRU~yu@eJ*;x7!;mpKY6*l9I7YUv=ct6T)q3meVs0cgw;6H^Dgaot7Fhv z)y#RgkiU3Q}qIby@^0 z{yLfwedZb+F)#WlBBEea3R}_I9LoV?%^#C-ek@-TmpzH;Qc(XO7+>KIo?WTNVf_x! zDx}E7v7QXxa2U+`de_JLiWMB)(4A)!h*RLiS7CO97!w=Y*Jd zrZ-Oqa3*jEt``>x@Y}{PAwIrOr=eX2GDhk5wNL_Ch5QSHFP4w7gREt+ z5Of$DY+dbrnz}AtcFG%-Kb?}CdGrTXqH{thSg0^_ zK4@fhS8+Z_Y4BL+7S%!xN>w1N;#Y=Y&o~?g> zL-22dbiu2!9slHJ2D`2_Im*hjM)PoIDld4~s2a?}h+ZN8oqQcZ@y8EEt}lyMVOjVG zG?_jQee%lzxlsqGKZC_gcoWZtE=(VDJmP1Bq!to?;(4Viw9r;M#k@97iKYBVxR9nf zTYk}xr?@w!ma)o6T5&q#^AcS3NKe$;>Fmf_WLk4s{nj^h?d1cX{21#~;QH{1&ny@q z^Z7zDjhT-}?RvIX$%nRc$F|a;;b%-PK&6N0;?4ekMra?2I?-vs7a!##GtRs$WrD#aV$Xi`7c; zD_^5-t8{~R?ME8YSDi=N(D9IS-541|gfI5znw&^!Ec{S=szJ3_H5M1fO%q_k$O0;< z)kx8f@(p!cXq5_Lk7{M@ia6xFzgIOychb#chP6hs;yuZmK5VuIys6xJxy80UtQ1WB zI2$ayOxs-Yx?Xtmd#jYgj?$@#HnqAQ9G)lZWjQ3Ty(9{+tAS00IcPfMcG=DSqPV(= zX?8z&>dn>Pvu{PTkZYn4)q-F`<`5 z$@+X;{qkJovF-cyd6IxW%YpKS-KXu=ipj``@WvLa{mX-p-ndHvt-_ zkR2(5Wl0R#YE?_nDfNAXR>v?q;}qxaVVs2)*%%cvLle6ix=N>3Z6Mo5sm|%3%f8XG zXl6}E6{xVX0t++hU^>7bpY*n^yG%l{Bg0aaI;2x`B+<(35LybFsW-b8eY%0&gC&7m zU^9REDZNh{bnWV*zNXCD#~-_nVEgWq&CC8 zXQ{LFQ8(`Z|YVR`%hu-jD61s*FLkKT;PSB@8(^*leaT?38!hoWG!3 zR*4yZ+OM}+7k@oSJ6=rz_dIDrW1Vk3B}$x-+%-o#B|m=g3Ojgvi^L;^KU%h&W~x1q zf`!a-5ov#px;o$B>n(FFK7X@AUOZpnsWjcMyXNfh`pj_F`uc_&RI&DKy2z$|!C<1# zMDw1Fa545B0aMa*)N1B%?m`N!{rt}2s;eSX`+h@egnPxO>$Twc@#;3CccN{0j zWFQ#xa)Z;!{;%(0d+of(b?V1S%mr~5_ix6@0{ykhG*te0j;__$N{4dMjOd;>dRD8! zjd(nV+uw?KwP{q{o5Rgr8q=cIAEM=G?rDc9oWfrVTm|nbgz^M=slrda*Pa!!Og4+( zQjdu?ueR0S!;UJ0<6P*RQePqime(B=ddhK|G&YtSAFCe@##FBveP4=iuUV@`G@BZ1 z^;&xSf5e~7Xt+I=Z-1)K z;f#-chR^kr-!(iJ_M$!U_U9n81gBxs6&>ZEK2w50rlBQxq;XbHMOXQo)mP^YUimOD z@-XQkXPvm11{bi6DHae<^7jeH*~5_Fjn7zbix;Z)$ys*@q{q0Dhqy5gCzQX;78Cm{x81*6wKO;O4ooMeP0&fQr620P5xC8H zWgRwdx~^*shH}&}yFO3B?s?YTIF-;vlWRL;%FmrD4cRP$5}za>zmMU=_OoObd5o8} zIL-GwoyL?a^*G;`-+x{)I!)PpJ<4k0fU9+a#^37Duit-S)oO$d)^fcfHk}9vH)xC) zwz8`cxxC4_FA|h?bd~#=VC&(mG8x$cdT(=a*?n-MTzB5D!7>hy;bUI1vFp_Ljh*Io z^}Ca@ct`l-nMWG@xE2|n+ijZKc|=ohmXh#A29M8GWn5wlOYhmb5Bu>x-juQ2K?kl( z1Wjh^^t99M%FjWUR- z?{BWL;AS*=D2OZ6aA{&gS-j9bAWVa-?9sa8^c#e+PD(Tr)>&M$@UpTI?S3nI#@VHl zO1@;%6s^!^y_!!mcKgk-Nu7RB@+`ElaTy7Q9t)V zm9MJsnvE2?aA8f-N?t7Ub+Rx-H(%P`@?FGND!df)IXVb*@R|{q?sz|Dk;p~1=mw;Gg&)-E z%I&O90w=22E(@4RrBl2TD+;pilh^)PsqVrRK`LOE*g zT;j;KnJ1!@vY)Tt6U@zh;qb|2ebwY_B&k+nr@i7uW$#mhbUL@Sx~SqBc-apF8TK0o z&*h5y`oocNpMv#cy>6SPpVs$iWfPVTBgvbO`=G-)TjwcxS~@F{&vuR0JrOD*D*%C& zi!}4r>)?icWr@uE4w*eVa?8y+ua&PkOa1bNIQ6F6XV6vR#_e!Nwhb*7d)DtoBOgtj^|XQ;&C)mn=>u;!$^83>hZ^-w~yLwuG+MI6ZW9;p2{18B95y?b^m?5M>XRT z*>e98t<+tJ;A*B8&y-)of%;ar@VSp>f;;zwj{=k7Hu%7g+?aJ}x@#4+JJ<%Asa8e6 z${C$@ZD|of@A7+B2`W`$G~W8L*C9xflh2*qe5bZ@1FMxs=Sy1j3?nWc@2O&p(dNbO zkBZ8YBNx{8s?58PA|LXTmXcKW9+)(@GxZjJuA=AR0#HK2`{P{=R3nx{`h^L1$2o=zoCRab;D$^w zM8!M5cjsO6heh9pNnx0}QMpS7u$qFDri3GZ^r>9uzH4j2luRq}9}CuoZUSbUdUW2` zOA~uYF2{jlW$D-WiU_ct!eWw*s&1dM^3hXvu7JKC)l0@V?=_uI_#p>2dt@oe%D)tLD+ON@o@NA^)V7a36$cx8!rB zuf;!K&MD4pBpqiTsF$hXM>eRnk zrg#VqUQLy@CrqXWYILZ(lo}%^4fad8flg_Bbd`(e$z zCP{MT#EZ-(_R{*&-ZD@nvyU%MUhl4qPX+Io7Pl8~%7Pj*S=#%19!?3G9+i1Toozk= zS}okDw~m9S1%fNh-mQiflGejBGzG!0qu`FR*m$rL1=G1Y`mit%Xfr_{MHdW|Mh zHW_8cb+S)4r+GeA9d(ywf(vSnHK=3F7~YNQptri33gI&;Gr}lo` zr#l((Uqru*V~vNg8Fq#DzkCO>B4T*|OXAPp8L9tZ#r&NJ_z$w`e^3OgN=g1fm&{1d z!1QO%RW4pvTRj5{BS!*#BNH=gZo-S!PC^1RLvBJ9W+{3pTLB|eGf{VYBL#P9MFV#W z19n3~ULG!24p%E%s}D|W0#_?bYX=TjZbH2e(rY7*kMeI~IzocqRU9q22^nb_XjthP z2zg+*>%vHDV{byoz|PK2M-QL_0BAlmXdK+E z9ravktR0B{@bCwt@CTVK!EYY*^lh9Rxd{mg{_fZIPkyZ(Xnzlk*1*P!?xTW^ftHc( zH%D~8S^mk>-=q7JGvigk-`@=oM@9qr^I1K*A#LM%0 zs}+pw{#LLwQHX{~fSs9!k5zz=hKXNL5WvsOFU-Wq`g=3~tL<+M{g8-RJ2>in@WcOiy#2}c zFB(tq|6qanf3OdN|Lo-7Lis;%{Rggp3xR)2`9HnuKXCn92>e^h|LI--9$bIB?0;m; zA6el)Qf{9AnX(&Mf8^;d9|`$i4-6Ox=n4K97zmm<*jnnjePlO^W{#Fd1djGjM!$^% zK3x2+0s#C;`S($Ts=Sdgor0OW5di=|r=mf?K)^%*&?2Cdx3O{j*q|qnrxUX_wjp5u zTT$@iOKc_qRyKAaMn)lFHg;wvf#33^Kc`7N|9g!O|H5XLjz;!$!j>PIrJ#|4jiJ#W zV;45&KaX5o(+zZi{NCR&ee(%GK||Cba?X7z226Crj<}QcGZ6|^R|w{;p>Bc6KP^mv z%;yC>wzYQyK?f9Nhp~o}mTC?**wa-=|`EG+*b1N?^dZ_A(bnArYRkCpiY<$uy+ zXZV2ozseXGnA!eC24MIXTLu6N(?9Fc(|_pwYdZjcA!Gat8Pi|LnEyh?@)t7Jf0cb8 z@=viaurd4#Rsed&f5w23ftmiF+snwr%>K`~Ffy?)G5${s9PRbYERF2pVYs;d-q9KV zZ};xMbznVv$G`O{MkaOuD?AJdiI9viJj@^cj2<55|3heWvi3&K1iwc__uILz;Ar)+ T^#RyF$IQ&k^l>8m9`pYI(AHH# literal 0 HcmV?d00001 diff --git a/tests/fixtures/test-item-metadata-error.json b/tests/fixtures/test-item-metadata-error.json new file mode 100644 index 0000000..2fe5f4a --- /dev/null +++ b/tests/fixtures/test-item-metadata-error.json @@ -0,0 +1,7 @@ +{ + "metadata": [ + { + "key": "dc.title" + } + ] +} diff --git a/tests/fixtures/test-item-metadata.json b/tests/fixtures/test-item-metadata.json new file mode 100644 index 0000000..c6c0c9f --- /dev/null +++ b/tests/fixtures/test-item-metadata.json @@ -0,0 +1,12 @@ +{ + "metadata": [ + { + "key": "dc.title", + "value": "Test Thesis" + }, + { + "key": "dc.contributor.author", + "value": "Jane Q. Smith" + } + ] +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 46856c2..eb532ac 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,74 +1,45 @@ -import os - -import boto3 from click.testing import CliRunner -from moto import mock_sqs from submitter.cli import main -os.environ["AWS_DEFAULT_REGION"] = "us-east-1" - -@mock_sqs -def test_cli_sample_data_loader(): - mocked_sqs = boto3.resource("sqs") - input_queue = "test-input" - queue = mocked_sqs.create_queue(QueueName=input_queue) +def test_cli_sample_data_loader(mocked_sqs): + queue = mocked_sqs.get_queue_by_name(QueueName="empty_input_queue") - # confirm queue starts empty sqs_messages = queue.receive_messages() assert len(sqs_messages) == 0 runner = CliRunner() - result = runner.invoke(main, ["sample-data-loader", "--queue", input_queue]) + result = runner.invoke(main, ["sample-data-loader", "--queue", "empty_input_queue"]) assert result.exit_code == 0 - # confirm messages now in queue sqs_messages = queue.receive_messages() assert len(sqs_messages) > 0 -@mock_sqs -def test_cli_start(): - mocked_sqs = boto3.resource("sqs") - input_queue = "test-input" - output_queue = "test-output" - queue = mocked_sqs.create_queue(QueueName=input_queue) - out = mocked_sqs.create_queue(QueueName=output_queue) +def test_cli_start(caplog, mocked_dspace, mocked_sqs): + # Required because pytest and CliRunner handle log capturing in incompatible ways + caplog.set_level(100000) + input_queue = mocked_sqs.get_queue_by_name(QueueName="input_queue_with_messages") + result_queue = mocked_sqs.get_queue_by_name(QueueName="empty_result_queue") - # confirm queue starts empty - sqs_messages = queue.receive_messages() - assert len(sqs_messages) == 0 + results = result_queue.receive_messages() + assert len(results) == 0 runner = CliRunner() - result = runner.invoke(main, ["sample-data-loader", "--queue", input_queue]) - assert result.exit_code == 0 - - # confirm messages now in queue - sqs_messages = queue.receive_messages() - assert len(sqs_messages) > 0 - - # confirm no messages in out queue before start - out_messages = out.receive_messages() - assert len(out_messages) == 0 - result = runner.invoke( main, [ "start", "--wait", 1, - "--input-queue", - input_queue, - "--output-queue", - output_queue, + "--queue", + "input_queue_with_messages", ], ) assert result.exit_code == 0 - # confirm queue is empty again - sqs_messages = queue.receive_messages() + sqs_messages = input_queue.receive_messages() assert len(sqs_messages) == 0 - - out_messages = out.receive_messages() + out_messages = result_queue.receive_messages() assert len(out_messages) > 0 diff --git a/tests/test_submission.py b/tests/test_submission.py new file mode 100644 index 0000000..d672061 --- /dev/null +++ b/tests/test_submission.py @@ -0,0 +1,128 @@ +import traceback + +from dspace import Bitstream, Item + +from submitter.submission import Submission + + +def test_init_submission_from_message(input_message_good): + submission = Submission.from_message(input_message_good) + assert submission.destination == "DSpace@MIT" + assert submission.collection_handle == "0000/collection01" + assert submission.metadata_location == "tests/fixtures/test-item-metadata.json" + assert submission.files == [ + { + "BitstreamName": "test-file-01.pdf", + "FileLocation": "tests/fixtures/test-file-01.pdf", + "BitstreamDescription": "A test bitstream", + } + ] + assert submission.result_attributes == { + "PackageID": {"DataType": "String", "StringValue": "etdtest01"}, + "SubmissionSource": {"DataType": "String", "StringValue": "etd"}, + } + assert submission.result_message is None + assert submission.result_queue == "empty_result_queue" + + +def test_get_metadata_entries_from_file(): + submission = Submission( + destination=None, + collection_handle=None, + metadata_location="tests/fixtures/test-item-metadata.json", + files=None, + attributes=None, + result_queue=None, + ) + metadata = submission.get_metadata_entries_from_file() + assert next(metadata) == {"key": "dc.title", "value": "Test Thesis"} + + +def test_result_error_message(input_message_good): + submission = Submission.from_message(input_message_good) + error = KeyError() + submission.result_error_message(error, "A test error") + assert submission.result_message["ResultType"] == "error" + assert submission.result_message["ErrorInfo"] == "A test error" + assert submission.result_message["ExceptionMessage"] == str(error) + assert submission.result_message["ExceptionTraceback"] == traceback.format_exc() + + +def test_result_success_message(input_message_good): + item = Item() + item.handle = "0000/12345" + item.lastModified = "yesterday" + bitstream = Bitstream() + bitstream.name = "A test bitstream" + bitstream.uuid = "1234-5678-9000" + bitstream.checkSum = { + "value": "a4e0f4930dfaff904fa3c6c85b0b8ecc", + "checkSumAlgorithm": "MD5", + } + item.bitstreams = [bitstream] + submission = Submission.from_message(input_message_good) + submission.result_success_message(item) + assert submission.result_message["ResultType"] == "success" + assert submission.result_message["ItemHandle"] == item.handle + assert submission.result_message["lastModified"] == item.lastModified + assert submission.result_message["Bitstreams"] == [ + { + "BitstreamName": bitstream.name, + "BitstreamUUID": bitstream.uuid, + "BitstreamChecksum": bitstream.checkSum, + } + ] + + +def test_submit_success(mocked_dspace, test_client, input_message_good): + submission = Submission.from_message(input_message_good) + submission.submit(test_client) + assert submission.result_message["ResultType"] == "success" + + +def test_submit_create_item_error( + mocked_dspace, test_client, input_message_item_create_error +): + submission = Submission.from_message(input_message_item_create_error) + submission.submit(test_client) + assert submission.result_message["ResultType"] == "error" + assert ( + submission.result_message["ErrorInfo"] + == "Error occurred while creating item metadata from file" + ) + + +def test_submit_add_bitstreams_error( + mocked_dspace, test_client, input_message_bitstream_create_error +): + submission = Submission.from_message(input_message_bitstream_create_error) + submission.submit(test_client) + assert submission.result_message["ResultType"] == "error" + assert ( + submission.result_message["ErrorInfo"] + == "Error occurred while adding bitstreams to item from files" + ) + + +def test_submit_item_post_error( + mocked_dspace, test_client, input_message_item_post_error +): + submission = Submission.from_message(input_message_item_post_error) + submission.submit(test_client) + assert submission.result_message["ResultType"] == "error" + assert ( + submission.result_message["ErrorInfo"] + == "Error occurred while posting item to DSpace" + ) + + +def test_submit_bitstream_post_error( + mocked_dspace, test_client, input_message_bitstream_post_error +): + submission = Submission.from_message(input_message_bitstream_post_error) + submission.submit(test_client) + assert submission.result_message["ResultType"] == "error" + assert submission.result_message["ErrorInfo"] == ( + "Error occurred while posting bitstreams to item in DSpace. Item with handle " + "0000/item01 and any successfully posted bitstreams have been deleted" + ) From c7822e02dfc9cc8e28a999f3d585db0afb0acf0d Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Thu, 16 Sep 2021 11:54:02 -0400 Subject: [PATCH 2/3] Update error result message to match ADR --- Pipfile | 1 + Pipfile.lock | 22 +++++++++++++++------- submitter/submission.py | 3 +++ tests/test_submission.py | 3 +++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index d79b17a..39dc340 100644 --- a/Pipfile +++ b/Pipfile @@ -20,6 +20,7 @@ coveralls = "*" requests-mock = "*" pytest-cov = "*" pytest-env = "*" +freezegun = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 9372353..058b1ff 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6ae732f1c1cc7845eab2c51f30d7a092ea2c119be63c44c12f4f7050f542f14f" + "sha256": "78e8e91859fd11a4d5055affeff153f7f9f97f23524067975296f745a531b8bc" }, "pipfile-spec": 6, "requires": { @@ -91,7 +91,7 @@ "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.6" } }, @@ -276,7 +276,7 @@ "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==5.5" }, "coveralls": { @@ -325,6 +325,14 @@ "index": "pypi", "version": "==3.9.2" }, + "freezegun": { + "hashes": [ + "sha256:177f9dd59861d871e27a484c3332f35a6e3f5d14626f2bf91be37891f18927f3", + "sha256:2ae695f7eb96c62529f03a038461afe3c692db3465e215355e1bb4b0ab408712" + ], + "index": "pypi", + "version": "==1.1.0" + }, "gitdb": { "hashes": [ "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0", @@ -463,11 +471,11 @@ "sqs" ], "hashes": [ - "sha256:21c838b63f44e24b9b5015a2cdcc5be7c1e1004e58a69fb7cac71383bce34535", - "sha256:e86b0d92bc5f80802a8ae0f338a4fdac15dab82c54eb12db93b356b69407effc" + "sha256:461955aaccd257151591b1e36c9b2e7ddf7f42e17056f15ccbd64d0eb618742d", + "sha256:f5d131d0be71890809c94556930f865d25814e2d2e29d74fab749f963a11b518" ], "index": "pypi", - "version": "==2.2.6" + "version": "==2.2.7" }, "mypy-extensions": { "hashes": [ @@ -744,7 +752,7 @@ "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.6" }, "werkzeug": { diff --git a/submitter/submission.py b/submitter/submission.py index 2a4d044..6007dcf 100644 --- a/submitter/submission.py +++ b/submitter/submission.py @@ -1,6 +1,7 @@ import json import logging import traceback +from datetime import datetime import dspace import smart_open @@ -48,8 +49,10 @@ def get_metadata_entries_from_file(self): yield entry def result_error_message(self, error, info): + time = datetime.now() self.result_message = { "ResultType": "error", + "ErrorTimestamp": time.strftime("%Y-%m-%d %H:%M:%S"), "ErrorInfo": info, "ExceptionMessage": str(error), "ExceptionTraceback": traceback.format_exc(), diff --git a/tests/test_submission.py b/tests/test_submission.py index d672061..e5735e8 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -1,6 +1,7 @@ import traceback from dspace import Bitstream, Item +from freezegun import freeze_time from submitter.submission import Submission @@ -38,11 +39,13 @@ def test_get_metadata_entries_from_file(): assert next(metadata) == {"key": "dc.title", "value": "Test Thesis"} +@freeze_time("2021-09-01 05:06:07") def test_result_error_message(input_message_good): submission = Submission.from_message(input_message_good) error = KeyError() submission.result_error_message(error, "A test error") assert submission.result_message["ResultType"] == "error" + assert submission.result_message["ErrorTimestamp"] == "2021-09-01 05:06:07" assert submission.result_message["ErrorInfo"] == "A test error" assert submission.result_message["ExceptionMessage"] == str(error) assert submission.result_message["ExceptionTraceback"] == traceback.format_exc() From 6255aa4de08bb26052e84e2dab30aadb8dbd5462 Mon Sep 17 00:00:00 2001 From: Helen Bailey Date: Thu, 23 Sep 2021 10:06:11 -0400 Subject: [PATCH 3/3] Minor changes based on PR feedback --- setup.py | 2 +- submitter/sqs.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index b8146a5..1c49758 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ "smart-open", ( "dspace-python-client @ git+https://github.com/mitlibraries/" - "dspace-python-client@0.1.0#egg=dspace" + "dspace-python-client@0.1.0#egg=dspace-python-client" ), ], entry_points={"console_scripts": ["submitter=submitter.cli:main"]}, diff --git a/submitter/sqs.py b/submitter/sqs.py index b2ea842..c0aa350 100644 --- a/submitter/sqs.py +++ b/submitter/sqs.py @@ -12,7 +12,7 @@ def message_loop(queue, wait): logger.info("Message loop started") - msgs = retrieve(queue, wait) + msgs = retrieve_messages_from_queue(queue, wait) if len(msgs) > 0: process(msgs) @@ -35,7 +35,7 @@ def process(msgs): # TODO: handle and test submit message errors raise e submission.submit(client) - write( + write_message_to_queue( submission.result_attributes, json.dumps(submission.result_message), submission.result_queue, @@ -46,7 +46,7 @@ def process(msgs): logger.info("Deleted message %s from input queue", message_id) -def retrieve(input_queue, wait): +def retrieve_messages_from_queue(input_queue, wait): sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName=input_queue) @@ -62,7 +62,7 @@ def retrieve(input_queue, wait): return msgs -def write(attributes, body, output_queue): +def write_message_to_queue(attributes, body, output_queue): sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName=output_queue) queue.send_message(