Arkivum: Fixes for real endpoints

refs #7188

Fix POST to release files to send a compressionAlgorithm, correct relative
path, and JSON-encoded payload. Verify SSL certificates in production, but
not in debug. Add logging, test for post_move_from_ss.

storage_service/locations/fixtures/vcr_cassettes/arkivum_post_move_from_ss.yaml

@@ -0,0 +1,37 @@
+interactions:
+- request:
+    body: '{"compressionAlgorithm": ["c0f8/498f/b92e/4a8b/8941/1b34/ba06/2ed8/ep9-c0f8498f-b92e-4a8b-8941-1b34ba062ed8",
+      ".7z"], "checksum": "b05403212c66bdc8ccc597fedf6cd5fe", "checksumAlgorithm":
+      "md5", "size": "10"}'
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Content-Length: ['207']
+      Content-Type: [application/json]
+      User-Agent: [python-requests/2.3.0 CPython/2.7.3 Linux/3.5.0-52-generic]
+    method: POST
+    uri: https://198.50.244.148:8443/api/2/files/release/aips/c0f8/498f/b92e/4a8b/8941/1b34/ba06/2ed8/ep9-c0f8498f-b92e-4a8b-8941-1b34ba062ed8.7z
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAJ1UTU/cMBC98ytWOePGcRzH5rbiUHFAQkAPbdWDY4/ZiGycOk4poP732vnYhYVS
+        2lOkN28mb8Yz7/FotUqk8oNsTjegbvthm6xOVu3QNMcvQ8t33dxYV/vNq1y77Rz0fW3bt2hX9QNE
+        GI+YCkkNeNARymhGcE4w5ZjxMWzqBs7GWKJZKUSZYYQpM4gSnqOqyBgiXDFjVJZxWSb7pNZYt5U+
+        iInZjwGPCgZvTxuQLmLeDXA84WoztLc9+D11B0bo6+rb8YJCq9x9Nws2sulhH+qVs3ehzKR46XuM
+        1S8hBz/qftaHR/DXIseBnP+QEJxRhAUi4prkJ5idEPIlecqzYzNJayur75eIll5eWNsss8tBMZEV
+        SJYkzM7kFAltSgRShRHmwoAku9TagQpV7w8anNu7lh08H0oCPz20GvTae1dXg5/ij7t24nscDDxC
+        51aPcJ5nnM94/Y9vHVPaG+j9lQ8TG3MvLs/O15efl3Aje79WKuzlOwYayacbGSq+jxtaqE39HrJV
+        sjmYwVYXz3ciaeV26gE6gRQ2nApuUCUIICp5hbigGcqqnFYSMwKafygflj/YuxbcR2eHbnpzIZ4G
+        PvXgDvBOOmjnVU2Y4SUmrEAMcIlC/bAqRhnEMlEoQnWJqU52iX4zJqWy7vo06kyj0DQKTaPQNApN
+        o9A0Kk2j1PQ/enLQNbUaj3j/vg52SvrZSkhO87zkR/MFLVsksTBCZRxpQypES2oQV6VBBYYqD/ta
+        STKVSoJZ3dTtcydMKlzQ4EYZUYxVWnGlVCFKA9owpQsDr6e+7ZQ79t+8ciEubplNdhkOvp8cY8d0
+        8H0IB/AnB+3D4IbxIJMrtQE9NIF59Os3LohH4/8FAAA=
+    headers:
+      accept-ranges: [bytes]
+      content-encoding: [gzip]
+      content-type: [application/json; charset=ISO-8859-1]
+      date: ['Mon, 29 Sep 2014 23:07:28 GMT']
+      server: [A-Stor/2.2.9.8409 (Noelios-Restlet-Engine/1.1.10)]
+      transfer-encoding: [chunked]
+    status: {code: 202, message: Accepted}
+version: 1

storage_service/locations/models/arkivum.py

@@ -6,6 +6,7 @@
 import subprocess
 
 # Core Django, alphabetical
+from django.conf import settings
 from django.db import models
 
 # Third party dependencies, alphabetical
@@ -20,6 +21,11 @@
 
 LOGGER = logging.getLogger(__name__)
 
+if settings.DEBUG:
+    VERIFY = False
+else:
+    VERIFY = True
+
 
 class Arkivum(models.Model):
     space = models.OneToOneField('Space', to_field='uuid')
@@ -28,9 +34,9 @@ class Arkivum(models.Model):
         help_text='Hostname of the Arkivum web instance. Eg. arkivum.example.com:8443')
     # Optionally be able to rsync
     remote_user = models.CharField(max_length=64, null=True, blank=True,
-        help_text="Username on the remote machine accessible via passwordless ssh. (Optional)")
+        help_text="Optional: Username on the remote machine accessible via passwordless ssh.")
     remote_name = models.CharField(max_length=256, null=True, blank=True,
-        help_text="Name or IP of the remote machine. (Optional)")
+        help_text="Optional: Name or IP of the remote machine.")
 
     class Meta:
         verbose_name = "Arkivum"
@@ -54,7 +60,7 @@ def delete_path(self, delete_path):
         # TODO folders
         url = 'https://' + self.host + '/files/' + delete_path
         LOGGER.info('URL: %s', url)
-        response = requests.delete(url, verify=False)
+        response = requests.delete(url, verify=VERIFY)
         LOGGER.info('Response: %s, Response text: %s', response.status_code, response.text)
         if response.status_code != 204:
             raise StorageException('Unable to delete %s', delete_path)
@@ -92,16 +98,20 @@ def post_move_from_storage_service(self, staging_path, destination_path, package
             'size': str(os.path.getsize(staging_path)),
             'checksum': checksum.hexdigest(),
             'checksumAlgorithm': 'md5',
-            'compressionAlgorithm': '',
+            'compressionAlgorithm': os.path.splitext(package.current_path)[1],
         }
+        payload = json.dumps(payload)
 
         # POST to Arkivum host/api/2/files/release/relative_path
-        relative_path = destination_path.replace(self.space.path, '', 1)
-        url = 'https://' + self.host + '/api/2/files/release' + relative_path
-        # FIXME Arkivum URL does not actually exist yet
-        response = requests.post(url, data=payload, verify=False)
-        if response.status_code != 200:
-            logging.warning('Arkivum responded with %s: %s', response.status_code, response.text)
+        relative_path = os.path.relpath(destination_path, self.space.path)
+        url = 'https://' + self.host + '/api/2/files/release/' + relative_path
+        LOGGER.info('URL: %s, Payload: %s', url, payload)
+
+        response = requests.post(url, headers={'Content-Type': 'application/json'}, data=payload, verify=VERIFY)
+
+        LOGGER.info('Response: %s, Response text: %s', response.status_code, response.text)
+        if response.status_code not in (requests.codes.ok, requests.codes.accepted):
+            LOGGER.warning('Arkivum responded with %s: %s', response.status_code, response.text)
             raise StorageException('Unable to notify Arkivum of %s', package)
         # Response has request ID for polling status
         try:
@@ -117,11 +127,11 @@ def post_move_from_storage_service(self, staging_path, destination_path, package
         # TODO Uncompressed: Post info about bag (really only support AIPs)
 
     def update_package_status(self, package):
-        # Get local copy
-        local_path = package.fetch_local_path()
         LOGGER.info('Package status: %s', package.status)
         # If no request ID, try POSTing to Arkivum again
         if 'request_id' not in package.misc_attributes:
+            # Get local copy
+            local_path = package.fetch_local_path()
             self.post_move_from_storage_service(local_path, package.full_path, package)
         # If still no request ID, cannot check status
         if 'request_id' not in package.misc_attributes:
@@ -131,15 +141,15 @@ def update_package_status(self, package):
         url = 'https://' + self.host + '/api/2/files/release/' + package.misc_attributes['request_id']
         LOGGER.info('URL: %s', url)
 
-        response = requests.get(url, verify=False)
+        response = requests.get(url, verify=VERIFY)
 
         LOGGER.info('Response: %s, Response text: %s', response.status_code, response.text)
         if response.status_code != 200:
             return (None, 'Response from Arkivum server was {}'.format(response))
 
         # Look for ['fileInformation']['replicationState'] == 'green'
-        resp_json = response.json()
-        replication = resp_json['fileInformation'].get('replicationState')
+        response_json = response.json()
+        replication = response_json['fileInformation'].get('replicationState')
         if replication == 'green':
             # Set status to UPLOADED
             package.status = Package.UPLOADED

storage_service/locations/tests/test_arkivum.py

@@ -80,12 +80,12 @@ def test_delete(self):
 
     #     # TODO test folder in new test
 
-    # @vcr.use_cassette('locations/fixtures/vcr_cassettes/arkivum_post_move_from_ss.yaml')
+    @vcr.use_cassette('locations/fixtures/vcr_cassettes/arkivum_post_move_from_ss.yaml')
     def test_post_move_from_ss(self):
         # POST to Arkivum about file
         open('unittest.txt', 'w').write('test file\n')
         self.arkivum_object.post_move_from_storage_service('unittest.txt', self.package.full_path, self.package)
-        assert self.package.misc_attributes['request_id']
+        assert self.package.misc_attributes['request_id'] == 'a09f9c18-df2b-474f-8c7f-50eb3dedba2d'
         # Cleanup
         os.remove('unittest.txt')