upload: expose the rest of the Upload API through UploadArea

hca/upload/upload_area.py

@@ -36,6 +36,24 @@ def deployment_stage(self):
     def uuid(self):
         return self.uri.area_uuid
 
+    def delete(self):
+        """
+        Request deletion of this Upload Area by the Upload Service.
+        Although this call will return quickly, deletion of large Upload Areas takes some time,
+        so is performed asynchronously.
+        :return: True
+        :raises UploadApiException: if the Area was not deleted
+        """
+        return self.upload_service.api_client.delete_area(area_uuid=self.uuid)
+
+    def exists(self):
+        """
+        Check if the Upload Area represented by this object actually exists.
+        :return: True if the Area exists, False otherwise.
+        :rtype: bool
+        """
+        return self.upload_service.api_client.area_exists(area_uuid=self.uuid)
+
     def get_credentials(self):
         """
         Return a set of credentials that may be used to access the Upload Area folder in the S3 bucket
@@ -70,6 +88,24 @@ def list(self, detail=False):
             for file_info in files_info:
                 yield file_info
 
+    def store_file(self, filename, file_content, content_type):
+        """
+        Store a small file in an Upload Area
+
+        :param str area_uuid: A RFC4122-compliant ID for the upload area
+        :param str filename: The name the file will have in the Upload Area
+        :param str file_content: The contents of the file
+        :param str content_type: The MIME-type for the file
+        :return: information about the stored file (similar to that returned by files_info)
+        :rtype: dict
+        :raises UploadApiException: if file could not be stored
+        """
+
+        return self.upload_service.api_client.store_file(area_uuid=self.uuid,
+                                                         filename=filename,
+                                                         file_content=file_content,
+                                                         content_type=content_type)
+
     def upload_files(self, file_paths, file_size_sum=0, dcp_type="data", target_filename=None,
                      use_transfer_acceleration=True, report_progress=False):
         """
@@ -100,6 +136,68 @@ def upload_files(self, file_paths, file_size_sum=0, dcp_type="data", target_file
                 error += "\nPlease retry or contact an hca administrator at data-help@humancellatlas.org for help.\n"
                 raise UploadException(error)
 
+    def validate_files(self, file_list, validator_image, original_validation_id="", environment={}):
+        """
+        Invoke supplied validator Docker image and give it access to the file/s.
+        The validator must be based off the base validator Docker image.
+
+        :param list file_list: A list of files within the Upload Area to be validated
+        :param str validator_image: the location of a docker image to use for validation
+        :param str original_validation_id: [optional]
+        :param dict environment: [optional] list of environment variable to set for the validator
+        :return: ID of scheduled validation
+        :rtype: dict
+        :raises UploadApiException: if information could not be obtained
+        """
+        return self.upload_service.api_client.validate_files(area_uuid=self.uuid,
+                                                             file_list=file_list,
+                                                             validator_image=validator_image,
+                                                             original_validation_id=original_validation_id,
+                                                             environment=environment)
+
+    def checksum_status(self, filename):
+        """
+        Retrieve checksum status and values for a file
+
+        :param str filename: The name of the file within the Upload Area
+        :return: a dict with checksum information
+        :rtype: dict
+        :raises UploadApiException: if information could not be obtained
+        """
+        return self.upload_service.api_client.checksum_status(area_uuid=self.uuid, filename=filename)
+
+    def checksum_statuses(self):
+        """
+        Retrieve counts of files in Upload Area in each checksumming state: scheduled for checksumming, checksumming,
+        checksummed, and unscheduled.
+
+        :return: a dict with key for each state and value being the count of files in that state
+        :rtype: dict
+        :raises UploadApiException: if information could not be obtained
+        """
+        return self.upload_service.api_client.checksum_statuses(area_uuid=self.uuid)
+
+    def validation_status(self, filename):
+        """
+        Get status and results of latest validation job for a file.
+
+        :param str filename: The name of the file within the Upload Area
+        :return: a dict with validation information
+        :rtype: dict
+        :raises UploadApiException: if information could not be obtained
+        """
+        return self.upload_service.api_client.validation_status(area_uuid=self.uuid, filename=filename)
+
+    def validation_statuses(self):
+        """
+        Get count of validation statuses for all files in upload_area
+
+        :return: a dict with key for each state and value being the count of files in that state
+        :rtype: dict
+        :raises UploadApiException: if information could not be obtained
+        """
+        return self.upload_service.api_client.validation_statuses(area_uuid=self.uuid)
+
     def _setup_s3_agent_for_file_upload(self, file_count=0, file_size_sum=0, use_transfer_acceleration=True):
         creds_provider = CredentialsManager(upload_area=self)
         self.s3agent = S3Agent(credentials_provider=creds_provider, transfer_acceleration=use_transfer_acceleration)

test/integration/upload/test_upload_area.py

@@ -4,6 +4,8 @@
 import os
 import sys
 import unittest
+import uuid
+from mock import Mock, patch
 
 import responses
 
@@ -13,10 +15,17 @@
 from test import TEST_DIR
 from test.integration.upload import UploadTestCase
 from hca.upload.lib.s3_agent import WRITE_PERCENT_THRESHOLD
+from hca.upload import UploadAreaURI, UploadService
 
 
 class TestUploadArea(UploadTestCase):
 
+    def _create_upload_area(self):
+        upload = UploadService(deployment_stage=self.deployment_stage)
+        area_uuid = str(uuid.uuid4())
+        area_uri = UploadAreaURI(self._make_area_uri(area_uuid=area_uuid))
+        return upload.upload_area(area_uri=area_uri)
+
     def test_get_credentials(self):
 
         area = self.mock_current_upload_area()
@@ -31,6 +40,132 @@ def test_get_credentials(self):
         self.assertIn('aws_session_token', creds)
         self.assertIn('expiry_time', creds)
 
+    def test_delete(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            mock_delete_area = Mock(return_value=True)
+            mock_api_client = Mock(delete_area=mock_delete_area)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+            area.delete()
+
+            mock_delete_area.assert_called_once_with(area_uuid=area.uuid)
+
+    def test_exists(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            mock_area_exists = Mock(return_value=False)
+            mock_api_client = Mock(area_exists=mock_area_exists)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+            result = area.exists()
+
+            mock_area_exists.assert_called_once_with(area_uuid=area.uuid)
+            self.assertFalse(result)
+
+    def test_validate_files(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            validation_id = "validation234"
+            mock_validate_files_method = Mock(return_value=validation_id)
+            mock_api_client = Mock(validate_files=mock_validate_files_method)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+            docker_img = "bogo_image"
+            orig_val_id = "validation123"
+            files = ['file1', 'file2']
+            env = {'KEY': 'someval'}
+
+            result = area.validate_files(file_list=files, validator_image=docker_img,
+                                         original_validation_id=orig_val_id, environment=env)
+
+            mock_validate_files_method.assert_called_once_with(area_uuid=area.uuid,
+                                                               file_list=files,
+                                                               validator_image=docker_img,
+                                                               original_validation_id=orig_val_id,
+                                                               environment=env)
+            self.assertEqual(validation_id, result)
+
+    def test_store_file(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            file_info = {'files': 'info'}
+            mock_store_file_method = Mock(return_value=file_info)
+            mock_api_client = Mock(store_file=mock_store_file_method)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+            filename = "bogofile"
+            content = "bogobogobogo"
+            content_type = "application/bogo"
+
+            result = area.store_file(filename=filename, file_content=content, content_type=content_type)
+
+            mock_store_file_method.assert_called_once_with(area_uuid=area.uuid,
+                                                           filename=filename,
+                                                           file_content=content,
+                                                           content_type=content_type)
+            self.assertEqual(file_info, result)
+
+    def test_checksum_status(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            checksum_status = {'checksum': 'status'}
+            mock_checksum_status_method = Mock(return_value=checksum_status)
+            mock_api_client = Mock(checksum_status=mock_checksum_status_method)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+            filename = "bogofile"
+
+            result = area.checksum_status(filename=filename)
+
+            mock_checksum_status_method.assert_called_once_with(area_uuid=area.uuid,
+                                                                filename=filename)
+            self.assertEqual(checksum_status, result)
+
+    def test_validation_status(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            validation_status = {'validation': 'status'}
+            mock_validation_status_method = Mock(return_value=validation_status)
+            mock_api_client = Mock(validation_status=mock_validation_status_method)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+            filename = "bogofile"
+
+            result = area.validation_status(filename=filename)
+
+            mock_validation_status_method.assert_called_once_with(area_uuid=area.uuid,
+                                                                  filename=filename)
+            self.assertEqual(validation_status, result)
+
+    def test_checksum_statuses(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            checksum_statuses = {'checksum': 'statuses'}
+            mock_checksum_statuses_method = Mock(return_value=checksum_statuses)
+            mock_api_client = Mock(checksum_statuses=mock_checksum_statuses_method)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+
+            result = area.checksum_statuses()
+
+            mock_checksum_statuses_method.assert_called_once_with(area_uuid=area.uuid)
+            self.assertEqual(checksum_statuses, result)
+
+    def test_validation_statuses(self):
+        with patch('hca.upload.upload_service.ApiClient') as mock_api_client_class:
+            validation_statuses = {'validation': 'statuses'}
+            mock_validation_statuses_method = Mock(return_value=validation_statuses)
+            mock_api_client = Mock(validation_statuses=mock_validation_statuses_method)
+            mock_api_client_class.return_value = mock_api_client
+
+            area = self._create_upload_area()
+
+            result = area.validation_statuses()
+
+            mock_validation_statuses_method.assert_called_once_with(area_uuid=area.uuid)
+            self.assertEqual(validation_statuses, result)
+
 
 class TestUploadAreaFileUpload(UploadTestCase):