Scottx611x/fix add file source translation

refinery/core/models.py

@@ -816,12 +816,15 @@ def _associate_datafiles_with_investigation(self, investigation):
         file_store_items_from_existing_data_set = \
             existing_investigation.get_file_store_items(local_only=True)
         new_file_store_items = investigation.get_file_store_items()
-        new_sources = [f.source for f in new_file_store_items]
+        new_sources = [os.path.basename(f.source) for f in
+                       new_file_store_items]
         for prior_file_store_item in file_store_items_from_existing_data_set:
-            if prior_file_store_item.source not in new_sources:
+            if os.path.basename(prior_file_store_item.source) \
+                    not in new_sources:
                 prior_file_store_item.delete_datafile()
             for new_file_store_item in new_file_store_items:
-                if prior_file_store_item.source == new_file_store_item.source:
+                if os.path.basename(prior_file_store_item.source) == \
+                        os.path.basename(new_file_store_item.source):
                     prior_file_store_item.transfer_data_file(
                         new_file_store_item
                     )

refinery/data_set_manager/test_views.py

@@ -1,6 +1,9 @@
+import mock
 import uuid
 
+from django.conf import settings
 from django.contrib.auth.models import User
+from django.test import override_settings
 
 from rest_framework.test import (APIClient, APIRequestFactory, APITestCase,
                                  force_authenticate)
@@ -66,3 +69,99 @@ def test_post_returns_400_node_uuid_not_present(self):
         force_authenticate(post_request, user=self.user)
         post_response = self.view(post_request)
         self.assertEqual(post_response.status_code, 400)
+
+    @override_settings(REFINERY_DEPLOYMENT_PLATFORM="aws")
+    def test_aws_post_returns_400_no_identity_id(self):
+        post_request = self.factory.post(
+            self.url_root,
+            data={'node_uuid': uuid.uuid4()},
+            format="json"
+        )
+        post_request.user = self.user
+        force_authenticate(post_request, user=self.user)
+        post_response = self.view(post_request)
+        self.assertEqual(post_response.status_code, 400)
+
+    @override_settings(REFINERY_DEPLOYMENT_PLATFORM="aws")
+    def test_aws_post_returns_202_with_identity_id(self):
+        post_request = self.factory.post(
+            self.url_root,
+            data={
+                'node_uuid': self.node.uuid,
+                'identity_id': uuid.uuid4()
+            },
+            format="json"
+        )
+        post_request.user = self.user
+        force_authenticate(post_request, user=self.user)
+        post_response = self.view(post_request)
+        self.assertEqual(post_response.status_code, 202)
+
+    @override_settings(REFINERY_DEPLOYMENT_PLATFORM="not aws")
+    def test_non_aws_post_returns_400_if_identity_id(self):
+        post_request = self.factory.post(
+            self.url_root,
+            data={
+                'node_uuid': self.node.uuid,
+                'identity_id': uuid.uuid4()
+            },
+            format="json"
+        )
+        post_request.user = self.user
+        force_authenticate(post_request, user=self.user)
+        post_response = self.view(post_request)
+        self.assertEqual(post_response.status_code, 400)
+
+    @override_settings(REFINERY_DEPLOYMENT_PLATFORM="aws",
+                       UPLOAD_BUCKET="test_bucket",
+                       REFINERY_DATA_IMPORT_DIR="/import/path",
+                       CELERY_ALWAYS_EAGER=True)
+    @mock.patch("data_set_manager.models.Node.update_solr_index")
+    def test_aws_post_file_store_item_source_translated(self,
+                                                        update_solr_mock):
+        file_store_item = self.node.get_file_store_item()
+        file_store_item.source = "{}/{}/test.txt".format(
+            settings.REFINERY_DATA_IMPORT_DIR,
+            self.user.username
+        )
+        file_store_item.save()
+        post_request = self.factory.post(
+            self.url_root,
+            data={
+                'node_uuid': self.node.uuid,
+                'identity_id': "test_identity_id"
+            },
+            format="json"
+        )
+        post_request.user = self.user
+        force_authenticate(post_request, user=self.user)
+        self.view(post_request)
+        self.assertEqual(self.node.get_file_store_item().source,
+                         's3://test_bucket/test_identity_id/test.txt')
+        self.assertTrue(update_solr_mock.called)
+
+    @override_settings(REFINERY_DEPLOYMENT_PLATFORM="not aws",
+                       REFINERY_DATA_IMPORT_DIR="/import/path",
+                       CELERY_ALWAYS_EAGER=True)
+    @mock.patch("data_set_manager.models.Node.update_solr_index")
+    def test_non_aws_post_file_store_item_source_translated(self,
+                                                            update_solr_mock):
+        file_store_item_source = "{}/{}/test.txt".format(
+            settings.REFINERY_DATA_IMPORT_DIR,
+            self.user.username
+        )
+        file_store_item = self.node.get_file_store_item()
+        file_store_item.source = file_store_item_source
+        file_store_item.save()
+
+        post_request = self.factory.post(
+            self.url_root,
+            data={'node_uuid': self.node.uuid},
+            format="json"
+        )
+        post_request.user = self.user
+        force_authenticate(post_request, user=self.user)
+        self.view(post_request)
+        self.assertEqual(self.node.get_file_store_item().source,
+                         file_store_item_source)
+        self.assertTrue(update_solr_mock.called)

refinery/data_set_manager/views.py

@@ -1047,25 +1047,49 @@ def post(self, request):
             node_uuid = request.data["node_uuid"]
         except KeyError:
             return HttpResponseBadRequest("`node_uuid` required")
+
+        identity_id = request.data.get("identity_id")
+        if settings.REFINERY_DEPLOYMENT_PLATFORM == "aws" and not identity_id:
+            return HttpResponseBadRequest("`identity_id` required")
+        elif settings.REFINERY_DEPLOYMENT_PLATFORM != "aws" and identity_id:
+            return HttpResponseBadRequest("`identity_id` not permitted for "
+                                          "non-AWS deployments")
+
         try:
             node = Node.objects.get(uuid=node_uuid)
         except Node.DoesNotExist:
             logger.error("Node with UUID '%s' does not exist", node_uuid)
             return HttpResponseNotFound()
-        except DataSet.MultipleObjectsReturned:
+        except Node.MultipleObjectsReturned:
             logger.critical("Multiple Nodes found with UUID '%s'", node_uuid)
             return HttpResponseServerError()
 
         if request.user != node.study.get_dataset().get_owner():
             return HttpResponseForbidden()
 
-        logger.debug("Adding file to Node '%s'", node)
-
         file_store_item = node.get_file_store_item()
         if (file_store_item and not file_store_item.datafile and
                 file_store_item.source.startswith(
                     (settings.REFINERY_DATA_IMPORT_DIR, 's3://')
                 )):
+            logger.debug("Adding file to Node '%s'", node)
+
+            file_store_item.source = os.path.basename(file_store_item.source)
+            file_store_item.save()
+
+            if identity_id:
+                file_source_translator = generate_file_source_translator(
+                    identity_id=identity_id
+                )
+            else:
+                file_source_translator = generate_file_source_translator(
+                    username=request.user.username
+                )
+            translated_datafile_source = file_source_translator(
+                file_store_item.source
+            )
+            file_store_item.source = translated_datafile_source
+
             # Remove the FileStoreItem's import_task_id to treat it as a
             # brand new import_file task when called below.
             # We then have to update its Node's Solr index entry, so the

refinery/ui/source/js/data-set-import/controllers/file-upload-s3.js

@@ -93,7 +93,8 @@
           file.success = true;
           if (vm.isNodeUpdate) {
             addFileToDataSetService.update({
-              node_uuid: vm.nodeUuid
+              node_uuid: vm.nodeUuid,
+              identity_id: AWS.config.credentials.identityId
             }).$promise
               .then(function () {
                 vm.addFileStatus = 'success';