Merge pull request #698 from mskcc/develop

copy files

beagle/settings.py

@@ -12,6 +12,7 @@
 
 import os
 import ldap
+import json
 import datetime
 from django_auth_ldap.config import LDAPSearch
 
@@ -358,3 +359,4 @@
 WES_ASSAYS = os.environ.get('BEAGLE_NOTIFIER_WES_ASSAYS', 'WholeExomeSequencing').split(',')
 NOTIFIER_WES_CC = os.environ.get('BEAGLE_NOTIFIER_WHOLE_EXOME_SEQUENCING_CC', '')
 
+DEFAULT_MAPPING = json.loads(os.environ.get("BEAGLE_COPY_MAPPING", "{}"))

beagle_etl/copy_service/__init__.py

@@ -0,0 +1 @@
+from .copy_service import CopyService

beagle_etl/copy_service/copy_service.py

@@ -0,0 +1,27 @@
+import os
+import logging
+from shutil import copyfile
+from django.conf import settings
+
+
+logger = logging.getLogger(__name__)
+
+
+class CopyService(object):
+
+    @staticmethod
+    def copy(path_from, path_to):
+        logger.info('Copy path from {path_from} to {path_to}'.format(path_from=path_from,
+                                                                     path_to=path_to))
+        dirname = os.path.dirname(path_to)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        copyfile(path_from, path_to)
+
+    @staticmethod
+    def remap(path, mapping=settings.DEFAULT_MAPPING):
+        for prefix, dst in mapping.items():
+            if path.startswith(prefix):
+                path = path.replace(prefix, dst)
+        logger.info('New path {path}'.format(path=path))
+        return path

beagle_etl/jobs/lims_etl_jobs.py

@@ -22,6 +22,7 @@
 from file_system.helper.checksum import sha1, FailedToCalculateChecksum
 from runner.operator.helper import format_sample_name, format_patient_id
 from beagle_etl.lims_client import LIMSClient
+from beagle_etl.copy_service import CopyService
 from django.contrib.auth.models import User
 
 
@@ -360,6 +361,14 @@ def create_pooled_normal(filepath, file_group_id):
         "preservation": preservation_type,
         "recipe": recipe
     }
+    try:
+        new_path = CopyService.remap(filepath)
+        if new_path != filepath:
+            CopyService.copy(filepath, new_path)
+    except Exception as e:
+        logger.error("Failed to copy file %s." % (filepath,))
+        raise FailedToFetchPoolNormalException("Failed to copy file %s. Error %s" % (filepath, str(e)))
+
     try:
         f = File.objects.create(file_name=os.path.basename(filepath), path=filepath, file_group=file_group_obj,
                                 file_type=file_type_obj)
@@ -590,14 +599,20 @@ def format_metadata(original_metadata):
 
 def create_file_object(path, file_group, lims_metadata, metadata, file_type):
     try:
-        f = File.objects.create(file_name=os.path.basename(path), path=path, file_group=file_group,
+        new_path = CopyService.remap(path)
+        if path != new_path:
+            CopyService.copy(path, new_path)
+    except Exception as e:
+        raise FailedToFetchSampleException("Failed to copy file %s. Error %s" % (path, str(e)))
+    try:
+        f = File.objects.create(file_name=os.path.basename(new_path), path=new_path, file_group=file_group,
                                 file_type=file_type)
         f.save()
 
         fm = FileMetadata(file=f, metadata=metadata)
         fm.save()
         Job.objects.create(run=TYPES["CALCULATE_CHECKSUM"],
-                           args={'file_id': str(f.id), 'path': path},
+                           args={'file_id': str(f.id), 'path': new_path},
                            status=JobStatus.CREATED, max_retry=3, children=[])
         import_metadata = ImportMetadata.objects.create(file=f, metadata=lims_metadata)
     except Exception as e:

beagle_etl/tests/copy_service/test_copy_service.py

@@ -0,0 +1,23 @@
+from django.test import TestCase
+from beagle_etl.copy_service import CopyService
+from django.conf import settings
+from django.test.utils import override_settings
+from unittest.mock import patch
+
+
+class JobViewTest(TestCase):
+
+    def setUp(self):
+        self.mapping = dict()
+        self.mapping['/path/to'] = '/new/path/to'
+        patch.dict('os.environ', {'BEAGLE_COPY_MAPPING': "{'/path/to':'/default/path/to'}"})
+
+    def test_remap(self):
+        old_path = '/path/to/file/file1.fastq'
+        new_path = CopyService.remap(old_path, self.mapping)
+        self.assertEqual(new_path, '/new/path/to/file/file1.fastq')
+
+    def test_remap_no_mapping(self):
+        old_path = '/some/other/path/to/file/file1.fastq'
+        new_path = CopyService.remap(old_path, self.mapping)
+        self.assertEqual(new_path, old_path)