adds cron job to rsync packages

Dockerfile

@@ -20,6 +20,7 @@ RUN apt-get update \
     ssh \
     vim \
     wget \
+    rsync \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -53,7 +54,7 @@ COPY aurora/ /data/htdocs/aurora/aurora
 COPY setup_objects.py /data/htdocs/aurora/
 
 # Install Python modules
-RUN pip install -r /data/htdocs/aurora/requirements.txt
+RUN pip install --upgrade pip && pip install -r /data/htdocs/aurora/requirements.txt
 
 EXPOSE 8000 3310
 

aurora/aurora/config.py.example

@@ -26,7 +26,12 @@ TIME_ZONE = 'America/New_York'
 # Directory configs
 ORG_ROOT_DIR = '/data/'
 STORAGE_ROOT_DIR = '/data1/'
-DELIVERY_QUEUE_ROOT_DIR = '/delivery/'
+DELIVERY_DIR = '/delivery/'
+DELIVERY = {
+    'user': 'root',
+    'host': 'http://ursa_major',
+    'dir': '/storage/'
+}
 STATIC_ROOT = '/data/htdocs/aurora/aurora/static/'
 PROJECT_ROOT_DIR = '/data/htdocs/aurora/'
 

aurora/aurora/settings.py

@@ -133,7 +133,8 @@
 STATIC_ROOT = CF.STATIC_ROOT
 ORG_ROOT_DIR = CF.ORG_ROOT_DIR
 STORAGE_ROOT_DIR = CF.STORAGE_ROOT_DIR
-DELIVERY_QUEUE_ROOT_DIR = CF.DELIVERY_QUEUE_ROOT_DIR
+DELIVERY_DIR = CF.DELIVERY_DIR
+DELIVERY = CF.DELIVERY
 
 
 # Transfer settings
@@ -158,6 +159,7 @@
 # Django Cron
 CRON_CLASSES = [
     "bag_transfer.lib.cron.DiscoverTransfers",
+    "bag_transfer.lib.cron.DeliverTransfers",
 ]
 
 

aurora/bag_transfer/accession/views.py

@@ -43,8 +43,8 @@ class AccessionRecordView(AccessioningArchivistMixin, View):
     form_class = AccessionForm
 
     def package_transfer(self, transfer, request):
-        tar_dir = join(settings.DELIVERY_QUEUE_ROOT_DIR, transfer.machine_file_identifier)
-        tarfilepath = "{}package_{}.tar.gz".format(settings.DELIVERY_QUEUE_ROOT_DIR, transfer.machine_file_identifier)
+        tar_dir = join(settings.DELIVERY_DIR, transfer.machine_file_identifier)
+        tarfilepath = "{}package_{}.tar.gz".format(settings.DELIVERY_DIR, transfer.machine_file_identifier)
         if not isdir(tar_dir):
             makedirs(tar_dir)
         # add metadata file to package
@@ -56,7 +56,7 @@ def package_transfer(self, transfer, request):
             bagtar.add(transfer.machine_file_path, arcname=basename(transfer.machine_file_identifier))
             bagtar.close()
         # zip up the package
-        with tarfile.open(join(settings.DELIVERY_QUEUE_ROOT_DIR, '{}.tar.gz'.format(transfer.machine_file_identifier)), "w:gz") as tar:
+        with tarfile.open(join(settings.DELIVERY_DIR, '{}.tar.gz'.format(transfer.machine_file_identifier)), "w:gz") as tar:
             tar.add(tar_dir, arcname=basename(tar_dir))
             tar.close()
         rmtree(tar_dir)

aurora/bag_transfer/lib/cron.py

@@ -1,7 +1,11 @@
 import datetime
+from os import listdir
+from os.path import join
+import subprocess
 
 from django_cron import CronJobBase, Schedule
 
+from aurora import settings
 from bag_transfer.lib import files_helper as FH
 from bag_transfer.lib.transfer_routine import *
 from bag_transfer.lib.bag_checker import bagChecker
@@ -17,7 +21,7 @@ class DiscoverTransfers(CronJobBase):
     code = 'transfers.discover_transfers'
 
     def do(self):
-        Pter.cron_open()
+        Pter.cron_open(self.code)
         BAGLog.log_it('CSTR')
 
         transferRoutine = TransferRoutine(1)
@@ -98,9 +102,35 @@ def do(self):
                 # Delete tmp files
                 FH.remove_file_or_dir('/data/tmp/{}'.format(new_arc.bag_it_name))
 
-        print '############################'
-        print 'CRON END'
-        print datetime.datetime.now()
-        print '############################'
-        print '\n\n\n'
-        BAGLog.log_it('CEND')
+        Pter.cron_close(self.code)
+
+
+class DeliverTransfers(CronJobBase):
+    RUN_EVERY_MINS = 1
+
+    schedule = Schedule(run_every_mins=RUN_EVERY_MINS)
+    code = 'transfers.deliver_transfers'
+
+    def do(self):
+        Pter.cron_open(self.code)
+        for transfer in listdir(settings.DELIVERY_DIR):
+            rsynccmd = "rsync -avh --remove-source-files {} {}@{}:{}".format(join(settings.DELIVERY_DIR, transfer),
+                                                                             settings.DELIVERY['user'],
+                                                                             settings.DELIVERY['host'],
+                                                                             settings.DELIVERY['dir'])
+            print(rsynccmd)
+            rsyncproc = subprocess.Popen(rsynccmd,
+                                         shell=True,
+                                         stdin=subprocess.PIPE,
+                                         stdout=subprocess.PIPE,)
+            while True:
+                next_line = rsyncproc.stdout.readline().decode("utf-8")
+                if not next_line:
+                    break
+                print(next_line)
+
+            ecode = rsyncproc.wait()
+            if ecode != 0:
+                continue
+
+        Pter.cron_close(self.code)

aurora/bag_transfer/lib/log_print.py

@@ -1,6 +1,6 @@
 import datetime
 
-long_wrapper_str = '############################'
+long_wrapper_str = '##########################################'
 medium_wrapper_str = '###############'
 
 
@@ -37,8 +37,12 @@ def flines(lst, start=False,end=False,tab=2,pref='',line_after=False):
     plines(lst,tab=tab,pref=pref,line_after=line_after)
 
 
-def cron_open():
-    plines(['CRON STARTING', datetime.datetime.now()], 1)
+def cron_open(cron_code):
+    plines(['{} cron start'.format(cron_code).upper(), datetime.datetime.now()], 1)
+
+
+def cron_close(cron_code):
+    plines(['{} cron end'.format(cron_code).upper(), datetime.datetime.now()], 1)
 
 
 def spacer():