Use one directory for all housekeeping

Before, your working directory would be littered with .drmr
subdirectories, and locating a failed job could be tedious. Now, one
.drmr directory is created in the directory where you submit a script,
and all resource manager scripts and output can be found there.

Also fix completion job scripts in drmrarray.

drmr/__init__.py

@@ -10,4 +10,4 @@
 
 __author__ = 'The Parker Lab'
 __email__ = 'parkerlab-software@umich.edu'
-__version__ = '1.0.0'
+__version__ = '1.0.2'

drmr/drm/base.py

@@ -10,6 +10,7 @@
 from __future__ import print_function
 
 import copy
+import datetime
 import inspect
 import logging
 import os
@@ -138,7 +139,8 @@ def normalize_time_limit(self, job_data):
 
     def set_control_directory(self, job_data):
         """Add the path of the control directory to the job data."""
-        control_path = drmr.util.absjoin(job_data.get('working_directory', os.getcwd()), '.drmr')
+        timestamp = job_data.get('timestamp') or datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+        control_path = drmr.util.absjoin(job_data.get('submission_directory', os.getcwd()), '.drmr', '{}-{}'.format(job_data['master_job_name'], timestamp))
         job_data['control_directory'] = control_path
         return control_path
 

scripts/drmr

@@ -14,6 +14,7 @@
 from __future__ import print_function
 
 import argparse
+import datetime
 import copy
 import logging
 import os
@@ -215,7 +216,7 @@ def create_jobs(resource_manager, template_data, script, wait_list=None, mail_at
     if to_label:
         logger.debug('Skipping everything after label "{}"'.format(to_label))
 
-    master_job_name = template_data['job_name']
+    master_job_name = template_data['master_job_name']
     if start_held:
         logger.debug('Submitting start hold job.')
         hold_job_name = master_job_name + '.start'
@@ -296,7 +297,9 @@ if __name__ == '__main__':
     template_data = {
         'account': config['account'],
         'destination': config['destination'],
-        'job_name': args.job_name or os.path.basename(args.input),
+        'master_job_name': args.job_name or os.path.basename(args.input),
+        'submission_directory': os.path.abspath(os.getcwd()),
+        'timestamp': datetime.datetime.now().strftime('%Y%m%d%H%M%S'),
         'working_directory': os.path.abspath(os.getcwd()),
     }
 

scripts/drmrarray

@@ -14,6 +14,7 @@
 from __future__ import print_function
 
 import argparse
+import datetime
 import logging
 import os
 import sys
@@ -103,13 +104,10 @@ def create_jobs(resource_manager, template_data, script, wait_list=None):
     if wait_list is None:
         wait_list = []
 
-    job_name = template_data['job_name']
     job_directives = {}
 
     job_data = template_data.copy()
-    job_data.update({
-        'job_name': job_name
-    })
+    job_data['job_name'] = job_data['master_job_name']
 
     commands = []
     for i, line in enumerate(script, 1):
@@ -162,8 +160,10 @@ if __name__ == '__main__':
     template_data = {
         'account': config['account'],
         'destination': config['destination'],
-        'job_name': args.job_name or os.path.basename(args.input),
+        'master_job_name': args.job_name or os.path.basename(args.input),
         'slot_limit': args.slot_limit,
+        'submission_directory': drmr.util.absjoin(os.getcwd()),
+        'timestamp': datetime.datetime.now().strftime('%Y%m%d%H%M%S'),
         'working_directory': os.path.abspath(os.getcwd()),
     }
     if args.mail_on_error:
@@ -181,7 +181,9 @@ if __name__ == '__main__':
     try:
         completion_job_id = create_jobs(resource_manager, template_data, script, wait_list)
         if args.finish_jobs or args.mail_at_finish:
-            completion_job_id = resource_manager.submit_completion_jobs(template_data, [completion_job_id], mail_at_finish=args.mail_at_finish)
+            job_data = template_data.copy()
+            job_data['job_name'] = job_data['master_job_name']
+            completion_job_id = resource_manager.submit_completion_jobs(job_data, [completion_job_id], mail_at_finish=args.mail_at_finish)
         print(completion_job_id)
     except drmr.exceptions.SubmissionError as e:
         print('\nYour script could not be submitted.')

setup.py

@@ -19,7 +19,7 @@
 
 setup(
     name='drmr',
-    version='1.0.1',
+    version='1.0.2',
     description="A tool for submitting pipeline scripts to distributed resource managers.",
     long_description=readme + '\n\n',
     author="The Parker Lab",