Adds environment variable support

.travis.yml

@@ -13,8 +13,8 @@ install:
   - pip install -r requirements-dev.txt
 # command to run tests
 script:
-    - make tests
-    - flake8 --exclude pycondor/__init__.py pycondor
+    - pytest pycondor
+    - flake8 pycondor
     - travis-sphinx --nowarn --source=docs build
 
 after_success:

docs/api.rst

@@ -13,8 +13,6 @@ The main functionality of PyCondor is contained within the ``Job`` and ``Dagman`
 Job object
 ==========
 
-Stuff about the Job object.
-
 .. autoclass:: Job
     :members:
     :inherited-members:
@@ -25,8 +23,6 @@ Stuff about the Job object.
 Dagman object
 =============
 
-Stuff about the Job object.
-
 .. autoclass:: Dagman
     :members:
     :inherited-members:

docs/changelog.rst

@@ -12,11 +12,12 @@ Version 0.1.5 (TBD)
 **New Features**:
 
 * Added ``dagman_progress`` command line tool for displaying a progress bar for Dagman jobs. (See `PR #45 <https://github.com/jrbourbeau/pycondor/pull/45>`_)
+* Added environment variable option for setting submit, error, log, and output directories. (See `PR #50 <https://github.com/jrbourbeau/pycondor/pull/50>`_)
 
 **Bug Fixes**:
 
 * Fixed bug where the queue parameter for a Job was not written to the job submit file when the Job was built by a Dagman. (See `PR #42 <https://github.com/jrbourbeau/pycondor/pull/42>`_)
-* Fixed bug that caused a filename mismatch between a ``Job`` submit file and the error/log/output files when a named argument is added to a ``Job``, and the ``Job`` is built with ``fancyname=True``. (See `PR #45 <https://github.com/jrbourbeau/pycondor/pull/48>`_)
+* Fixed bug that caused a filename mismatch between a ``Job`` submit file and the error/log/output files when a named argument is added to a Job, and the Job is built with ``fancyname=True``. (See `PR #48 <https://github.com/jrbourbeau/pycondor/pull/48>`_)
 
 
 Version 0.1.4 (2017-06-08)

pycondor/basenode.py

@@ -1,5 +1,4 @@
 
-import os
 import time
 import glob
 
@@ -17,7 +16,7 @@ def __init__(self, name, submit=None, extra_lines=None, verbose=0):
             extra_lines = [extra_lines]
 
         self.name = utils.string_rep(name)
-        self.submit = submit if submit is not None else os.getcwd()
+        self.submit = submit
         self.extra_lines = extra_lines
         self._built = False
 

pycondor/dagman.py

@@ -1,4 +1,5 @@
 
+import os
 import subprocess
 
 from . import utils
@@ -86,7 +87,13 @@ def _get_parent_child_string(node):
 
 
 class Dagman(BaseNode):
-    """Dagman object
+    """
+    Dagman object consisting of a series of Jobs and sub-Dagmans to manage.
+
+    Note that the ``submit`` parameter can be explicitly given or configured
+    by setting the ``PYCONDOR_SUBMIT_DIR`` environment variable. An explicitly
+    given value for ``submit`` will be used over the environment variable,
+    while the environment variable will be used over a default value.
 
     Parameters
     ----------
@@ -96,8 +103,8 @@ class Dagman(BaseNode):
         this Dagman.
 
     submit : str
-        Path to directory where condor dagman submit files will be written.
-        (Defaults to the directory was the job was submitted from).
+        Path to directory where condor dagman submit files will be written
+        (defaults to the directory was the Dagman was submitted from).
 
     extra_lines : list or None, optional
         List of additional lines to be added to submit file.
@@ -114,14 +121,12 @@ class Dagman(BaseNode):
         The list of jobs for this Dagman instance to manage.
 
     parents : list
-        List of parent Jobs and Dagmans. Ensures that Jobs and other
-        Dagmans in the parents list will complete before this Dagman
-        is submitted to HTCondor.
+        List of parent Jobs and Dagmans. Ensures that Jobs and Dagmans in the
+        parents list will complete before this Dagman is submitted to HTCondor.
 
     children : list
-        List of child Jobs and Dagmans. Ensures that Jobs and other
-        Dagmans in the children list will be submitted after this Dagman
-        is has completed.
+        List of child Jobs and Dagmans. Ensures that Jobs and Dagmans in the
+        children list will be submitted only after this Dagman has completed.
 
     """
 
@@ -231,10 +236,20 @@ def build(self, makedirs=True, fancyname=True):
                     'Skipping the build process...'.format(self.name))
             return self
 
-        # Create DAG submit file path
         name = self._get_fancyname() if fancyname else self.name
-        submit_file = '{}/{}.submit'.format(self.submit, name)
+        # Get Dagman submit file directory
+        path = None
+        dir_env_var = os.getenv('PYCONDOR_SUBMIT_DIR')
+        if self.submit is not None:
+            path = self.submit
+        elif dir_env_var:
+            path = dir_env_var
+        # Create Dagman submit file path
+        submit_file = os.path.join(path if path else '',
+                                   '{}.submit'.format(name))
+        # submit_file = '{}/{}.submit'.format(self.submit, name)
         self.submit_file = submit_file
+        self.submit_name = name
         utils.checkdir(self.submit_file, makedirs)
 
         # Write dag submit file

pycondor/job.py

@@ -10,30 +10,42 @@
 
 
 class Job(BaseNode):
-    """Job object
+    """
+    Job object consisting of an executable to be run, potentially with a
+    series of different command-line arguments.
+
+    Note that the ``submit``, ``error``, ``log``, and ``output`` parameters
+    can be explicitly given or configured by setting ``PYCONDOR_SUBMIT_DIR``,
+    ``PYCONDOR_ERROR_DIR``, ``PYCONDOR_LOG_DIR``, and ``PYCONDOR_OUTPUT_DIR``
+    environment variables. An explicitly given value will be used over an
+    environment variable, while an environment variable will be used over a
+    default value.
 
     Parameters
     ----------
     name : str
         Name of the Job instance. This will also be the name of the
         corresponding error, log, output, and submit files associated with
-        this job.
+        this Job.
 
     executable : str
         Path to corresponding executable for Job.
 
     error : str or None, optional
-        Path to directory where condor job error files will be written.
+        Path to directory where condor Job error files will be written (default
+        is None, will not be included in Job submit file).
 
     log : str or None, optional
-        Path to directory where condor job log files will be written.
+        Path to directory where condor Job log files will be written (default
+        is None, will not be included in Job submit file).
 
     output : str or None, optional
-        Path to directory where condor job output files will be written.
+        Path to directory where condor Job output files will be written
+        (default is None, will not be included in Job submit file).
 
     submit : str, optional
-        Path to directory where condor job submit files will be written.
-        (Defaults to the directory was the job was submitted from).
+        Path to directory where condor Job submit files will be written
+        (defaults to the directory was the Job was submitted from).
 
     request_memory : str or None, optional
         Memory request to be included in submit file.
@@ -72,23 +84,28 @@ class Job(BaseNode):
 
     verbose : int
         Level of logging verbosity option are 0-warning, 1-info,
-        2-debugging(default is 0).
+        2-debugging (default is 0).
 
     Attributes
     ----------
     args : list
         The list of arguments for this Job instance.
 
     parents : list
-        Only applies when Job is in a Dagman. List of parent Jobs and Dagmans.
-        Ensures that Jobs and other Dagmans in the parents list will complete
+        Only set when included in a Dagman. List of parent Jobs and Dagmans.
+        Ensures that Jobs and Dagmans in the parents list will complete
         before this Job is submitted to HTCondor.
 
     children : list
-        Only applies when Job is in a Dagman. List of child Jobs and Dagmans.
-        Ensures that Jobs and other Dagmans in the children list will be
-        submitted after this Job is has completed.
+        Only set when included in a Dagman. List of child Jobs and Dagmans.
+        Ensures that Jobs and Dagmans in the children list will be
+        submitted only after this Job has completed.
 
+    Examples
+    --------
+    >>> import pycondor
+    >>> job = pycondor.Job('myjob', 'myscript.py')
+    >>> job.build_submit()
 
     """
 
@@ -215,14 +232,12 @@ def _make_submit_script(self, makedirs=True, fancyname=True, indag=False):
 
         # Check that paths/files exist
         if not os.path.exists(self.executable):
-            raise IOError('The path {} does not exist'.format(self.executable))
+            raise IOError(
+                'The executable {} does not exist'.format(self.executable))
         for directory in [self.submit, self.log, self.output, self.error]:
             if directory is not None:
                 utils.checkdir(directory + '/', makedirs)
 
-        name = self._get_fancyname() if fancyname else self.name
-        submit_file = '{}/{}.submit'.format(self.submit, name)
-
         # Start constructing lines to go into job submit file
         lines = []
         submit_attrs = ['universe', 'executable', 'request_memory',
@@ -233,20 +248,37 @@ def _make_submit_script(self, makedirs=True, fancyname=True, indag=False):
                 attr_str = utils.string_rep(getattr(self, attr))
                 lines.append('{} = {}'.format(attr, attr_str))
 
-        # Set up log, output, and error files paths
+        # Set up submit, log, output, and error files paths
+        name = self._get_fancyname() if fancyname else self.name
+        self.submit_name = name
         self._has_arg_names = any([arg.name for arg in self.args])
-        for attr in ['log', 'output', 'error']:
+        for attr in ['submit', 'log', 'output', 'error']:
+            dir_path = ''
+            dir_env_var = os.getenv('PYCONDOR_{}_DIR'.format(attr.upper()))
+            # Check if directory is provided
             if getattr(self, attr) is not None:
-                path = getattr(self, attr)
-                # If path has trailing '/', then it it removed.
-                # Else, path is unmodified
-                path = path.rstrip('/')
+                dir_path = getattr(self, attr)
+            # If not, check if directory environment variable is set
+            elif dir_env_var:
+                dir_path = dir_env_var
+
+            if attr == 'submit':
+                submit_file = os.path.join(dir_path, '{}.submit'.format(name))
+                # Add submit_file data member to job for later use
+                self.submit_file = submit_file
+                utils.checkdir(submit_file, makedirs)
+                continue
+            # Add log/output/error files to submit file lines
+            if dir_path:
                 if self._has_arg_names:
-                    lines.append('{} = {}/$(job_name).{}'.format(
-                                 attr, path, attr))
+                    file_path = os.path.join(dir_path,
+                                             '$(job_name).{}'.format(attr))
                 else:
-                    lines.append('{} = {}/{}.{}'.format(attr, path,
-                                 name, attr))
+                    file_path = os.path.join(dir_path,
+                                             '{}.{}'.format(name, attr))
+                lines.append('{} = {}'.format(attr, file_path))
+                setattr(self, '{}_file'.format(attr), file_path)
+                utils.checkdir(file_path, makedirs)
 
         # Add any extra lines to submit file, if specified
         if self.extra_lines:
@@ -296,10 +328,6 @@ def _make_submit_script(self, makedirs=True, fancyname=True, indag=False):
         with open(submit_file, 'w') as f:
             f.writelines('\n'.join(lines))
 
-        # Add submit_file data member to job for later use
-        self.submit_file = submit_file
-        self.submit_name = name
-
         return
 
     def build(self, makedirs=True, fancyname=True):

pycondor/tests/test_dagman.py

@@ -3,6 +3,9 @@
 import filecmp
 import pytest
 import pycondor
+from .utils import clear_pycondor_environment_variables
+
+clear_pycondor_environment_variables()
 
 
 def test_add_job_int_fail():
@@ -21,12 +24,12 @@ def test_job_dag_submit_file_same(tmpdir):
     example_script = os.path.join('examples/savelist.py')
     submit_dir = str(tmpdir.mkdir('submit'))
     # Build Job object that will be built outside of a Dagman
-    job_outside_dag = pycondor.Job('job_outside_dag', example_script,
+    job_outside_dag = pycondor.Job('test_job', example_script,
                                    submit=submit_dir, queue=5)
     job_outside_dag.build(fancyname=False)
 
     # Build Job object that will be built inside of a Dagman
-    job_inside_dag = pycondor.Job('job_inside_dag', example_script,
+    job_inside_dag = pycondor.Job('test_job', example_script,
                                   submit=submit_dir, queue=5)
     dagman = pycondor.Dagman('exampledagman', submit=submit_dir)
     dagman.add_job(job_inside_dag)

pycondor/tests/test_job.py

@@ -2,6 +2,9 @@
 import os
 import pytest
 import pycondor
+from .utils import clear_pycondor_environment_variables
+
+clear_pycondor_environment_variables()
 
 
 def test_add_arg_type_fail():
@@ -63,7 +66,7 @@ def test_build_executeable_not_found_fail():
         ex = '/path/to/executable'
         job = pycondor.Job('jobname', ex)
         job.build(makedirs=False)
-    error = 'The path {} does not exist'.format(ex)
+    error = 'The executable {} does not exist'.format(ex)
     assert error == str(excinfo.value)
 
 
@@ -85,3 +88,18 @@ def test_queue_written_to_submit_file(tmpdir):
     with open(job.submit_file, 'r') as f:
         lines = f.readlines()
     assert 'queue 5' in lines
+
+
+def test_job_env_variable_dir(tmpdir):
+    # Set pycondor environment variables
+    for dir_name in ['submit', 'output', 'error', 'log']:
+        dir_path = str(tmpdir.mkdir(dir_name))
+        os.environ['PYCONDOR_{}_DIR'.format(dir_name.upper())] = dir_path
+
+    example_script = os.path.join('examples/savelist.py')
+    job = pycondor.Job('jobname', example_script)
+    job.build()
+    for dir_name in ['submit', 'output', 'error', 'log']:
+        tmpdir_path = os.path.join(str(tmpdir), dir_name)
+        job_path = os.path.dirname(getattr(job, '{}_file'.format(dir_name)))
+        assert tmpdir_path == job_path

pycondor/tests/utils.py

@@ -0,0 +1,8 @@
+
+import os
+
+
+def clear_pycondor_environment_variables():
+    # Unset any pycondor directory environment variables
+    for i in ['submit', 'output', 'error', 'log']:
+        os.environ['PYCONDOR_{}_DIR'.format(i.upper())] = ''

setup.cfg

@@ -1,2 +1,8 @@
 [bdist_wheel]
 universal=1
+
+[tool:pytest]
+addopts = -sv
+
+[flake8]
+exclude = __init__.py,__pycache__