Merge a45fc68 into f18bf1b

.travis.yml

@@ -61,7 +61,7 @@ matrix:
         # Coverage test, pass the results to coveralls.
         - os: linux
           python: 3.5
-          env: MAIN_CMD='coverage' SETUP_CMD='run hpsspy/test/hpsspy_test_suite.py'
+          env: MAIN_CMD='coverage' SETUP_CMD='run setup.py test'
 
         # PEP 8 compliance.
         - os: linux

doc/changes.rst

@@ -5,7 +5,9 @@ Release Notes
 0.4.1 (unreleased)
 ------------------
 
-* No changes yet.
+* Handle directory names that contain underscore characters (PR `#4`_).
+
+.. _`#4`: https://github.com/weaverba137/hpsspy/pull/4
 
 0.4.0 (2017-08-10)
 ------------------

doc/configuration.rst

@@ -160,10 +160,12 @@ imposes some additional requirements, conventions and idioms:
     For example ``batch.tar`` means "archive a batch/ directory".
     For longer file names, the "suffix" of the file will be used.
     ``data_d1_batch.tar`` also means "archive a batch/ directory", because
-    ``data_d1_`` is stripped off.
+    ``data_d1_`` is stripped off.  The directory name will be verified, so
+    if the directory to back up is actually ``d1_batch/``, ``batch/`` will be
+    searched for, then ``d1_batch/``.
   - An archive filename that ends with ``_files.tar``, *e.g.* ``foo/bar_files.tar``
     is a signal to :command:`missing_from_hpss` to construct
-    the archive file in a certain way, not by decending into a directory,
+    the archive file in a certain way, not by descending into a directory,
     but by constructing an explicit list of files and building an archive
     file out of that.
 
@@ -175,15 +177,16 @@ imposes some additional requirements, conventions and idioms:
     ``"foo/(bar|baz|flub)/.*$" : "foo/foo_\\1.tar"``.  The name of the
     directory matched in parentheses will be substituted into the file name.
   - Archive arbitrary subdirectories of a *set* of subdirectories:
-    ``"d1/foo/(ab|bc|cd|de|ef)/([^/]+)/.*$":"d1/foo/\\1/d1_foo_\\1_\\2.tar"``
+    ``"d1/foo/(ab|bc|cd|de|ef)/([^/]+)/.*$" : "d1/foo/\\1/d1_foo_\\1_\\2.tar"``
   - Match files in a directory, but not any files in any
     subdirectory: ``"foo/[^/]+$" : "foo_files.tar"``.  See also the
     ``_files.tar`` convention mentioned above.
   - Group some but not all subdirectories in a directory into a single
     archive file for efficiency: ``"foo/([0-9])([0-9][0-9])/.*$" : "foo/foo_\\1XX.tar"``.
     Note the ending of the archive file, and that the directories have to
     have a very uniform naming convention (three and only three digits
-    in this example).
+    in this example).  Also, the placeholder ``X`` needs to be at the *end* of
+    the file name.
   - Do not create an archive file, just copy the file, as is, to HPSS:
     ``"d1/README\\.txt$" : "d1/README.txt"``.  Similarly, for a set of TXT files:
     ``"d1/([^/]+\\.txt)$" : "d1/\\1"``.

hpsspy/os/__init__.py

@@ -10,7 +10,7 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 #
-from . import path
+# from . import path
 from ._os import *
 
 import re

hpsspy/os/_os.py

@@ -1,8 +1,8 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 # -*- coding: utf-8 -*-
 """
-hpsspy._os
-~~~~~~~~~~
+hpsspy.os._os
+~~~~~~~~~~~~~
 
 Contains the actual functions in :mod:`hpsspy.os`.
 """

hpsspy/scan.py

@@ -311,12 +311,27 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
                         fp.write(Lfile_lines)
             else:
                 Lfile = None
-                htar_dir = [basename(h).split('_')[-1].split('.')[0]]
-                if 'X' in htar_dir[0]:
-                    htar_re = re.compile(htar_dir[0].replace('X', '.') + '$')
-                    htar_dir = [d for d in listdir(full_chdir)
-                                if isdir(join(full_chdir, d)) and
-                                htar_re.match(d) is not None]
+                #
+                # Be careful, because the directory name may itself
+                # contain underscore characters, or X characters.
+                #
+                htar_base = basename(h).rsplit('.', 1)[0]  # remove .tar
+                htar_dir = []
+                for b in iterrsplit(htar_base, '_'):
+                    if b.endswith('X'):
+                        htar_re = re.compile(b.replace('X', '.') + '$')
+                        htar_dir = [d for d in listdir(full_chdir)
+                                    if isdir(join(full_chdir, d)) and
+                                    htar_re.match(d) is not None]
+                    else:
+                        if isdir(join(full_chdir, b)):
+                            htar_dir = [b]
+                    if len(htar_dir) > 0:
+                        break
+                if len(htar_dir) == 0:
+                    logger.error(("Could not find directories corresponding " +
+                                  "to %s!"), h)
+                    continue
             logger.debug("chdir('%s')", full_chdir)
             chdir(full_chdir)
             h_dir = join(hpss_root, disk_chdir)
@@ -368,6 +383,29 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
     return
 
 
+def iterrsplit(s, c):
+    """Split string `s` on `c` and rejoin on `c` from the end of `s`.
+
+    Parameters
+    ----------
+    s : :class:`str`
+        String to split
+    c : :class:`str`
+        Split on this string.
+
+    Returns
+    -------
+    :class:`str`
+        Iteratively return the joined parts of `s`.
+    """
+    ss = s.split(c)
+    i = -1
+    while abs(i) <= len(ss):
+        yield c.join(ss[i:])
+        i -= 1
+    return
+
+
 def scan_disk(disk_roots, disk_files_cache, clobber=False):
     """Scan a directory tree on disk and cache the files found there.
 

hpsspy/test/__init__.py

@@ -1,3 +1,33 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
+"""
+hpsspy.test
+~~~~~~~~~~~
+
+Used to initialize the unit test framework via ``python setup.py test``.
+"""
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+# The line above will help with 2to3 support.
+import unittest
+
+
+def hpsspy_test_suite():
+    """Returns unittest.TestSuite of hpsspy tests.
+
+    This is factored out separately from runtests() so that it can be used by
+    ``python setup.py test``.
+    """
+    from os.path import dirname
+    py_dir = dirname(dirname(__file__))
+    return unittest.defaultTestLoader.discover(py_dir,
+                                               top_level_dir=dirname(py_dir))
+
+
+def runtests():
+    """Run all tests in hpsspy.test.test_*.
+    """
+    # Load all TestCase classes from hpsspy/test/test_*.py
+    tests = hpsspy_test_suite()
+    # Run them
+    unittest.TextTestRunner(verbosity=2).run(tests)