Merge b18c2fe into f18bf1b

.travis.yml

@@ -61,7 +61,7 @@ matrix:
         # Coverage test, pass the results to coveralls.
         - os: linux
           python: 3.5
-          env: MAIN_CMD='coverage' SETUP_CMD='run hpsspy/test/hpsspy_test_suite.py'
+          env: MAIN_CMD='coverage' SETUP_CMD='run setup.py test'
 
         # PEP 8 compliance.
         - os: linux

doc/changes.rst

@@ -5,7 +5,9 @@ Release Notes
 0.4.1 (unreleased)
 ------------------
 
-* No changes yet.
+* Handle directory names that contain underscore characters (PR `#4`_).
+
+.. _`#4`: https://github.com/weaverba137/hpsspy/pull/4
 
 0.4.0 (2017-08-10)
 ------------------

doc/configuration.rst

@@ -160,10 +160,12 @@ imposes some additional requirements, conventions and idioms:
     For example ``batch.tar`` means "archive a batch/ directory".
     For longer file names, the "suffix" of the file will be used.
     ``data_d1_batch.tar`` also means "archive a batch/ directory", because
-    ``data_d1_`` is stripped off.
+    ``data_d1_`` is stripped off.  The directory name will be verified, so
+    if the directory to back up is actually ``d1_batch/``, ``batch/`` will be
+    searched for, then ``d1_batch/``.
   - An archive filename that ends with ``_files.tar``, *e.g.* ``foo/bar_files.tar``
     is a signal to :command:`missing_from_hpss` to construct
-    the archive file in a certain way, not by decending into a directory,
+    the archive file in a certain way, not by descending into a directory,
     but by constructing an explicit list of files and building an archive
     file out of that.
 
@@ -175,15 +177,16 @@ imposes some additional requirements, conventions and idioms:
     ``"foo/(bar|baz|flub)/.*$" : "foo/foo_\\1.tar"``.  The name of the
     directory matched in parentheses will be substituted into the file name.
   - Archive arbitrary subdirectories of a *set* of subdirectories:
-    ``"d1/foo/(ab|bc|cd|de|ef)/([^/]+)/.*$":"d1/foo/\\1/d1_foo_\\1_\\2.tar"``
+    ``"d1/foo/(ab|bc|cd|de|ef)/([^/]+)/.*$" : "d1/foo/\\1/d1_foo_\\1_\\2.tar"``
   - Match files in a directory, but not any files in any
     subdirectory: ``"foo/[^/]+$" : "foo_files.tar"``.  See also the
     ``_files.tar`` convention mentioned above.
   - Group some but not all subdirectories in a directory into a single
     archive file for efficiency: ``"foo/([0-9])([0-9][0-9])/.*$" : "foo/foo_\\1XX.tar"``.
     Note the ending of the archive file, and that the directories have to
     have a very uniform naming convention (three and only three digits
-    in this example).
+    in this example).  Also, the placeholder ``X`` needs to be at the *end* of
+    the file name.
   - Do not create an archive file, just copy the file, as is, to HPSS:
     ``"d1/README\\.txt$" : "d1/README.txt"``.  Similarly, for a set of TXT files:
     ``"d1/([^/]+\\.txt)$" : "d1/\\1"``.

hpsspy/scan.py

@@ -311,12 +311,26 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
                         fp.write(Lfile_lines)
             else:
                 Lfile = None
-                htar_dir = [basename(h).split('_')[-1].split('.')[0]]
-                if 'X' in htar_dir[0]:
-                    htar_re = re.compile(htar_dir[0].replace('X', '.') + '$')
-                    htar_dir = [d for d in listdir(full_chdir)
-                                if isdir(join(full_chdir, d)) and
-                                htar_re.match(d) is not None]
+                #
+                # Be careful, because the directory name may itself
+                # contain underscore characters, or X characters.
+                #
+                htar_base = basename(h).rsplit('.', 1)[0]  # remove .tar
+                htar_dir = []
+                for b in iterrsplit(htar_base, '_'):
+                    if b.endswith('X'):
+                        htar_re = re.compile(b.replace('X', '.') + '$')
+                        htar_dir = [d for d in listdir(full_chdir)
+                                    if isdir(join(full_chdir, d)) and
+                                    htar_re.match(d) is not None]
+                    else:
+                        if isdir(join(full_chdir, b)):
+                            htar_dir = [b]
+                    if len(htar_dir) > 0:
+                        break
+                if len(htar_dir) == 0:
+                    logger.error("Could not find directories corresponding to %s!", h)
+                    continue
             logger.debug("chdir('%s')", full_chdir)
             chdir(full_chdir)
             h_dir = join(hpss_root, disk_chdir)
@@ -368,6 +382,29 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
     return
 
 
+def iterrsplit(s, c):
+    """Split string `s` on `c` and rejoin on `c` from the end of `s`.
+
+    Parameters
+    ----------
+    s : :class:`str`
+        String to split
+    c : :class:`str`
+        Split on this string.
+
+    Returns
+    -------
+    :class:`str`
+        Iteratively return the joined parts of `s`.
+    """
+    ss = s.split(c)
+    i = -1
+    while abs(i) <= len(ss):
+        yield c.join(ss[i:])
+        i -= 1
+    return
+
+
 def scan_disk(disk_roots, disk_files_cache, clobber=False):
     """Scan a directory tree on disk and cache the files found there.
 

hpsspy/test/__init__.py

@@ -1,3 +1,33 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
+"""
+hpsspy.test
+~~~~~~~~~~~
+
+Used to initialize the unit test framework via ``python setup.py test``.
+"""
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+# The line above will help with 2to3 support.
+import unittest
+
+
+def hpsspy_test_suite():
+    """Returns unittest.TestSuite of hpsspy tests.
+
+    This is factored out separately from runtests() so that it can be used by
+    ``python setup.py test``.
+    """
+    from os.path import dirname
+    py_dir = dirname(dirname(__file__))
+    return unittest.defaultTestLoader.discover(py_dir,
+                                               top_level_dir=dirname(py_dir))
+
+
+def runtests():
+    """Run all tests in hpsspy.test.test_*.
+    """
+    # Load all TestCase classes from hpsspy/test/test_*.py
+    tests = hpsspy_test_suite()
+    # Run them
+    unittest.TextTestRunner(verbosity=2).run(tests)

hpsspy/test/test_scan.py

@@ -20,7 +20,7 @@
 from logging.handlers import MemoryHandler
 from pkg_resources import resource_filename, resource_stream
 from ..scan import (compile_map, files_to_hpss, physical_disks,
-                    validate_configuration)
+                    validate_configuration, iterrsplit)
 
 
 class TestHandler(MemoryHandler):
@@ -85,6 +85,13 @@ def assertLog(self, index=-1, message=''):
         self.assertEqual(logger.handlers[0].buffer[index].getMessage(),
                          message)
 
+    def test_iterrsplit(self):
+        """Test reverse re-joining a string.
+        """
+        results = ['d', 'c_d', 'b_c_d', 'a_b_c_d']
+        for i, s in enumerate(iterrsplit('a_b_c_d', '_')):
+            self.assertEqual(s, results[i])
+
     def test_compile_map(self):
         """Test compiling regular expressions in the JSON configuration file.
         """

hpsspy/test/test_util.py

@@ -20,6 +20,11 @@
 from .. import HpssOSError
 from ..util import HpssFile, get_hpss_dir, get_tmpdir, hsi, htar
 
+mock_available = True
+try:
+    from unittest.mock import patch, MagicMock
+except:
+    mock_available = False
 
 class MockHpss(unittest.TestCase):
     """Provide access to mock HPSS commands.
@@ -127,6 +132,58 @@ def test_HpssFile(self):
                 else:
                     self.assertIsNone(f.htar_contents())
             self.assertEqual(f.st_mtime, int(mtimes[i].strftime('%s')))
+        #
+        # Test funky modes.
+        #
+        f = HpssFile(lspath, 's', 'rw-rw----', 1, 'bweaver', 'bweaver',
+                     1000, 'Feb', 2, '2016', 'fake.socket')
+        with self.assertRaises(AttributeError) as err:
+            m = f.st_mode
+        self.assertEqual(str(err.exception),
+                         "Unknown file type, s, for fake.socket!")
+
+    @unittest.skipUnless(mock_available, "Skipping test that requires unittest.mock.")
+    def test_HpssFile_isdir(self):
+        """Test the isdir property on symbolic links.
+        """
+        lspath = '/home/b/bweaver'
+        with patch('hpsspy.os.stat') as s:
+            m = MagicMock()
+            m.isdir = True
+            s.return_value = m
+            f = HpssFile(lspath, 'l', 'rwxrwxrwx', 1, 'bweaver', 'bweaver',
+                         21, 'Aug', 22, '2014', 'cosmo@ -> /nersc/projects/cosmo')
+            self.assertTrue(f.islink)
+            self.assertTrue(f.isdir)
+            s.assert_called_with('/nersc/projects/cosmo')
+        with patch('hpsspy.os.stat') as s:
+            m = MagicMock()
+            m.isdir = False
+            s.return_value = m
+            f = HpssFile(lspath, 'l', 'rwxrwxrwx', 1, 'bweaver', 'bweaver',
+                         21, 'Aug', 22, '2014', 'cosmo@ -> cosmo.txt')
+            self.assertTrue(f.islink)
+            self.assertFalse(f.isdir)
+            s.assert_called_with('/home/b/bweaver/cosmo.txt')
+
+    @unittest.skipUnless(mock_available, "Skipping test that requires unittest.mock.")
+    def test_HpssFile_htar_contents(self):
+        """Test retrieval of htar file contents.
+        """
+        lspath = '/home/b/bweaver'
+        f = HpssFile(lspath, '-', 'rw-rw-r--', 1, 'bweaver', 'bweaver',
+                     12345, 'Aug', 22, '2014', 'bundle.tar')
+        self.assertIsNone(f.htar_contents())
+        f.ishtar = True
+        f._contents = ['foo.txt']
+        self.assertListEqual(f.htar_contents(), ['foo.txt'])
+        f._contents = None
+        with patch('hpsspy.util.htar') as h:
+            h.return_value = ('HTAR: -rw-rw-r-- bweaver/bweaver 100 2012-07-03 12:00 foo.txt\nHTAR: -rw-rw-r-- bweaver/bweaver 100 2012-07-03 12:00 bar.txt', '')
+            self.assertListEqual(f.htar_contents(),
+                                 [('-', 'rw-rw-r--', 'bweaver', 'bweaver', '100', '2012', '07', '03', '12:00', 'foo.txt'),
+                                  ('-', 'rw-rw-r--', 'bweaver', 'bweaver', '100', '2012', '07', '03', '12:00', 'bar.txt')])
+            h.assert_called_with('-t', '-f', '/home/b/bweaver/bundle.tar')
 
     def test_get_hpss_dir(self):
         """Test searching for the HPSS_DIR variable.
@@ -162,12 +219,8 @@ def test_htar(self):
         """
         command = ['-cvf', 'foo/bar.tar', '-H', 'crc:verify=all', 'bar']
         out, err = htar(*command)
-        if self.PY3:
-            self.assertEqual(out.decode('utf8').strip(), ' '.join(command))
-            self.assertEqual(err.decode('utf8').strip(), '')
-        else:
-            self.assertEqual(out.strip(), ' '.join(command))
-            self.assertEqual(err.strip(), '')
+        self.assertEqual(out.strip(), ' '.join(command))
+        self.assertEqual(err.strip(), '')
 
 
 def test_suite():

hpsspy/util.py

@@ -146,7 +146,7 @@ def st_mode(self):
             try:
                 mode = self._file_modes[self.raw_type]
             except KeyError:
-                raise
+                raise AttributeError("Unknown file type, {0.raw_type}, for {0.name}!".format(self))
             if self.raw_permission[0] == 'r':
                 mode |= stat.S_IRUSR
             if self.raw_permission[1] == 'w':
@@ -341,4 +341,4 @@ def htar(*args):
     err = errfile.read()
     outfile.close()
     errfile.close()
-    return (out, err)
+    return (out.decode('utf8'), err.decode('utf8'))

setup.py

@@ -76,7 +76,7 @@
 #
 # Test suite
 #
-setup_keywords['test_suite'] = 'hpsspy.test.hpsspy_test_suite.hpsspy_test_suite'
+setup_keywords['test_suite'] = 'hpsspy.test.hpsspy_test_suite'
 #
 # Run setup command.
 #