Merge c000d56 into 6a6154d

.github/workflows/python-package.yml

@@ -16,7 +16,7 @@ jobs:
             fail-fast: true
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.7', '3.8', '3.9', '3.10']
+                python-version: ['3.8', '3.9', '3.10', '3.11']
         steps:
             - name: Checkout code
               uses: actions/checkout@v2
@@ -40,7 +40,7 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
         steps:
             - name: Checkout code
               uses: actions/checkout@v2
@@ -69,7 +69,7 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
         steps:
             - name: Checkout code
               uses: actions/checkout@v2
@@ -93,7 +93,7 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest]
-                python-version: ['3.9']
+                python-version: ['3.10']
         steps:
             - name: Checkout code
               uses: actions/checkout@v2

.gitignore

@@ -47,6 +47,10 @@ nosetests.xml
 .project
 .pydevproject
 
+# VS Code
+.vscode
+.env
+
 # Other
 .*.swp
 *~

.readthedocs.yml

@@ -5,9 +5,16 @@
 # Required
 version: 2
 
+# Set the version of Python and other tools you might need
+build:
+    os: ubuntu-22.04
+    tools:
+        python: "3.10"
+
 # Build documentation in the doc/ directory with Sphinx
 sphinx:
     configuration: doc/conf.py
+    fail_on_warning: true
 
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: all
@@ -16,3 +23,5 @@ formats: all
 python:
     version: 3
     # system_packages: true
+    install:
+        - requirements: doc/rtd-requirements.txt

LICENSE.rst

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2022, Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
+Copyright (c) 2014-2023, Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

README.rst

@@ -45,7 +45,7 @@ Requirements
 HPSSPy assumes that the HPSS utilities `hsi and htar`_ are installed.  You may
 need a NERSC account to download and install these utilities.
 
-.. _`hsi and htar`: https://www.nersc.gov/users/data-and-file-systems/hpss/storing-and-retrieving-data/software-downloads/
+.. _`hsi and htar`: https://docs.nersc.gov/filesystems/archive/#common-commands
 
 License
 -------

doc/conf.py

@@ -119,6 +119,16 @@
 # Include functions that begin with an underscore, e.g. _private().
 napoleon_include_private_with_doc = True
 
+# This value contains a list of modules to be mocked up. This is useful when
+# some external dependencies are not met at build time and break the
+# building process.
+autodoc_mock_imports = []
+for missing in ('pytz', ):
+    try:
+        foo = import_module(missing)
+    except ImportError:
+        autodoc_mock_imports.append(missing)
+
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for

doc/rtd-requirements.txt

@@ -0,0 +1,4 @@
+setuptools>60
+Sphinx>6,<7
+sphinx-rtd-theme>1
+urllib3<2

hpsspy/scan.py

@@ -330,6 +330,7 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
                 Lfile = os.path.join(get_tmpdir(),
                                      os.path.basename(h.replace('.tar',
                                                                 '.txt')))
+                logger.debug(Lfile)
                 htar_dir = None
                 Lfile_lines = ('\n'.join([os.path.basename(f)
                                           for f in missing[h]['files']]) +
@@ -360,7 +361,13 @@ def process_missing(missing_cache, disk_root, hpss_root, dirmode='2770',
                     continue
             logger.debug("os.chdir('%s')", full_chdir)
             os.chdir(full_chdir)
-            h_dir = os.path.join(hpss_root, disk_chdir)
+            #
+            # Avoid adding a trailing slash.
+            #
+            if disk_chdir:
+                h_dir = os.path.join(hpss_root, disk_chdir)
+            else:
+                h_dir = hpss_root
             if h_dir not in created_directories:
                 logger.debug("makedirs('%s', mode='%s')", h_dir, dirmode)
                 if not test:
@@ -491,24 +498,33 @@ def scan_disk(disk_roots, disk_files_cache, overwrite=False):
         with open(disk_files_cache, 'w', newline='') as t:
             writer = csv.writer(t)
             writer.writerow(['Name', 'Size', 'Mtime'])
-            try:
-                for disk_root in disk_roots:
-                    logger.debug("Starting os.walk at %s.", disk_root)
+            for disk_root in disk_roots:
+                logger.debug("Starting os.walk at %s.", disk_root)
+                try:
                     for root, dirs, files in os.walk(disk_root):
                         logger.debug("Scanning disk directory %s.", root)
                         for f in files:
                             fullname = os.path.join(root, f)
                             if not os.path.islink(fullname):
                                 cachename = fullname.replace(disk_root+'/', '')
-                                s = os.stat(fullname)
-                                writer.writerow([cachename,
-                                                 s.st_size,
-                                                 int(s.st_mtime)])
-            except OSError as e:
-                logger.error('Exception encountered while creating ' +
-                             'disk cache file!')
-                logger.error(e.strerror)
-                return False
+                                try:
+                                    s = os.stat(fullname)
+                                except PermissionError as perr:
+                                    logger.error("%s: %s",
+                                                 perr.strerror, perr.filename)
+                                    continue
+                                try:
+                                    writer.writerow([cachename,
+                                                     s.st_size,
+                                                     int(s.st_mtime)])
+                                except UnicodeEncodeError as e:
+                                    logger.error("Could not write %s to cache file due to unusual characters!",
+                                                 fullname.encode(errors='surrogatepass'))
+                                    logger.error("Message was: %s.", str(e))
+                except OSError as oerr:
+                    logger.error('Exception encountered while traversing %s!', disk_root)
+                    logger.error(oerr.strerror)
+                    return False
     return True
 
 
@@ -577,9 +593,25 @@ def physical_disks(release_root, config):
     if ((len(pd) == 1) and (pd[0] == broot)):
         return (release_root,)
     if pd[0].startswith('/'):
-        return tuple([os.path.join(d, os.path.basename(release_root))
-                      for d in pd])
-    return tuple([release_root.replace(broot, d) for d in pd])
+        roots = [os.path.join(d, os.path.basename(release_root))
+                 for d in pd]
+    else:
+        roots = [release_root.replace(broot, d) for d in pd]
+    #
+    # Is any root a pure symlink to another root?
+    #
+    remove = list()
+    for r in roots:
+        if os.path.islink(r):
+            rr = os.readlink(r)
+            if rr.startswith('.'):
+                rr = os.path.normpath(os.path.join(config['root'], rr))
+            if rr in roots:
+                remove.append(r)
+    rs = set(roots)
+    for r in remove:
+        rs.remove(r)
+    return tuple(rs)
 
 
 def _options():

hpsspy/test/t/missing_cache.json

@@ -1,9 +1,36 @@
 {
-    "test_basic_htar.tar": {
-        "files": ["test_basic_htar/test_file1.txt",
-                  "test_basic_htar/test_file2.sha256sum"],
+    "files/test_basic_htar.tar": {
+        "files": ["files/test_basic_htar/test_file1.txt",
+                  "files/test_basic_htar/test_file2.sha256sum"],
         "size": 12345,
         "newer": true,
         "exists": true
+    },
+    "test_basic_files.tar": {
+        "files": ["test_file3.txt",
+                  "test_file4.sha256sum"],
+        "size": 54321,
+        "newer": true,
+        "exists": true
+    },
+    "dir_set/test_dir_set_XX.tar": {
+        "files": ["dir_set/01/test_dir_set_01.txt",
+                  "dir_set/02/test_dir_set_02.txt"],
+        "size": 54321,
+        "newer": true,
+        "exists": true
+    },
+    "big_file/test_basic_file.dump": {
+        "files": ["big_file/test_basic_file.dump"],
+        "size": 76543,
+        "newer": true,
+        "exists": true
+    },
+    "bad_dir/test_basic_htar.tar": {
+        "files": ["bad_dir/test_basic_htar/test_file1.txt",
+                  "bad_dir/test_basic_htar/test_file2.sha256sum"],
+        "size": 33411,
+        "newer": true,
+        "exists": true
     }
 }

hpsspy/test/t/test_scan_disk_cache.csv

@@ -0,0 +1,4 @@
+Name,Size,Mtime
+README.html,50,1552494004
+d1/batch/a.txt,10,1552494014
+d1/batch/b.txt,20,1552494094

hpsspy/test/test_os.py

@@ -44,9 +44,14 @@ def __init__(self, return_values, raises=None):
         def __call__(self, *args, **kwargs):
             self.args.append(tuple(args))
             self.kwargs.append(kwargs)
-            if self.raises:
-                raise self.raises
             r = self.return_values[self.counter]
+            if isinstance(self.raises, list):
+                if self.raises[self.counter] is not None:
+                    raise self.raises[self.counter]
+            elif self.raises is not None:
+                raise self.raises
+            else:
+                pass
             self.counter += 1
             return r