DM-27694: Add env var expansion to schemeless ButlerURI

mypy.ini

@@ -8,6 +8,9 @@ ignore_missing_imports = True
 [mypy-astropy.*]
 ignore_missing_imports = True
 
+[mypy-erfa.*]
+ignore_missing_imports = True
+
 [mypy-backoff]
 ignore_missing_imports = True
 

python/lsst/daf/butler/core/_butlerUri/schemeless.py

@@ -129,7 +129,8 @@ def _fixupPathUri(parsed: urllib.parse.ParseResult, root: Optional[Union[str, Bu
         to be turned into absolute paths before they can be used.  This is
         always done regardless of the ``forceAbsolute`` parameter.
 
-        Scheme-less paths are normalized.
+        Scheme-less paths are normalized and environment variables are
+        expanded.
         """
         # assume we are not dealing with a directory URI
         dirLike = False
@@ -148,6 +149,10 @@ def _fixupPathUri(parsed: urllib.parse.ParseResult, root: Optional[Union[str, Bu
         # we quoted it in the constructor so unquote here
         expandedPath = os.path.expanduser(urllib.parse.unquote(parsed.path))
 
+        # We might also be receiving a path containing environment variables
+        # so expand those here
+        expandedPath = os.path.expandvars(expandedPath)
+
         # Ensure that this becomes a file URI if it is already absolute
         if os.path.isabs(expandedPath):
             replacements["scheme"] = "file"

python/lsst/daf/butler/core/time_utils.py

@@ -30,6 +30,12 @@
 import astropy.utils.exceptions
 import yaml
 
+# As of astropy 4.2, the erfa interface is shipped independently and
+# ErfaWarning is no longer an AstropyWarning
+try:
+    import erfa
+except ImportError:
+    erfa = None
 
 # These constants can be used by client code.
 # EPOCH is used to construct times as read from database, its precision is
@@ -81,6 +87,8 @@ def astropy_to_nsec(astropy_time: astropy.time.Time) -> int:
     # warnings in case we are dealing with simulated data from the future
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=astropy.utils.exceptions.AstropyWarning)
+        if erfa is not None:
+            warnings.simplefilter("ignore", category=erfa.ErfaWarning)
         value = astropy_time.tai
     # anything before epoch or after MAX_TIME is truncated
     if value < EPOCH:
@@ -146,6 +154,8 @@ def times_equal(time1: astropy.time.Time,
     # to the equality
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=astropy.utils.exceptions.AstropyWarning)
+        if erfa is not None:
+            warnings.simplefilter("ignore", category=erfa.ErfaWarning)
         time1 = time1.tai
         time2 = time2.tai
     delta = (time2.jd1 - time1.jd1) + (time2.jd2 - time1.jd2)

python/lsst/daf/butler/core/timespan.py

@@ -33,6 +33,13 @@
 import sqlalchemy
 import warnings
 
+# As of astropy 4.2, the erfa interface is shipped independently and
+# ErfaWarning is no longer an AstropyWarning
+try:
+    import erfa
+except ImportError:
+    erfa = None
+
 from . import ddl
 from .time_utils import astropy_to_nsec, EPOCH, MAX_TIME, times_equal
 from ._topology import TopologicalExtentDatabaseRepresentation
@@ -68,6 +75,8 @@ def __str__(self) -> str:
         # simulated data in the future
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=astropy.utils.exceptions.AstropyWarning)
+            if erfa is not None:
+                warnings.simplefilter("ignore", category=erfa.ErfaWarning)
             if self.begin is None:
                 head = "(-∞, "
             else:
@@ -422,6 +431,8 @@ def update(cls, timespan: Optional[Timespan], *,
             # can result in warnings for simulated data from the future
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=astropy.utils.exceptions.AstropyWarning)
+                if erfa is not None:
+                    warnings.simplefilter("ignore", category=erfa.ErfaWarning)
                 if timespan.begin is None or timespan.begin < EPOCH:
                     begin = EPOCH
                 else:

python/lsst/daf/butler/registry/queries/exprParser/parserYacc.py

@@ -30,11 +30,20 @@
 #  Imports of standard modules --
 # -------------------------------
 import re
+import warnings
 
 # -----------------------------
 #  Imports for other modules --
 # -----------------------------
 import astropy.time
+
+# As of astropy 4.2, the erfa interface is shipped independently and
+# ErfaWarning is no longer an AstropyWarning
+try:
+    import erfa
+except ImportError:
+    erfa = None
+
 from .exprTree import (BinaryOp, function_call, Identifier, IsIn, NumericLiteral, Parens,
                        RangeLiteral, StringLiteral, TimeLiteral, TupleNode, UnaryOp)
 from .ply import yacc
@@ -122,7 +131,12 @@ def _parseTimeString(time_str):
             scale = "tai"
 
     try:
-        value = astropy.time.Time(value, format=fmt, scale=scale)
+        # Hide warnings about future dates
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=astropy.utils.exceptions.AstropyWarning)
+            if erfa is not None:
+                warnings.simplefilter("ignore", category=erfa.ErfaWarning)
+            value = astropy.time.Time(value, format=fmt, scale=scale)
     except ValueError:
         # astropy makes very verbose exception that is not super-useful in
         # many context, just say we don't like it.

tests/test_timespan.py

@@ -26,6 +26,13 @@
 import astropy.time
 import astropy.utils.exceptions
 
+# As of astropy 4.2, the erfa interface is shipped independently and
+# ErfaWarning is no longer an AstropyWarning
+try:
+    import erfa
+except ImportError:
+    erfa = None
+
 from lsst.daf.butler import Timespan
 
 
@@ -134,6 +141,8 @@ def testFuture(self):
         # so hide these expected warnings from the test output
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=astropy.utils.exceptions.AstropyWarning)
+            if erfa is not None:
+                warnings.simplefilter("ignore", category=erfa.ErfaWarning)
             ts1 = Timespan(begin=astropy.time.Time('2213-06-17 13:34:45.775000', scale='utc', format='iso'),
                            end=astropy.time.Time('2213-06-17 13:35:17.947000', scale='utc', format='iso'))
             ts2 = Timespan(begin=astropy.time.Time('2213-06-17 13:34:45.775000', scale='utc', format='iso'),

tests/test_uri.py

@@ -101,6 +101,19 @@ def testRelative(self):
         child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False)
         self.assertEqual(child.relative_to(parent), "e/f/g.txt")
 
+    def testEnvVar(self):
+        """Test that environment variables are expanded."""
+
+        with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}):
+            uri = ButlerURI("${MY_TEST_DIR}/d.txt")
+        self.assertEqual(uri.path, "/a/b/c/d.txt")
+        self.assertEqual(uri.scheme, "file")
+
+        # This will not expand
+        uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False)
+        self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt")
+        self.assertFalse(uri.scheme)
+
     def testMkdir(self):
         tmpdir = ButlerURI(self.tmpdir)
         newdir = tmpdir.join("newdir/seconddir")