Merge pull request #184 from lsst/tickets/DM-21015

DM 21015: Fix AWS mock credentials.

python/lsst/daf/butler/core/s3utils.py

@@ -19,7 +19,10 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__all__ = ("s3CheckFileExists", "bucketExists")
+__all__ = ("s3CheckFileExists", "bucketExists", "setAwsEnvCredentials",
+           "unsetAwsEnvCredentials")
+
+import os
 
 try:
     import boto3
@@ -126,3 +129,43 @@ def bucketExists(bucketName, client=None):
         return True
     except s3.exceptions.NoSuchBucket:
         return False
+
+
+def setAwsEnvCredentials(accessKeyId='dummyAccessKeyId', secretAccessKey="dummySecretAccessKey"):
+    """Set AWS credentials environmental variables AWS_ACCESS_KEY_ID and
+    AWS_SECRET_ACCESS_KEY.
+
+    Parameters
+    ----------
+    accessKeyId : `str`
+        Value given to AWS_ACCESS_KEY_ID environmental variable. Defaults to
+        'dummyAccessKeyId'
+    secretAccessKey : `str`
+        Value given to AWS_SECRET_ACCESS_KEY environmental variable. Defaults
+        to 'dummySecretAccessKey'
+
+    Returns
+    -------
+    setEnvCredentials : `bool`
+        True when environmental variables were set, False otherwise.
+
+    Notes
+    -----
+    If either AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are not set, both
+    values are overwritten.
+    """
+    if "AWS_ACCESS_KEY_ID" not in os.environ or "AWS_SECRET_ACCESS_KEY" not in os.environ:
+        os.environ["AWS_ACCESS_KEY_ID"] = accessKeyId
+        os.environ["AWS_SECRET_ACCESS_KEY"] = secretAccessKey
+        return True
+    return False
+
+
+def unsetAwsEnvCredentials():
+    """Unsets AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental
+    variables.
+    """
+    if "AWS_ACCESS_KEY_ID" in os.environ:
+        del os.environ["AWS_ACCESS_KEY_ID"]
+    if "AWS_SECRET_ACCESS_KEY" in os.environ:
+        del os.environ["AWS_SECRET_ACCESS_KEY"]

tests/test_butler.py

@@ -51,7 +51,9 @@ def mock_s3(cls):
 from examplePythonTypes import MetricsExample
 from lsst.daf.butler.core.repoRelocation import BUTLER_ROOT_TAG
 from lsst.daf.butler.core.location import ButlerURI
-from lsst.daf.butler.core.s3utils import s3CheckFileExists
+from lsst.daf.butler.core.s3utils import (s3CheckFileExists, setAwsEnvCredentials,
+                                          unsetAwsEnvCredentials)
+
 
 TESTDIR = os.path.abspath(os.path.dirname(__file__))
 
@@ -619,6 +621,9 @@ def setUp(self):
         uri = ButlerURI(config[".datastore.datastore.root"])
         self.bucketName = uri.netloc
 
+        # set up some fake credentials if they do not exist
+        self.usingDummyCredentials = setAwsEnvCredentials()
+
         if self.useTempRoot:
             self.root = self.genRoot()
         rooturi = f"s3://{self.bucketName}/{self.root}"
@@ -649,6 +654,10 @@ def tearDown(self):
         bucket = s3.Bucket(self.bucketName)
         bucket.delete()
 
+        # unset any potentially set dummy credentials
+        if self.usingDummyCredentials:
+            unsetAwsEnvCredentials()
+
     def checkFileExists(self, root, relpath):
         """Checks if file exists at a given path (relative to root).
 

tests/test_butlerFits.py

@@ -45,6 +45,7 @@ def mock_s3(cls):
 from lsst.daf.butler import DatasetType
 from lsst.daf.butler.core.location import ButlerURI
 from datasetsHelper import FitsCatalogDatasetsHelper, DatasetTestHelper
+from lsst.daf.butler.core.s3utils import setAwsEnvCredentials, unsetAwsEnvCredentials
 
 try:
     import lsst.afw.image
@@ -197,6 +198,9 @@ def setUp(self):
         rooturi = f"s3://{self.bucketName}/{self.root}"
         config.update({"datastore": {"datastore": {"root": rooturi}}})
 
+        # set up some fake credentials if they do not exist
+        self.usingDummyCredentials = setAwsEnvCredentials()
+
         # MOTO needs to know that we expect Bucket bucketname to exist
         # (this used to be the class attribute bucketName)
         s3 = boto3.resource("s3")
@@ -223,6 +227,10 @@ def tearDown(self):
         bucket = s3.Bucket(self.bucketName)
         bucket.delete()
 
+        # unset any potentially set dummy credentials
+        if self.usingDummyCredentials:
+            unsetAwsEnvCredentials()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_s3utils.py

@@ -33,7 +33,8 @@ def mock_s3(cls):
         """
         return cls
 
-from lsst.daf.butler.core.s3utils import bucketExists, s3CheckFileExists
+from lsst.daf.butler.core.s3utils import (bucketExists, s3CheckFileExists,
+                                          setAwsEnvCredentials, unsetAwsEnvCredentials)
 from lsst.daf.butler.core.location import Location, ButlerURI
 
 
@@ -46,6 +47,9 @@ class S3UtilsTestCase(unittest.TestCase):
     fileName = "testFileName"
 
     def setUp(self):
+        # set up some fake credentials if they do not exist
+        self.usingDummyCredentials = setAwsEnvCredentials()
+
         s3 = boto3.client("s3")
         try:
             s3.create_bucket(Bucket=self.bucketName)
@@ -70,6 +74,10 @@ def tearDown(self):
         bucket = s3.Bucket(self.bucketName)
         bucket.delete()
 
+        # unset any potentially set dummy credentials
+        if self.usingDummyCredentials:
+            unsetAwsEnvCredentials()
+
     def testBucketExists(self):
         self.assertTrue(bucketExists(f"{self.bucketName}"))
         self.assertFalse(bucketExists(f"{self.bucketName}_NO_EXIST"))