Merge pull request #2301 from ajelenak/ros3-token

Add support for AWS session token

docs/high/file.rst

@@ -96,11 +96,11 @@ of supported drivers and their options:
           Raw data filename extension. Default is '-r.h5'.
 
     'ros3'
-        Allows read-only access to HDF5 files in AWS S3 or S3 compatible object
+        Enables read-only access to HDF5 files in the AWS S3 or S3-compatible object
         stores. HDF5 file name must be one of \http://, \https://, or s3://
         resource location. An s3:// location will be translated into an AWS
         `path-style <https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access>`_
-        location. Keywords:
+        location by h5py. Keywords:
 
         aws_region:
           AWS region of the S3 bucket with the file, e.g. ``b"us-east-1"``.
@@ -112,14 +112,18 @@ of supported drivers and their options:
         secret_key:
           AWS secret access key. Default is ``b''``.
 
-        The argument values must be ``bytes`` objects. All three arguments are
-        required to activate AWS authentication.
+        session_token:
+          AWS temporary session token. Default is ``b''``.' Must be used
+          together with temporary secret_id and secret_key. Available from HDF5 1.14.2.
+
+        The argument values must be ``bytes`` objects. Arguments aws_region,
+        secret_id, and secret_key are required to activate AWS authentication.
 
         .. note::
-           Pre-built h5py packages on PyPI do not include this S3 support. If
+           Pre-built h5py packages on PyPI do not include ros3 driver support. If
            you want this feature, you could use packages from conda-forge, or
            :ref:`build h5py from source <source_install>` against an HDF5 build
-           with S3 support. Alternatively, use the :ref:`file-like object
+           with ros3. Alternatively, use the :ref:`file-like object
            <file_fileobj>` support with a package like s3fs.
 
 

h5py/_hl/files.py

@@ -170,7 +170,15 @@ def make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0, locking,
     except KeyError:
         raise ValueError('Unknown driver type "%s"' % driver)
     else:
-        set_fapl(plist, **kwds)
+        if driver == 'ros3':
+            token = kwds.pop('session_token', None)
+            set_fapl(plist, **kwds)
+            if token:
+                if hdf5_version < (1, 14, 2):
+                    raise ValueError('HDF5 >= 1.14.2 required for AWS session token')
+                plist.set_fapl_ros3_token(token)
+        else:
+            set_fapl(plist, **kwds)
 
     return plist
 

h5py/api_functions.txt

@@ -302,8 +302,10 @@ hdf5:
   herr_t    H5Pget_fapl_family ( hid_t fapl_id, hsize_t *memb_size, hid_t *memb_fapl_id )
   herr_t    H5Pset_family_offset ( hid_t fapl_id, hsize_t offset)
   herr_t    H5Pget_family_offset ( hid_t fapl_id, hsize_t *offset)
-  ROS3 herr_t    H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out);
-  ROS3 herr_t    H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa);
+  ROS3 herr_t    H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out)
+  ROS3 herr_t    H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa)
+  ROS3 1.14.2 herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token)
+  ROS3 1.14.2 herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token)
   DIRECT_VFD herr_t H5Pget_fapl_direct(hid_t fapl_id, size_t *boundary, size_t *block_size, size_t *cbuf_size);
   DIRECT_VFD herr_t H5Pset_fapl_direct(hid_t fapl_id, size_t alignment, size_t block_size, size_t cbuf_size);
   herr_t    H5Pset_fapl_log(hid_t fapl_id, char *logfile, unsigned int flags, size_t buf_size)

h5py/api_types_hdf5.pxd

@@ -344,6 +344,9 @@ cdef extern from "hdf5.h":
       char    secret_key[129]
 
     unsigned int H5FD_CURR_ROS3_FAPL_T_VERSION # version of struct
+
+    IF HDF5_VERSION >= (1, 14, 2):
+      size_t H5FD_ROS3_MAX_SECRET_TOK_LEN
 # === H5G - Groups API ========================================================
 
   ctypedef enum H5G_link_t:

h5py/h5p.pyx

@@ -1115,6 +1115,36 @@ cdef class PropFAID(PropInstanceID):
             H5Pget_fapl_ros3(self.id, &config)
             return config
 
+        IF HDF5_VERSION >= (1, 14, 2):
+            @with_phil
+            def get_fapl_ros3_token(self):
+                """ () => BYTES token
+
+                Get session token from the file access property list.
+                """
+                cdef size_t size = 0
+                cdef char *token = NULL
+
+                size = H5FD_ROS3_MAX_SECRET_TOK_LEN + 1
+                try:
+                    token = <char*>emalloc(size)
+                    token[0] = 0
+                    H5Pget_fapl_ros3_token(self.id, size, token)
+                    pytoken = <bytes>token
+                finally:
+                    efree(token)
+
+                return pytoken
+
+
+            @with_phil
+            def set_fapl_ros3_token(self, char *token=""):
+                """ (BYTES token="")
+
+                Set session token in the file access property list.
+                """
+                H5Pset_fapl_ros3_token(self.id, token)
+
 
     @with_phil
     def set_fapl_log(self, char* logfile, unsigned int flags, size_t buf_size):

h5py/tests/test_ros3.py

@@ -12,16 +12,18 @@
 """
 
 import h5py
+from h5py._hl.files import make_fapl
 import pytest
 
+
 pytestmark = [
     pytest.mark.skipif(
         h5py.version.hdf5_version_tuple < (1, 10, 6) or not h5py.h5.get_config().ros3,
-        reason="ros3 driver not available"),
-    pytest.mark.nonetwork
+        reason="ros3 driver not available")
 ]
 
 
+@pytest.mark.nonetwork
 def test_ros3():
     """ ROS3 driver and options """
 
@@ -41,10 +43,23 @@ def test_ros3_s3_fails():
         h5py.File('foo://wrong/scheme', 'r', driver='ros3')
 
 
+@pytest.mark.nonetwork
 def test_ros3_s3uri():
     """Use S3 URI with ROS3 driver"""
     with h5py.File('s3://dandiarchive/ros3test.hdf5', 'r', driver='ros3',
                    aws_region=b'us-east-2') as f:
         assert f
         assert 'mydataset' in f.keys()
         assert f["mydataset"].shape == (100,)
+
+
+@pytest.mark.skipif(h5py.version.hdf5_version_tuple < (1, 14, 2),
+                    reason='AWS S3 access token support in HDF5 >= 1.14.2')
+def test_ros3_temp_token():
+    """Set and get S3 access token"""
+    token = b'#0123FakeToken4567/8/9'
+    fapl = make_fapl('ros3', libver=None, rdcc_nslots=None, rdcc_nbytes=None,
+                     rdcc_w0=None, locking=None, page_buf_size=None, min_meta_keep=None,
+                     min_raw_keep=None, alignment_threshold=1, alignment_interval=1,
+                     meta_block_size=None, session_token=token)
+    assert token, fapl.get_fapl_ros3_token()

news/ros3-token.rst

@@ -0,0 +1,33 @@
+New features
+------------
+
+* AWS temporary session token can now be used for the ros3 driver
+  (``session_token`` keyword). This token is part of AWS temporary security
+  credentials which also include access and secret access keys. Available from
+  HDF5 1.14.2.
+
+Deprecations
+------------
+
+* <news item>
+
+Exposing HDF5 functions
+-----------------------
+
+* H5Pget_fapl_ros3_token
+* H5Pset_fapl_ros3_token
+
+Bug fixes
+---------
+
+* <news item>
+
+Building h5py
+-------------
+
+* <news item>
+
+Development
+-----------
+
+* <news item>