Merge pull request #54 from HDFGroup/props

Props

.gitmodules

@@ -1,3 +1,5 @@
 [submodule "hdf5-json"]
 	path = hdf5-json
 	url = https://github.com/HDFGroup/hdf5-json.git
+[submodules "hdf5-json"]
+	branch = stable

docs/DatasetOps/GET_Dataset.rst

@@ -56,6 +56,12 @@ shape
 A JSON object representing the shape of the dataset.  See :doc:`GET_DatasetShape` for
 details of the shape representation.
 
+creationProperties
+^^^^^^^^^^^^^^^^^^
+A JSON object that describes chunk layout, filters, fill value, and other aspects of the dataset.
+See: http://hdf5-json.readthedocs.org/en/latest/bnf/dataset.html#grammar-token-dcpl for a complete 
+description of fields that can be used.
+
 attributeCount
 ^^^^^^^^^^^^^^
 The number of attributes belonging to the dataset.
@@ -117,6 +123,13 @@ Sample Response
         "base": "H5T_IEEE_F32BE", 
         "class": "H5T_FLOAT"
     },
+    "creationProperties": {
+        "allocTime": "H5D_ALLOC_TIME_LATE",
+        "fillTime": "H5D_FILL_TIME_IFSET",
+        "layout": {
+            "class": "H5D_CONTIGUOUS"
+        }
+    },
     "attributeCount": 0,  
     "created": "2015-01-23T06:12:18Z", 
     "lastModified": "2015-01-23T06:12:18Z",     

docs/DatasetOps/POST_Dataset.rst

@@ -52,6 +52,14 @@ An integer array describing the maximum extent of each dimension (or 0 for unlim
 dimensions).  If maxdims is not provided that resulting dataset will be non-extensible.
 Not valid to include if ``H5S_NULL`` is specified for the shape.
 
+creationProperties
+^^^^^^^^^^^^^^^^^^
+A JSON object that can specify chunk layout, filters, fill value, and other aspects of the dataset.
+See: http://hdf5-json.readthedocs.org/en/latest/bnf/dataset.html#grammar-token-dcpl for a complete 
+description of fields that can be used.
+
+If creationProperties is not provided, default values will be used
+
 link["id"]
 ^^^^^^^^^^
 The UUID of the group the new group should be linked to.  If the UUID is not valid,
@@ -308,6 +316,86 @@ Sample Response - Committed Type
       ]
     }
     
+Sample Request - SZIP Compression with chunking
+-----------------------------------------------
+
+.. code-block:: http
+
+    POST /datasets HTTP/1.1
+    Content-Length: 67
+    User-Agent: python-requests/2.3.0 CPython/2.7.8 Darwin/14.0.0
+    host: szip.datasettest.test.hdfgroup.org
+    Accept: */*
+    Accept-Encoding: gzip, deflate
+    
+.. code-block:: json
+
+    {
+    "creationProperties": {
+        "filters": [
+            {
+                "bitsPerPixel": 8,
+                "coding": "H5_SZIP_EC_OPTION_MASK",
+                "id": 4,
+                "pixelsPerBlock": 32,
+                "pixelsPerScanline": 100
+            }
+        ],
+        "layout": {
+            "class": "H5D_CHUNKED",
+            "dims": [
+                100,
+                100
+            ]
+        }
+    },
+    "shape": [
+        1000,
+        1000
+    ],
+    "type": "H5T_IEEE_F32LE"
+   }
+   
+Sample Response - SZIP Compression with chunking
+------------------------------------------------
+
+.. code-block:: http
+
+    HTTP/1.1 201 Created
+    Date: Thu, 18 Jun 2015 08:41:53 GMT
+    Content-Length: 975
+    Content-Type: application/json
+    Server: TornadoServer/3.2.2
+    
+.. code-block:: json
+
+{
+    "id": "ad283c05-158c-11e5-bd67-3c15c2da029e",
+    "attributeCount": 0,
+    "created": "2015-06-18T07:36:04Z",
+    "lastModified": "2015-06-18T07:36:04Z",
+    "hrefs": [
+        {
+            "href": "http://newdset_szip.datasettest.test.hdfgroup.org/datasets/ad283c05-158c-11e5-bd67-3c15c2da029e",
+            "rel": "self"
+        },
+        {
+            "href": "http://newdset_szip.datasettest.test.hdfgroup.org/groups/ad2746d4-158c-11e5-a083-3c15c2da029e",
+            "rel": "root"
+        },
+        {
+            "href": "http://newdset_szip.datasettest.test.hdfgroup.org/datasets/ad283c05-158c-11e5-bd67-3c15c2da029e/attributes",
+            "rel": "attributes"
+        },
+        {
+            "href": "http://newdset_szip.datasettest.test.hdfgroup.org/datasets/ad283c05-158c-11e5-bd67-3c15c2da029e/value",
+            "rel": "value"
+        }
+    ]
+}
+
+
+
 Related Resources
 =================
 

docs/DatasetOps/index.rst

@@ -3,8 +3,8 @@ Datasets
 ######################
 
 Datasets are objects that a composed of a homogenous collection of data elements.   Each
-dataset has a *type* that specifies the structure of the individual elements: float, string,
-compound, etc.), and a *shape* that specifies that layout of the data elements (scalar, 
+dataset has a *type* that specifies the structure of the individual elements (float, string,
+compound, etc.), and a *shape* that specifies the layout of the data elements (scalar, 
 one-dimensional, multi-dimensional).  In addition meta-data can be attached to a dataset
 in the form of attributes.  See: :doc:`../AttrOps/index`.
 
@@ -13,12 +13,13 @@ Creating Datasets
 
 Use the :doc:`POST_Dataset` operation to create new datasets.  As part of the POST
 request, JSON descriptions for the type and shape of the dataset are included with the
-request.  
+request.  Optionally, creation properties can be used to specify the chunk layout (how
+the data elements are stored in the server) and compression filter (e.g. GZIP, LZF, SZIP).
 
 Getting information about a dataset
 -----------------------------------
 Use the :doc:`GET_Dataset` operation to retrieve information about a datasets type,
-shape, and number of attributes.  To list all the datasets within a domain use 
+shape, creation properties, and number of attributes.  To list all the datasets within a domain use 
 :doc:`GET_Datasets`.  To list the datasets linked to a particular group use 
 :doc:`../GroupOps/GET_Links` and look at links with a "collection" key of "datsets".
 
@@ -32,8 +33,14 @@ data elements, a point selection (series of element coordinates) can be passed t
 
 Reading data from a dataset
 ---------------------------
-To read either the entire dataset, hyperslab selection, or 'where' query, use the :doc:`GET_Value`
-operation.  To read a point selection, use the :doc:`POST_Value` operation  (POST is 
+To read either the entire dataset, or a specified selection, use the :doc:`GET_Value`
+operation.  Without any request parameters, the GET operation retuns all data values.  
+To read a specific hyperslab, use the select parameter to start and end indexes of the hyperslab
+(the selection can also include a step value to include a regular subset of the hyperslab).
+Finally, for one-dimensional datasets with compound types, a *where* parameter can be used to 
+select elements meeting a specified condition. 
+
+To read a specific list of elements (by index values), use the :doc:`POST_Value` operation  (POST is 
 used in this case rather than GET since the point selection values may be to 
 large to include in the URI.) 
 

server/app.py

@@ -626,8 +626,6 @@ def get(self):
         hrefs.append({'rel': 'owner', 'href': href + 'datasets/' + reqUuid })
         hrefs.append({'rel': 'root',  'href': href + 'groups/' + rootUUID})   
         shape = item['shape']
-        if 'fillvalue' in item:
-            shape['fillvalue'] = item['fillvalue']
         response['shape'] = shape
         response['created'] = unixTimeToUTC(item['ctime'])
         response['lastModified'] = unixTimeToUTC(item['mtime'])
@@ -771,15 +769,19 @@ def get(self):
         typeItem = item['type']
         response['type'] = hdf5dtype.getTypeResponse(typeItem)
         response['shape'] = item['shape']
-        if 'fillvalue' in item:
-            response['fillvalue'] = item['fillvalue']
+
+        if 'creationProperties' in item:
+            response['creationProperties'] = item['creationProperties']
         response['created'] = unixTimeToUTC(item['ctime'])
         response['lastModified'] = unixTimeToUTC(item['mtime'])
         response['attributeCount'] = item['attributeCount']
         response['hrefs'] = hrefs
 
         self.set_header('Content-Type', 'application/json')
-        self.write(json_encode(response))
+
+        json_rsp = json_encode(response)
+
+        self.write(json_rsp)
 
 
     def delete(self): 
@@ -2009,11 +2011,14 @@ def post(self):
                 if maxextent == 0:
                     maxdims[i] = None  # this indicates unlimited
 
+        creationProps = None
+        if "creationProperties" in body:
+            creationProps = body["creationProperties"]
         item = None
         try:
             with Hdf5db(filePath, app_logger=log) as db:
                 rootUUID = db.getUUIDByPath('/')
-                item = db.createDataset(datatype, dims, maxdims)
+                item = db.createDataset(datatype, dims, maxdims, creation_props=creationProps)
                 if group_uuid:
                     # link the new dataset
                     db.linkObject(group_uuid, item['id'], link_name)
@@ -2285,7 +2290,7 @@ def delete(self):
             raise HTTPError(500, "Unexpected error: unable to delete collection") 
 
 class InfoHandler(RequestHandler):
-      
+
     def get(self):
         log = logging.getLogger("h5serv")
         log.info('InfoHandler.get ' + self.request.host)