Skip to content

Commit

Permalink
update string type keys to match spec
Browse files Browse the repository at this point in the history
  • Loading branch information
@jreadey
jreadey committed Mar 18, 2015
1 parent 309e395 commit b54255a
Show file tree
Hide file tree
Showing 39 changed files with 425 additions and 378 deletions.
14 changes: 7 additions & 7 deletions docs/Types/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ JSON representation of a dataset using a fixed width string of 40 characters:
},
"type": {
"class": "H5T_STRING",
"cset": "H5T_CSET_ASCII",
"strpad": "H5T_STR_NULLPAD",
"strsize": 40
"charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLPAD",
"length": 40
},
"value": "Hello, World!"
}
Expand All @@ -107,7 +107,7 @@ or there is a great deal of variability in the lengths of strings.
string elements of an array in the server.

To specify a variable length string, create a JSON object with class, cset, strpad,
and strsize keys (see definitions of these keys below) where the value of "strsize" is:
and strsize keys (see definitions of these keys below) where the value of "length" is:
``H5T_VARIABLE``.

*Note:* Current only the ASCII character set is supported.
Expand All @@ -129,10 +129,10 @@ JSON representation of a attribute using a variable length string:
},
"type": {
"class": "H5T_STRING",
"cset": "H5T_CSET_ASCII",
"charSet": "H5T_CSET_ASCII",
"order": "H5T_ORDER_NONE",
"strpad": "H5T_STR_NULLTERM",
"strsize": "H5T_VARIABLE"
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE"
},
"value": [
"Hypermedia",
Expand Down
38 changes: 19 additions & 19 deletions server/hdf5dtype.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,15 +107,15 @@ def getTypeElement(dt):
type_info['base_size'] = 8 # machine pointer size
if h5t_check == str:
type_info['class'] = 'H5T_STRING'
type_info['strsize'] = 'H5T_VARIABLE'
type_info['cset'] = 'H5T_CSET_ASCII'
type_info['strpad'] = 'H5T_STR_NULLTERM'
type_info['length'] = 'H5T_VARIABLE'
type_info['charSet'] = 'H5T_CSET_ASCII'
type_info['strPad'] = 'H5T_STR_NULLTERM'
type_info['order'] = 'H5T_ORDER_NONE'
elif h5t_check == unicode:
type_info['class'] = 'H5T_STRING'
type_info['strsize'] = 'H5T_VARIABLE'
type_info['cset'] = 'H5T_CSET_UTF8'
type_info['strpad'] = 'H5T_STR_NULLTERM'
type_info['length'] = 'H5T_VARIABLE'
type_info['charSet'] = 'H5T_CSET_UTF8'
type_info['strPad'] = 'H5T_STR_NULLTERM'
type_info['order'] = 'H5T_ORDER_NONE'
elif type(h5t_check) == np.dtype:
# vlen data
Expand Down Expand Up @@ -209,9 +209,9 @@ def getBaseType(dt):
if dt.base.kind == 'S':
# Fixed length string type
type_info['class'] = 'H5T_STRING'
type_info['cset'] = 'H5T_CSET_ASCII'
type_info['strsize'] = dt.base.itemsize
type_info['strpad'] = 'H5T_STR_NULLPAD'
type_info['charSet'] = 'H5T_CSET_ASCII'
type_info['length'] = dt.base.itemsize
type_info['strPad'] = 'H5T_STR_NULLPAD'
type_info['order'] = 'H5T_ORDER_NONE'
elif dt.base.kind == 'V':
type_info['class'] = 'H5T_OPAQUE'
Expand Down Expand Up @@ -325,24 +325,24 @@ def createBaseDataType(typeItem):
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT')
dtRet = np.dtype(shape + baseType)
elif typeClass == 'H5T_STRING':
if 'strsize' not in typeItem:
raise KeyError("'strsize' not provided")
if 'cset' not in typeItem:
raise KeyError("'cset' not provided")
if 'length' not in typeItem:
raise KeyError("'length' not provided")
if 'charSet' not in typeItem:
raise KeyError("'charSet' not provided")

if typeItem['strsize'] == 'H5T_VARIABLE':
if typeItem['length'] == 'H5T_VARIABLE':
if shape:
raise TypeError("ArrayType is not supported for variable len types")
if typeItem['cset'] == 'H5T_CSET_ASCII':
if typeItem['charSet'] == 'H5T_CSET_ASCII':
dtRet = special_dtype(vlen=str)
elif typeItem['cset'] == 'H5T_CSET_UTF8':
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
dtRet = special_dtype(vlen=unicode)
else:
raise TypeError("unexpected 'cset' value")
raise TypeError("unexpected 'charSet' value")
else:
nStrSize = typeItem['strsize']
nStrSize = typeItem['length']
if type(nStrSize) != int:
raise TypeError("expecting integer value for 'strsize'")
raise TypeError("expecting integer value for 'length'")
dtRet = np.dtype(shape + 'S' + str(nStrSize)) # fixed size ascii string
elif typeClass == 'H5T_VLEN':
if shape:
Expand Down
36 changes: 18 additions & 18 deletions test/integ/attributetest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,10 @@ def testGetCompound(self):
self.assertEqual(field3['name'], 'wind')
field3Type = field3['type']
self.assertEqual(field3Type['class'], 'H5T_STRING')
self.assertEqual(field3Type['cset'], 'H5T_CSET_ASCII')
self.assertEqual(field3Type['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(field3Type['order'], 'H5T_ORDER_NONE')
self.assertEqual(field3Type['strsize'], 6)
self.assertEqual(field3Type['strpad'], 'H5T_STR_NULLPAD')
self.assertEqual(field3Type['length'], 6)
self.assertEqual(field3Type['strPad'], 'H5T_STR_NULLPAD')

def testGetCommitted(self):
domain = 'committed_type.' + config.get('domain')
Expand Down Expand Up @@ -222,10 +222,10 @@ def testGetVLenString(self):
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.assertEqual(typeItem['strsize'], 'H5T_VARIABLE')
self.assertEqual(typeItem['strpad'], 'H5T_STR_NULLTERM')
self.assertEqual(typeItem['length'], 'H5T_VARIABLE')
self.assertEqual(typeItem['strPad'], 'H5T_STR_NULLTERM')
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
Expand All @@ -251,10 +251,10 @@ def testGetFixedString(self):
typeItem = rspJson['type']

self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.assertEqual(typeItem['strsize'], 7)
self.assertEqual(typeItem['strpad'], 'H5T_STR_NULLPAD')
self.assertEqual(typeItem['length'], 7)
self.assertEqual(typeItem['strPad'], 'H5T_STR_NULLPAD')
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
Expand Down Expand Up @@ -447,10 +447,10 @@ def testGetScalarString(self):
self.assertTrue('dims' not in shape)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.assertEqual(typeItem['strsize'], 'H5T_VARIABLE')
self.assertEqual(typeItem['strpad'], 'H5T_STR_NULLTERM')
self.assertEqual(typeItem['length'], 'H5T_VARIABLE')
self.assertEqual(typeItem['strPad'], 'H5T_STR_NULLTERM')
data = rspJson['value']
self.assertEqual(data, "hello")

Expand Down Expand Up @@ -584,10 +584,10 @@ def testPutFixedString(self):
rootUUID = helper.getRootUUID(domain)
headers = {'host': domain}
data = "Hello, I'm a fixed-width string!"
str_type = { 'cset': 'H5T_CSET_ASCII',
str_type = { 'charSet': 'H5T_CSET_ASCII',
'class': 'H5T_STRING',
'strpad': 'H5T_STR_NULLPAD',
'strsize': 40}
'strPad': 'H5T_STR_NULLPAD',
'length': 40}

payload = {'type': str_type, 'shape': (1,), 'value': data}
req = self.endpoint + "/groups/" + rootUUID + "/attributes/" + attr_name
Expand All @@ -602,10 +602,10 @@ def testPutVariableString(self):
rootUUID = helper.getRootUUID(domain)
headers = {'host': domain}
data = ["Hypermedia", "as", "the", "engine", "of", "state."]
str_type = { 'cset': 'H5T_CSET_ASCII',
str_type = { 'charSet': 'H5T_CSET_ASCII',
'class': 'H5T_STRING',
'strpad': 'H5T_STR_NULLPAD',
'strsize': 'H5T_VARIABLE'}
'strPad': 'H5T_STR_NULLPAD',
'length': 'H5T_VARIABLE'}

payload = {'type': str_type, 'shape': (6,), 'value': data}
req = self.endpoint + "/groups/" + rootUUID + "/attributes/" + attr_name
Expand Down
36 changes: 18 additions & 18 deletions test/integ/datasettest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,10 @@ def testGetCompound(self):
self.assertTrue('type' in timeField)
timeFieldType = timeField['type']
self.assertEqual(timeFieldType['class'], 'H5T_STRING')
self.assertEqual(timeFieldType['cset'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['order'], 'H5T_ORDER_NONE')
self.assertEqual(timeFieldType['strsize'], 6)
self.assertEqual(timeFieldType['strpad'], 'H5T_STR_NULLPAD')
self.assertEqual(timeFieldType['length'], 6)
self.assertEqual(timeFieldType['strPad'], 'H5T_STR_NULLPAD')
tempField = fields[2]
self.assertEqual(tempField['name'], 'temp')
tempFieldType = tempField['type']
Expand Down Expand Up @@ -253,10 +253,10 @@ def testGetCompoundCommitted(self):
self.assertTrue('type' in timeField)
timeFieldType = timeField['type']
self.assertEqual(timeFieldType['class'], 'H5T_STRING')
self.assertEqual(timeFieldType['cset'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['order'], 'H5T_ORDER_NONE')
self.assertEqual(timeFieldType['strsize'], 6)
self.assertEqual(timeFieldType['strpad'], 'H5T_STR_NULLPAD')
self.assertEqual(timeFieldType['length'], 6)
self.assertEqual(timeFieldType['strPad'], 'H5T_STR_NULLPAD')
tempField = fields[2]
self.assertEqual(tempField['name'], 'temp')
tempFieldType = tempField['type']
Expand Down Expand Up @@ -311,11 +311,11 @@ def testGetCompoundArray(self):
strFieldTypeBase = strFieldType['base']

self.assertEqual(strFieldTypeBase['class'], 'H5T_STRING')
self.assertEqual(strFieldTypeBase['cset'], 'H5T_CSET_ASCII')
self.assertEqual(strFieldTypeBase['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(strFieldTypeBase['order'], 'H5T_ORDER_NONE')
self.assertEqual(strFieldTypeBase['strsize'], 32)
self.assertEqual(strFieldTypeBase['length'], 32)
# todo - fix, cf https://github.com/HDFGroup/h5serv/issues/20
#self.assertEqual(strFieldTypeBase['strpad'], 'H5T_STR_SPACEPAD')
#self.assertEqual(strFieldTypeBase['strPad'], 'H5T_STR_SPACEPAD')

def testGetCommitted(self):
domain = 'committed_type.' + config.get('domain')
Expand Down Expand Up @@ -379,10 +379,10 @@ def testGetFixedString(self):
typeItem = rspJson['type']

self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.assertEqual(typeItem['strsize'], 7)
self.assertEqual(typeItem['strpad'], 'H5T_STR_NULLPAD')
self.assertEqual(typeItem['length'], 7)
self.assertEqual(typeItem['strPad'], 'H5T_STR_NULLPAD')

def testGetEnum(self):
domain = 'enum_dset.' + config.get('domain')
Expand Down Expand Up @@ -537,10 +537,10 @@ def testPostScalar(self):
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
str_type = { 'cset': 'H5T_CSET_ASCII',
str_type = { 'charSet': 'H5T_CSET_ASCII',
'class': 'H5T_STRING',
'strpad': 'H5T_STR_NULLPAD',
'strsize': 40}
'strPad': 'H5T_STR_NULLPAD',
'length': 40}
payload = {'type': str_type}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
Expand Down Expand Up @@ -881,11 +881,11 @@ def testPostWithLink(self):
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)

type_vstr = {"cset": "H5T_CSET_ASCII",
type_vstr = {"charSet": "H5T_CSET_ASCII",
"order": "H5T_ORDER_NONE",
"class": "H5T_STRING",
"strpad": "H5T_STR_NULLTERM",
"strsize": "H5T_VARIABLE" }
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE" }
payload = {'type': type_vstr, 'shape': 10,
'link': {'id': root_uuid, 'name': 'linked_dset'} }
req = self.endpoint + "/datasets"
Expand Down
6 changes: 3 additions & 3 deletions test/integ/datasettypetest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,10 @@ def testGetCompound(self):
self.assertTrue('type' in timeField)
timeFieldType = timeField['type']
self.assertEqual(timeFieldType['class'], 'H5T_STRING')
self.assertEqual(timeFieldType['cset'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['charSet'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['order'], 'H5T_ORDER_NONE')
self.assertEqual(timeFieldType['strsize'], 6)
self.assertEqual(timeFieldType['strpad'], 'H5T_STR_NULLPAD')
self.assertEqual(timeFieldType['length'], 6)
self.assertEqual(timeFieldType['strPad'], 'H5T_STR_NULLPAD')


if __name__ == '__main__':
Expand Down
6 changes: 3 additions & 3 deletions test/integ/datatypetest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,10 +184,10 @@ def testPostVLenStringType(self):
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)
data_type = { 'cset': 'H5T_CSET_ASCII',
data_type = { 'charSet': 'H5T_CSET_ASCII',
'class': 'H5T_STRING',
'strpad': 'H5T_STR_NULLPAD',
'strsize': 'H5T_VARIABLE'}
'strPad': 'H5T_STR_NULLPAD',
'length': 'H5T_VARIABLE'}
payload = {'type': data_type}
req = self.endpoint + "/datatypes"
Expand Down
24 changes: 12 additions & 12 deletions test/unit/hdf5dtypeTest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,30 +57,30 @@ def testBaseStringTypeItem(self):
self.failUnlessEqual(typeItem['class'], 'H5T_STRING')
self.failUnlessEqual(typeItem['size'], 3)
self.failUnlessEqual(typeItem['base_size'], 3)
self.failUnlessEqual(typeItem['strsize'], 3)
self.failUnlessEqual(typeItem['length'], 3)
self.failUnlessEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.failUnlessEqual(typeItem['strpad'], 'H5T_STR_NULLPAD')
self.failUnlessEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.failUnlessEqual(typeItem['strPad'], 'H5T_STR_NULLPAD')
self.failUnlessEqual(typeItem['charSet'], 'H5T_CSET_ASCII')

def testBaseVLenAsciiTypeItem(self):
dt = special_dtype(vlen=str)
typeItem = hdf5dtype.getTypeItem(dt)
self.failUnlessEqual(typeItem['class'], 'H5T_STRING')
self.failUnlessEqual(typeItem['base_size'], 8)
self.failUnlessEqual(typeItem['strsize'], 'H5T_VARIABLE')
self.failUnlessEqual(typeItem['length'], 'H5T_VARIABLE')
self.failUnlessEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.failUnlessEqual(typeItem['strpad'], 'H5T_STR_NULLTERM')
self.failUnlessEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.failUnlessEqual(typeItem['strPad'], 'H5T_STR_NULLTERM')
self.failUnlessEqual(typeItem['charSet'], 'H5T_CSET_ASCII')

def testBaseVLenUnicodeTypeItem(self):
dt = special_dtype(vlen=unicode)
typeItem = hdf5dtype.getTypeItem(dt)
self.failUnlessEqual(typeItem['class'], 'H5T_STRING')
self.failUnlessEqual(typeItem['base_size'], 8)
self.failUnlessEqual(typeItem['strsize'], 'H5T_VARIABLE')
self.failUnlessEqual(typeItem['length'], 'H5T_VARIABLE')
self.failUnlessEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.failUnlessEqual(typeItem['strpad'], 'H5T_STR_NULLTERM')
self.failUnlessEqual(typeItem['cset'], 'H5T_CSET_UTF8')
self.failUnlessEqual(typeItem['strPad'], 'H5T_STR_NULLTERM')
self.failUnlessEqual(typeItem['charSet'], 'H5T_CSET_UTF8')

def testBaseEnumTypeItem(self):
mapping = {'RED': 0, 'GREEN': 1, 'BLUE': 2}
Expand Down Expand Up @@ -180,18 +180,18 @@ def testCreateBaseType(self):


def testCreateBaseStringType(self):
typeItem = { 'class': 'H5T_STRING', 'cset': 'H5T_CSET_ASCII', 'strsize': 6 }
typeItem = { 'class': 'H5T_STRING', 'charSet': 'H5T_CSET_ASCII', 'length': 6 }
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'string48')
self.assertEqual(dt.kind, 'S')

def testCreateVLenStringType(self):
typeItem = { 'class': 'H5T_STRING', 'cset': 'H5T_CSET_ASCII', 'strsize': 'H5T_VARIABLE' }
typeItem = { 'class': 'H5T_STRING', 'charSet': 'H5T_CSET_ASCII', 'length': 'H5T_VARIABLE' }
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'object')
self.assertEqual(dt.kind, 'O')

typeItem = { 'class': 'H5T_STRING', 'cset': 'H5T_CSET_UTF8', 'strsize': 'H5T_VARIABLE' }
typeItem = { 'class': 'H5T_STRING', 'charSet': 'H5T_CSET_UTF8', 'length': 'H5T_VARIABLE' }
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'object')
self.assertEqual(dt.kind, 'O')
Expand Down
8 changes: 4 additions & 4 deletions testjson/array_dset.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"datasets": {
"146c9fb5-7b51-11e4-a549-3c15c2da029e": {
"b4a4e002-cd15-11e4-bdfc-3c15c2da029e": {
"alias": [
"/DS1"
],
Expand Down Expand Up @@ -118,19 +118,19 @@
}
},
"groups": {
"146b5775-7b51-11e4-a63d-3c15c2da029e": {
"b4a3d914-cd15-11e4-bf3e-3c15c2da029e": {
"alias": [
"/"
],
"links": [
{
"class": "H5L_TYPE_HARD",
"collection": "datasets",
"id": "146c9fb5-7b51-11e4-a549-3c15c2da029e",
"id": "b4a4e002-cd15-11e4-bdfc-3c15c2da029e",
"title": "DS1"
}
]
}
},
"root": "146b5775-7b51-11e4-a63d-3c15c2da029e"
"root": "b4a3d914-cd15-11e4-bf3e-3c15c2da029e"
}

0 comments on commit b54255a

Please sign in to comment.