forked from h5py/h5py
/
vlen_leak.py
90 lines (71 loc) · 2.01 KB
/
vlen_leak.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# This file is part of h5py, a Python interface to the HDF5 library.
#
# http://www.h5py.org
#
# Copyright 2008-2013 Andrew Collette and contributors
#
# License: Standard 3-clause BSD; see "license.txt" for full license terms
# and contributor agreement.
"""
Demonstrates memory leak involving variable-length strings.
"""
import sys
import resource
import numpy as np
import h5py
FNAME = 'test.hdf5'
if 'linux' in sys.platform:
MAXRSS_BYTES = 1024. # in KiB on linux
else:
MAXRSS_BYTES = 1.
if sys.version_info[0] == 3:
xrange = range
unicode = str
memory = 0
def print_memory():
global memory
rubytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*MAXRSS_BYTES
print ("%.2f MB (%.2f since last call)" % (rubytes/(1024.**2), (rubytes-memory)/(1024.**2)))
memory = rubytes
def make_data(kind):
global data
global dt
if kind is bytes:
s = b"xx"
else:
s = b"xx".decode('utf8')
dt = h5py.special_dtype(vlen=kind)
data = np.array([s*100 for idx in xrange(1000)])
def ds_leak():
print("Testing vlens for dataset r/w")
print("-----------------------------")
with h5py.File(FNAME,'w') as f:
ds = f.create_dataset('dset', (1000,), dtype=dt)
for idx in xrange(500):
#print idx
if idx%100 == 0:
print_memory()
ds[...] = data
ds[...]
def attr_leak():
print("Testing vlens for attribute r/w")
print("-------------------------------")
with h5py.File(FNAME,'w') as f:
for idx in xrange(500):
if idx%100 == 0:
print_memory()
f.attrs.create('foo', dtype=dt, data=data)
f.attrs['foo']
if __name__ == '__main__':
print("h5py ", h5py.version.version)
print("HDF5 ", h5py.version.hdf5_version)
print("Bytes test")
print("==========")
make_data(bytes)
attr_leak()
ds_leak()
print("Unicode test")
print("============")
make_data(unicode)
attr_leak()
ds_leak()