-
Notifications
You must be signed in to change notification settings - Fork 22
/
util.py
93 lines (74 loc) · 2.21 KB
/
util.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
import io
from urllib.parse import quote as urlquote
import msgpack
def create_url(tmpl, **values):
"""Create url with values
Args:
tmpl (str): url template
values (dict): values for url
"""
quoted_values = {k: urlquote(str(v)) for k, v in values.items()}
return tmpl.format(**quoted_values)
def parse_csv_value(s):
"""Parse and convert value to suitable types
Args:
s (str): value on csv
Returns:
Suitable value (int, float, str, bool or None)
"""
try:
return int(s)
except (OverflowError, ValueError):
try:
return float(s)
except (OverflowError, ValueError):
pass
lower = s.lower()
if lower in ("false", "true"):
return "true" == lower
elif lower in ("", "none", "null"):
return None
else:
return s
def create_msgpack(items):
"""Create msgpack streaming bytes from list
Args:
items (list of dict): target list
Returns:
Converted msgpack streaming (bytes)
Examples:
>>> t1 = int(time.time())
>>> l1 = [{"a": 1, "b": 2, "time": t1}, {"a":3, "b": 6, "time": t1}]
>>> create_msgpack(l1)
``b'\x83\xa1a\x01\xa1b\x02\xa4time\xce]\xa5X\xa1\x83\xa1a\x03\xa1b\x06\xa4time\xce]\xa5X\xa1'``
"""
stream = io.BytesIO()
packer = msgpack.Packer()
for item in items:
try:
mp = packer.pack(item)
except (OverflowError, ValueError):
packer.reset()
mp = packer.pack(normalized_msgpack(item))
stream.write(mp)
return stream.getvalue()
def normalized_msgpack(value):
"""Convert int to str if overflow
Args:
value (int, float, str, bool or None): value to be normalized
Returns:
Normalized value
"""
if isinstance(value, (list, tuple)):
return [normalized_msgpack(v) for v in value]
elif isinstance(value, dict):
return dict(
[(normalized_msgpack(k), normalized_msgpack(v)) for (k, v) in value.items()]
)
if isinstance(value, int):
if -(1 << 63) < value < (1 << 64):
return value
else:
return str(value)
else:
return value