-
Notifications
You must be signed in to change notification settings - Fork 0
/
gen_data.py
executable file
·74 lines (63 loc) · 1.64 KB
/
gen_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#!/usr/bin/env python3
import itertools
import json
import pickle
import string
import random
data_simple_list = [
random.randint(0, 1000000)
for _ in range(6000)
]
data_points_list = [
[random.randint(0, 1000000), random.randint(0, 1000000)]
for _ in range(6000)
]
data_map = {
'_'.join(key): ''.join(
random.choice(string.ascii_letters)
for _ in range(128)
)
for key in itertools.permutations(
['long', 'key', 'name', 'foo', 'bar'],
4,
)
}
data_string = ''.join(
random.choice(string.ascii_letters)
for _ in range(1024 * 1024)
)
data_bytes = pickle.dumps(
dict(
string={'': data_string},
map={'': data_map},
struct_map={'': data_map},
simple_list={'': data_simple_list},
points_list={'': data_points_list},
),
protocol=2,
)
with open('test_data.bin', 'wb') as f:
f.write(data_bytes)
with open('specs/struct_map.thrift', 'w') as f:
f.write('struct StructMap {\n')
for i, key in enumerate(data_map):
f.write(' {}: required string {},\n'.format(i + 1, key))
f.write('}\n')
with open('specs/struct_map.proto', 'w') as f:
f.write('message StructMap {\n')
for i, key in enumerate(data_map):
f.write(' required string {} = {};\n'.format(key, i + 1))
f.write('}\n')
with open('specs/struct_map.avsc', 'w') as f:
f.write(json.dumps(
dict(
namespace="test.avro",
type="record",
name="StructMap",
fields=[
dict(type='string', name=field)
for field in data_map
]
),
indent=4,
))