-
Notifications
You must be signed in to change notification settings - Fork 0
/
produce.ini
28 lines (24 loc) · 951 Bytes
/
produce.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
[tweets/share/%{collection}/%{name}.txt]
dep.select = tweets/select/%{collection}/%{name}.gz
recipe = zcat %{select} | bin/poultry show -t {t.id} > %{target}
#recipe = zcat tweets/select/%{collection}/%{name}.gz | bin/poultry show -t {t.id} > %{target}
[tweets/hydrate/%{collection}/%{name}.gz]
#dep.share = tweets/share/%{name}.txt
recipe =
[ ! -d tweets/hydrate/%{collection}/ ] && mkdir tweets/hydrate/%{collection}/
bin/twarc.py \
-c parts/etc/poultry.cfg \
-p twitter \
--hydrate tweets/share/%{collection}/%{name}.txt | gzip > %{target}
[tweets/db/%{collection}/%{name}.inserted]
dep.hydrate = tweets/hydrate/%{collection}/%{name}.gz
dep.clusters = clusters/%{collection}.cfg
recipe =
[ ! -d tweets/db/%{collection}/ ] && mkdir tweets/db/%{collection}/
zcat %{hydrate} \
|\
bin/flock -v debug insert \
--collection=%{collection} \
--clusters %{clusters} \
--with-text \
&& touch %{target}