forked from wikireader/wikireader
/
ProcessAll
executable file
·177 lines (152 loc) · 2.9 KB
/
ProcessAll
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
#!/bin/sh
# process all languages
ERROR()
{
echo error: $*
exit 1
}
USAGE()
{
[ -z "$1" ] || echo error: $*
echo usage: $(basename "$0") '<options> <command>'
echo ' --help -h This message'
echo ' --verbose -v Display messages'
echo 'commands:'
echo ' help This message'
echo ' link Move all symlinks to latest versions'
echo ' run Rebuild all'
echo ' cjk Rebuild ko,ja,zh'
echo ' en Rebuild short for en:pedia (or other two letter language code)'
echo ' en:dict Rebuild specific item (<lang>:<type>)'
echo ' analyse Show summary of logs'
echo ' get Combine and fetch all recently rendered data'
exit 1
}
# main program
verbose=no
debug=no
getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvd --long=help,verbose,debug -- "$@") ||exit 1
# replace the arguments with the parsed values
eval set -- "${args}"
while :
do
case "$1" in
-v|--verbose)
verbose=yes
shift
;;
-d|--debug)
debug=yes
shift
;;
--)
shift
break
;;
-h|--help)
USAGE
;;
*)
USAGE invalid option: $1
;;
esac
done
AllItems='
cy
de
da
el
en::::::-2
en:dict
en:quote
en:trav
en:starw
es
fi
fr
hu
it
ja
ko
nl
no
pl
pt
ru
sr
tr
zh
'
AllLanguages=
AllCSV=
for item in ${AllItems}
do
AllCSV="${AllCSV},${item}"
lang="${item%%:*}"
eval flag="\${lf_${lang}}"
if [ X"${flag}" = X"" ]
then
eval lf_${lang}="${lang}"
AllLanguages="${AllLanguages} ${lang}"
fi
done
AllLanguages="${AllLanguages## }"
AllCSV="${AllCSV##,}"
[ $# -eq 0 ] && USAGE missing command
for command in $@
do
case "${command}" in
help)
USAGE
;;
# command to rebuild all
run)
progress -z 16:128ki -r "${AllCSV}"
;;
# Chinese, Japanese, Korean
cjk)
progress -z 16:128ki -r 'ja,ko,zh'
;;
# retrieve all data
get)
for l in ${AllItems}
do
progress -L"${l}" -c -G
done
;;
# link all data
link)
for l in ${AllLanguages}
do
progress -L"${l}" -x
done
;;
# analyse
analyse)
for l in ${AllItems}
do
echo Analysis for: ${l}
progress -L"${l}" -a
done
;;
# en is the exception (reduce the number of threads)
en)
progress -z 16:128ki -r en::::::-2
;;
rr-en|rren)
progress -z 16:128ki --re-render en::::::-2
;;
# run only one language
[a-z][a-z]|[a-z][a-z]:*)
progress -z 16:128ki -r "${command}"
;;
# re-render an already built English Wikipedia to fit into 4GB
en4gb|en4GB)
progress --re-render en:::NO:255:350000:-2:20000
;;
*)
ERROR invalid command: ${command}
esac
done