Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

executable file 347 lines (305 sloc) 10.633 kB
#!/bin/sh
# show the status of stamp files and run programs on the rendering servers
ERROR()
{
echo error: $*
exit 1
}
USAGE()
{
[ -z "$1" ] || echo error: $*
echo usage: $(basename "$0") '<options>'
echo ' --help -h this message'
echo ' --work -w show work directory files'
echo ' --farm=<n> -F <n> switch to the farm n ['${farm_number}']'
echo ' --image -i show image directory files'
echo ' --stamps -s show status of stamps'
echo ' --analyse -a analyse the log file'
echo ' --git -g run git pull --rebase'
echo ' --process[=<n>] -p[<n>] show python/php processes cut line 1..n [unlimited]'
echo ' --base=<dir> -b <dir> set the base directory ['${base_dir}']'
echo ' --tail[=<n>] -t[<n>] show last line of <base_dir>/log [3]'
echo ' --memory -m show memory status'
echo ' --list=<n-m> -l <n-m> restrict to certain machines [1-9]'
echo ' --language=<xx> -L <xx> process for language:suffix ['${language}':'${suffix}']'
echo ' --compress=<a:b> -z <a:b> default compression for rendering e.g. --compress=8:128ki'
echo ' --run=<rl> -r <rl> multirun example rl = "en:pedia:wiki:YES:1:131072:10 de:::YES:32:200000"'
echo ' start the Run script in a screen {index, parse, render}'
echo ' quote arguments OR use "," e.g. es,de,en:dict'
echo ' (see the Run script for exact specifications)'
echo ' --re-render=<rl> -R <rl> as --run but only does {render}'
echo ' --re-index=<rl> -I <rl> as --run but only does {index}'
echo ' --copy -c run the copy process i.e. {Copy --to=8 -l <lang>}'
echo ' --get[=<dir>] -G[<dir>] combine and get the data files [image]'
echo ' --sim4[=<dir>] -4[<dir>] install fonts, nls and run simulator [image]'
echo ' --new-session -N start a new screen session for run and copy to use'
echo ' --fetch-log -f fetch log files'
echo ' --xml-link -x link wiki .xml file to latest version'
echo ' --abort -A abort remote PHP/Python/make'
echo ' --unzip -u unzip compressed xml files, removing corresponding old files'
exit 1
}
# ListOfAllContentTypes, TypeToName, TypeToFilename
. "$(dirname "$0")/wiki-names.sh"
set_articles_link()
{
local fn
if ! fn="$(TypeToFilename "${suffix}")"
then
USAGE invalid suffix: ${suffix}, [${ListOfAllContentTypes}]
fi
articles_link="${language}${fn}"
}
set_farm()
{
case "${1}" in
1)
combine_host="3"
render_host='--host=simul3'
run_parallel="--machines=3 --parallel=12"
;;
2)
combine_host=8
render_host='--host=render9'
run_parallel="--machines=9 --parallel=3"
;;
*)
USAGE invalid --farm=${1}
;;
esac
}
# main program
# ------------
verbose=no
language=en
suffix=pedia
list=''
base_dir=wikireader
combine_host=
render_host=
run_parallel=
compression=
set_farm 1
set_articles_link
getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvb:swigup::t::aml:L:z:r:R:I:cG::4::NfxAF: --long=help,verbose,base:,stamps,work,image,git,unzip,process::,tail::,analyse,memory,list:,language:,compress:,run:,re-render:,re-index:,copy,get::,sim4::,new-session,fetch-log,xml-link,abort,farm: -- "$@") || exit 1
# replace the arguments with the parsed values
eval set -- "${args}"
while :
do
case "$1" in
-v|--verbose)
verbose=yes
shift
;;
-F|--farm)
set_farm "$2"
shift 2
;;
-b|--base)
base_dir="$2"
shift 2
;;
-s|--stamps)
RenderDo --yes ${render_host} ${list} --brief --post='sed s/stamp-//g' "cd '${base_dir}/work/${language}${suffix}'"' && printf "\33[34;1m%s\33[0;m " stamp* && echo'
shift
;;
-w|--work)
RenderDo --yes ${render_host} ${list} --brief "cd '${base_dir}' && ls -l 'work/${language}${suffix}'"
shift
;;
-N|--new-session)
for host in $(RenderDo ${render_host} ${list} --print-list)
do
StartScreen --cd="${base_dir}" "${host}"
done
shift
;;
-z|--compress)
c_a="${2%%:*}"
c_b="${2##*:}"
[ -z "${c_a}" -o -z "${c_b}" ] && USAGE invalid option: $1=$2
compression="--articles=${c_a} --block-size=${c_b}"
shift 2
;;
-r|--run)
run_list=$(echo $2 | sed 's/,/ /g')
c="scripts/Run ${run_parallel} ${compression} --clear ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;
-R|--re-render)
run_list=$(echo $2 | sed 's/,/ /g')
c="scripts/Run ${run_parallel} ${compression} --re-render ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;
-I|--re-index)
run_list=$(echo $2 | sed 's/,/ /g')
c="scripts/Run ${run_parallel} --index-only ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;
-c|--copy)
c="scripts/Copy --base='${base_dir}' --to='${combine_host}' --language='${language}' --suffix='${suffix}'"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
echo -n Waiting for rsync to finish: -
i=0
while :
do
rc="0 $(RenderDo --yes ${render_host} ${list} --brief 'pgrep -c rsync' | awk '{print "+ " $2}')"
rc=$((${rc}))
[ "${rc}" -eq 0 ] && break
case "${i}" in
0)
i=1
printf '\010\\'
;;
1)
i=2
printf '\010|'
;;
2)
i=3
printf '\010/'
;;
3)
i=0
printf '\010-'
;;
esac
done
echo
echo Creating Index files for: ${language}${suffix}
RenderDo --yes ${render_host} --list="${combine_host}" --brief "cd '${base_dir}' && make VERBOSE='${verbose}' WORKDIR='work' DESTDIR='image' WIKI_FILE_PREFIX=wiki WIKI_DIR_SUFFIX='${suffix}' WIKI_LANGUAGE='${language}' EXTRACT_VERSION_FROM='${articles_link}' print-WIKI_VERSION combine"
shift
;;
-i|--image)
RenderDo --yes ${render_host} ${list} --brief "cd '${base_dir}' && ls -l 'image/${language}${suffix}'"
shift
;;
-f|--fetch-log)
for host in $(RenderDo ${render_host} ${list} --print-list)
do
lf="${host}-${language}-${suffix}.log"
echo fetch log from: ${host} to: ${lf}
RenderDo --yes ${render_host} --list="${host##*[a-zA-Z_-]}" "cat '${base_dir}/${language}-${suffix}.log'" > "${lf}"
done
shift
;;
-g|--git)
RenderDo --yes ${render_host} ${list} --verbose "cd '${base_dir}' && git checkout . && git pull --rebase"
shift
;;
-u|--unzip)
RenderDo --yes ${render_host} ${list} --verbose "cd && '${base_dir}/scripts/UnzipFiles' -v ."
shift
;;
-p|--process)
if [ -n "$2" ]
then
RenderDo --yes ${render_host} ${list} --brief --post="cut -b 1-$2" 'echo;ps -o pid,cmd -u wr --forest'
else
RenderDo --yes ${render_host} ${list} --brief 'echo;ps -o pid,cmd -u wr --forest'
fi
shift 2
;;
-t|--tail)
t=3
[ -n "$2" ] && t="$2"
RenderDo --yes ${render_host} ${list} --brief "tail -n '${t}' '${base_dir}'/${language}-${suffix}.log"
shift 2
;;
-a|--analyse)
RenderDo --yes ${render_host} ${list} --brief "cd '${base_dir}' && ./scripts/AnalyseLog '${language}-${suffix}.log'"
shift
;;
-m|--memory)
RenderDo --yes ${render_host} ${list} --brief 'echo Memory Status \(Megabytes\); free -m'
shift
;;
-l|--list)
list="--list=$2"
shift 2
;;
-L|--language)
t="${2}::::::::::"
language="${t%%:*}"
[ -z "${language}" ] && language=en
t="${t#*:}"
suffix="${t%%:*}"
[ -z "${suffix}" ] && suffix=pedia
t="${t#*:}"
shift 2
set_articles_link
;;
-G|--get)
dest=image
if [ -n "$2" ]
then
dest="$2"
[ -e "${dest}" ] || mkdir -p "${dest}"
[ -d "${dest}" ] || USAGE ${dest} is not a directory
fi
echo Retrieving data for: ${language}${suffix}
h=$(RenderDo ${render_host} --list="${combine_host}" --print-list | head -n 1)
rsync -cavHx --progress --exclude='*-tmp' "wr@${h}:${base_dir}/image/${language}${suffix}" "${dest}/"
shift 2
;;
-4|--sim4)
dest=image
[ -n "$2" ] && dest="$2"
[ -d "${dest}" ] || USAGE ${dest} is not a directory
# attempt to find a directory to set the language/suffix
# in case that enpedia does not exist
first_dir=$(find "${dest}" -maxdepth 1 -mindepth 1 -type d -print -quit)
[ -d "${first_dir}" ] || USAGE ${first_dir} is not a directory
l1=en
s1=pedia
first_dir=$(basename "${first_dir}")
for s in ${ListOfAllContentTypes}
do
l0="${first_dir%${s}}"
if [ X"${first_dir}" != X"${l0}" ]
then
l1="${l0}"
s1="${s}"
break
fi
done
make DESTDIR="${dest}" WIKI_LANGUAGE="${l1}" WIKI_DIR_SUFFIX="${s1}" nls-install fonts-install sim4
shift 2
;;
-x|--xml-link)
for s in ${ListOfAllFilePrefixes}
do
echo Checking for: ${language}${s}
RenderDo --yes ${render_host} ${list} --brief "cd '${base_dir}' &&
rm -f '${language}${s}-pages-articles.xml' &&
f=\$(ls -1 ../'${language}${s}'-*-pages-articles.xml 2>/dev/null | tail -n 1) &&
[ -n \"\${f}\" ] &&
ln -fs \"\${f}\" '${language}${s}-pages-articles.xml' &&
ls -l '${language}${s}'-pages-articles.xml"
done
shift
;;
-A|--abort)
RenderDo --yes ${render_host} ${list} --brief 'killall php python make'
shift
;;
--)
shift
break
;;
-h|--help)
USAGE
;;
*)
USAGE invalid option: $1
;;
esac
done
[ $# -gt 0 ] && USAGE extraneous arguments: $*
echo
echo completed
Jump to Line
Something went wrong with that request. Please try again.