Skip to content

Commit

Permalink
[scripts] Allow setting a default compression
Browse files Browse the repository at this point in the history 
originally default was fixed at 1:131072 unless overridden for a
specific language.

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information
@hxw
hxw committed Oct 15, 2010
1 parent f6f200f commit e54d04f
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 17 deletions.
41 changes: 31 additions & 10 deletions scripts/Run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,21 @@ USAGE()
echo ' arg <language>:<dir_suffix>:<file_prefix>:<links>:<count>:<size>:<parallel>:<truncate>'
echo ' --help -h this message'
echo ' --verbose -v more messages'
echo ' --articles -a <n> articles per block ['${default_articles_per_block}']'
echo ' --block-size -b <n>[k] max size of block ['${default_block_size}']'
echo ' --index-only -i only build the index'
echo ' --no-run -n do not run final make'
echo ' --clear -c clear work and dest dirs - so everything will be built'
echo ' --re-parse -R clear parse and render stamps - to reuse existion index'
echo ' --re-render -r clear render stamps - for re-rendering with diffferent compression'
echo ' --machines=<m> -m <m> set machine count [${MACHINE_COUNT} or 9]'
echo ' --parallel=<n> -j <n> set make -j value [${PARALLEL_BUILD} or 3]'
echo ' --work=<dir> -w <dir> workdir [work]'
echo ' --dest=<dir> -d <dir> destdir [image]'
echo ' --temp=<dir> -t <dir> tempdir [/tmp]'
echo ' --machines=<m> -m <m> set machine count ['${machine_count}']'
echo ' --parallel=<n> -j <n> set make -j value ['${parallel_build}']'
echo ' --work=<dir> -w <dir> workdir ['${work}']'
echo ' --dest=<dir> -d <dir> destdir ['${dest}']'
echo ' --temp=<dir> -t <dir> tempdir ['${temp}']'
echo ' --farm=<f> -f <f> manual farm number [from hostname numeric suffix]'
echo ' --debug -D only display make operations, do not execute them'
echo ' Arguments: defaults: en:pedia:wiki:YES:1:131072:<n>:unlimited'
echo ' Arguments: defaults: en:pedia:wiki:YES:'${default_articles_per_block}':'${default_block_size}':<n>:unlimited'
echo ' links = [YES|NO] to enable inclusion of language links'
echo ' count:size = max articles/max bytes to compress into one block'
echo ' parallel = overrides the global --parallel=<n> for this item'
Expand Down Expand Up @@ -117,10 +119,12 @@ MaximumThreads=64
truncate='unlimited'
machine_count=${MACHINE_COUNT:-9}
parallel_build=${PARALLEL_BUILD:-3}
default_articles_per_block=1
default_block_size=131072

getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvp:incrRw:d:t:f:p:j:m:D --long=help,verbose,index-only,no-run,clear,re-render,re-parse,work:,dest:,temp:,farm:,parallel:,machines:,debug -- "$@") ||exit 1
args=$(${getopt} -o hvp:a:b:incrRw:d:t:f:p:j:m:D --long=help,verbose,articles:,block-size:,index-only,no-run,clear,re-render,re-parse,work:,dest:,temp:,farm:,parallel:,machines:,debug -- "$@") ||exit 1

# replace the arguments with the parsed values
eval set -- "${args}"
Expand All @@ -133,6 +137,20 @@ do
shift
;;

-a|--articles)
IsNumber "$2" || USAGE not a number in: $1=$2
default_articles_per_block="$2"
shift 2
;;

-b|--block-size)
default_block_size="$2"
[ X"${default_block_size%ki}ki" = X"${default_block_size}" ] && default_block_size=$((${default_block_size%ki} * 1024))
[ X"${default_block_size%k}k" = X"${default_block_size}" ] && default_block_size=$((${default_block_size%k} * 1000))
IsNumber "${default_block_size}" || USAGE not a number in: $1=$2
shift 2
;;

-i|--index-only)
IndexOnly=yes
clear=index
Expand Down Expand Up @@ -249,11 +267,11 @@ do

arg="${arg#*:}"
articles_per_block="${arg%%:*}"
IsNumber "${articles_per_block}" || articles_per_block=1
IsNumber "${articles_per_block}" || articles_per_block="${default_articles_per_block}"

arg="${arg#*:}"
article_block_size="${arg%%:*}"
IsNumber "${article_block_size}" || article_block_size=131072
IsNumber "${article_block_size}" || article_block_size="${default_block_size}"

arg="${arg#*:}"
threads="${arg%%:*}"
Expand Down Expand Up @@ -283,8 +301,11 @@ do
books)
articles_link="${language}wikibooks-pages-articles.xml"
;;
starw)
articles_link="${language}Wookieepedia-pages-articles.xml"
;;
*)
USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books]
USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books, starw]
;;
esac

Expand Down
28 changes: 21 additions & 7 deletions scripts/progress
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ USAGE()
echo ' --tail[=<n>] -t[<n>] show last line of <base_dir>/log [3]'
echo ' --memory -m show memory status'
echo ' --list=<n-m> -l <n-m> restrict to certain machines [1-9]'
echo ' --language=<xx> -L <xx> process for language:suffix [en:pedia]'
echo ' --language=<xx> -L <xx> process for language:suffix ['${language}':'${suffix}']'
echo ' --compress=<a:b> -z <a:b> default compression for rendering e.g. --compress=8:128ki'
echo ' --run=<rl> -r <rl> multirun example rl = "en:pedia:wiki:YES:1:131072:10 de:::YES:32:200000"'
echo ' start the Run script in a screen {index, parse, render}'
echo ' quote arguments OR use "," e.g. es,de,en:dict'
Expand Down Expand Up @@ -55,8 +56,11 @@ set_articles_link()
books)
articles_link="${language}wikibooks-pages-articles.xml"
;;
starw)
articles_link="${language}Wookieepedia-pages-articles.xml"
;;
*)
USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books]
USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books, starw]
;;
esac
}
Expand Down Expand Up @@ -91,13 +95,14 @@ base_dir=wikireader
combine_host=
render_host=
run_parallel=
compression=

set_farm 1
set_articles_link

getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvb:swigp::t::aml:L:r:R:I:cG::4::NfxAF: --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,run:,re-render:,re-index:,copy,get::,sim4::,new-session,fetch-log,xml-link,abort,farm: -- "$@") || exit 1
args=$(${getopt} -o hvb:swigp::t::aml:L:z:r:R:I:cG::4::NfxAF: --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,compress:,run:,re-render:,re-index:,copy,get::,sim4::,new-session,fetch-log,xml-link,abort,farm: -- "$@") || exit 1
# replace the arguments with the parsed values
eval set -- "${args}"

Expand Down Expand Up @@ -137,16 +142,25 @@ do
shift
;;

-z|--compress)
c_a="${2%%:*}"
c_b="${2##*:}"
[ -z "${c_a}" -o -z "${c_b}" ] && USAGE invalid option: $1=$2
compression="--articles=${c_a} --block-size=${c_b}"
shift 2
;;

-r|--run)
run_list=$(echo $2 | sed 's/,/ /g')
c="scripts/Run ${run_parallel} --clear ${run_list}"

c="scripts/Run ${run_parallel} ${compression} --clear ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;

-R|--re-render)
run_list=$(echo $2 | sed 's/,/ /g')
c="scripts/Run ${run_parallel} --re-render ${run_list}"
c="scripts/Run ${run_parallel} ${compression} --re-render ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;
Expand Down Expand Up @@ -283,7 +297,7 @@ do
l1=en
s1=pedia
first_dir=$(basename "${first_dir}")
for s in books dict pedia quote
for s in books dict pedia quote starw
do
l0="${first_dir%${s}}"
if [ X"${first_dir}" != X"${l0}" ]
Expand All @@ -298,7 +312,7 @@ do
;;

-x|--xml-link)
for s in wiki wiktionary wikiquote wikibooks
for s in wiki wiktionary wikiquote wikibooks Wookieepedia
do
RenderDo --yes ${render_host} ${list} --brief "cd '${base_dir}' &&
rm -f '${language}${s}-pages-articles.xml' &&
Expand Down

0 comments on commit e54d04f

Please sign in to comment.