Permalink
Browse files

[scripts] Allow setting a default compression

originally default was fixed at 1:131072 unless overridden for a
specific language.

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information...
1 parent f6f200f commit e54d04f93c9bbf8d22dee5ea37ff0260914267fe @hxw hxw committed Oct 15, 2010
Showing with 52 additions and 17 deletions.
  1. +31 −10 scripts/Run
  2. +21 −7 scripts/progress
View
@@ -16,19 +16,21 @@ USAGE()
echo ' arg <language>:<dir_suffix>:<file_prefix>:<links>:<count>:<size>:<parallel>:<truncate>'
echo ' --help -h this message'
echo ' --verbose -v more messages'
+ echo ' --articles -a <n> articles per block ['${default_articles_per_block}']'
+ echo ' --block-size -b <n>[k] max size of block ['${default_block_size}']'
echo ' --index-only -i only build the index'
echo ' --no-run -n do not run final make'
echo ' --clear -c clear work and dest dirs - so everything will be built'
echo ' --re-parse -R clear parse and render stamps - to reuse existion index'
echo ' --re-render -r clear render stamps - for re-rendering with diffferent compression'
- echo ' --machines=<m> -m <m> set machine count [${MACHINE_COUNT} or 9]'
- echo ' --parallel=<n> -j <n> set make -j value [${PARALLEL_BUILD} or 3]'
- echo ' --work=<dir> -w <dir> workdir [work]'
- echo ' --dest=<dir> -d <dir> destdir [image]'
- echo ' --temp=<dir> -t <dir> tempdir [/tmp]'
+ echo ' --machines=<m> -m <m> set machine count ['${machine_count}']'
+ echo ' --parallel=<n> -j <n> set make -j value ['${parallel_build}']'
+ echo ' --work=<dir> -w <dir> workdir ['${work}']'
+ echo ' --dest=<dir> -d <dir> destdir ['${dest}']'
+ echo ' --temp=<dir> -t <dir> tempdir ['${temp}']'
echo ' --farm=<f> -f <f> manual farm number [from hostname numeric suffix]'
echo ' --debug -D only display make operations, do not execute them'
- echo ' Arguments: defaults: en:pedia:wiki:YES:1:131072:<n>:unlimited'
+ echo ' Arguments: defaults: en:pedia:wiki:YES:'${default_articles_per_block}':'${default_block_size}':<n>:unlimited'
echo ' links = [YES|NO] to enable inclusion of language links'
echo ' count:size = max articles/max bytes to compress into one block'
echo ' parallel = overrides the global --parallel=<n> for this item'
@@ -117,10 +119,12 @@ MaximumThreads=64
truncate='unlimited'
machine_count=${MACHINE_COUNT:-9}
parallel_build=${PARALLEL_BUILD:-3}
+default_articles_per_block=1
+default_block_size=131072
getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
-args=$(${getopt} -o hvp:incrRw:d:t:f:p:j:m:D --long=help,verbose,index-only,no-run,clear,re-render,re-parse,work:,dest:,temp:,farm:,parallel:,machines:,debug -- "$@") ||exit 1
+args=$(${getopt} -o hvp:a:b:incrRw:d:t:f:p:j:m:D --long=help,verbose,articles:,block-size:,index-only,no-run,clear,re-render,re-parse,work:,dest:,temp:,farm:,parallel:,machines:,debug -- "$@") ||exit 1
# replace the arguments with the parsed values
eval set -- "${args}"
@@ -133,6 +137,20 @@ do
shift
;;
+ -a|--articles)
+ IsNumber "$2" || USAGE not a number in: $1=$2
+ default_articles_per_block="$2"
+ shift 2
+ ;;
+
+ -b|--block-size)
+ default_block_size="$2"
+ [ X"${default_block_size%ki}ki" = X"${default_block_size}" ] && default_block_size=$((${default_block_size%ki} * 1024))
+ [ X"${default_block_size%k}k" = X"${default_block_size}" ] && default_block_size=$((${default_block_size%k} * 1000))
+ IsNumber "${default_block_size}" || USAGE not a number in: $1=$2
+ shift 2
+ ;;
+
-i|--index-only)
IndexOnly=yes
clear=index
@@ -249,11 +267,11 @@ do
arg="${arg#*:}"
articles_per_block="${arg%%:*}"
- IsNumber "${articles_per_block}" || articles_per_block=1
+ IsNumber "${articles_per_block}" || articles_per_block="${default_articles_per_block}"
arg="${arg#*:}"
article_block_size="${arg%%:*}"
- IsNumber "${article_block_size}" || article_block_size=131072
+ IsNumber "${article_block_size}" || article_block_size="${default_block_size}"
arg="${arg#*:}"
threads="${arg%%:*}"
@@ -283,8 +301,11 @@ do
books)
articles_link="${language}wikibooks-pages-articles.xml"
;;
+ starw)
+ articles_link="${language}Wookieepedia-pages-articles.xml"
+ ;;
*)
- USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books]
+ USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books, starw]
;;
esac
View
@@ -23,7 +23,8 @@ USAGE()
echo ' --tail[=<n>] -t[<n>] show last line of <base_dir>/log [3]'
echo ' --memory -m show memory status'
echo ' --list=<n-m> -l <n-m> restrict to certain machines [1-9]'
- echo ' --language=<xx> -L <xx> process for language:suffix [en:pedia]'
+ echo ' --language=<xx> -L <xx> process for language:suffix ['${language}':'${suffix}']'
+ echo ' --compress=<a:b> -z <a:b> default compression for rendering e.g. --compress=8:128ki'
echo ' --run=<rl> -r <rl> multirun example rl = "en:pedia:wiki:YES:1:131072:10 de:::YES:32:200000"'
echo ' start the Run script in a screen {index, parse, render}'
echo ' quote arguments OR use "," e.g. es,de,en:dict'
@@ -55,8 +56,11 @@ set_articles_link()
books)
articles_link="${language}wikibooks-pages-articles.xml"
;;
+ starw)
+ articles_link="${language}Wookieepedia-pages-articles.xml"
+ ;;
*)
- USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books]
+ USAGE invalid suffix: ${suffix}, [pedia, dict, quote, books, starw]
;;
esac
}
@@ -91,13 +95,14 @@ base_dir=wikireader
combine_host=
render_host=
run_parallel=
+compression=
set_farm 1
set_articles_link
getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
-args=$(${getopt} -o hvb:swigp::t::aml:L:r:R:I:cG::4::NfxAF: --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,run:,re-render:,re-index:,copy,get::,sim4::,new-session,fetch-log,xml-link,abort,farm: -- "$@") || exit 1
+args=$(${getopt} -o hvb:swigp::t::aml:L:z:r:R:I:cG::4::NfxAF: --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,compress:,run:,re-render:,re-index:,copy,get::,sim4::,new-session,fetch-log,xml-link,abort,farm: -- "$@") || exit 1
# replace the arguments with the parsed values
eval set -- "${args}"
@@ -137,16 +142,25 @@ do
shift
;;
+ -z|--compress)
+ c_a="${2%%:*}"
+ c_b="${2##*:}"
+ [ -z "${c_a}" -o -z "${c_b}" ] && USAGE invalid option: $1=$2
+ compression="--articles=${c_a} --block-size=${c_b}"
+ shift 2
+ ;;
+
-r|--run)
run_list=$(echo $2 | sed 's/,/ /g')
- c="scripts/Run ${run_parallel} --clear ${run_list}"
+
+ c="scripts/Run ${run_parallel} ${compression} --clear ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;
-R|--re-render)
run_list=$(echo $2 | sed 's/,/ /g')
- c="scripts/Run ${run_parallel} --re-render ${run_list}"
+ c="scripts/Run ${run_parallel} ${compression} --re-render ${run_list}"
RenderDo --yes ${render_host} ${list} --brief "screen -X eval chdir 'chdir ${base_dir}' \"exec ${c}\""
shift 2
;;
@@ -283,7 +297,7 @@ do
l1=en
s1=pedia
first_dir=$(basename "${first_dir}")
- for s in books dict pedia quote
+ for s in books dict pedia quote starw
do
l0="${first_dir%${s}}"
if [ X"${first_dir}" != X"${l0}" ]
@@ -298,7 +312,7 @@ do
;;
-x|--xml-link)
- for s in wiki wiktionary wikiquote wikibooks
+ for s in wiki wiktionary wikiquote wikibooks Wookieepedia
do
RenderDo --yes ${render_host} ${list} --brief "cd '${base_dir}' &&
rm -f '${language}${s}-pages-articles.xml' &&

0 comments on commit e54d04f

Please sign in to comment.