Skip to content

Commit

Permalink
[scripts] fixes for downloader
Browse files Browse the repository at this point in the history
so it handles entries like en:dict correctly

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information
hxw committed Feb 18, 2011
1 parent 53c5ce5 commit 5c0f6fd
Showing 1 changed file with 11 additions and 14 deletions.
25 changes: 11 additions & 14 deletions scripts/DownloadWikiXML
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ USAGE()
echo ' --verbose -v more messages'
echo ' --url=<url> -u <url> base URL for downloads ['${RootURL}']'
echo ' --download=<dir> -d <dir> download directory [No Download]'
echo ' --suffix=<text> -s <text> file name suffix ['${suffix}']'
echo ' --ext=<text> -x <text> file name extension ['${ext}']'
echo ' --rate=<rate> -r <rate> download rate limit ['${rate}']'
exit 1
Expand All @@ -30,14 +29,13 @@ USAGE()
verbose=no
debug=no
RootURL=http://download.wikipedia.org
suffix=wiki
ext=pages-articles.xml.bz2
download=
rate=50k

getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
args=$(${getopt} -o hvu:d:s:x:r:D --long=help,verbose,url,download:,suffix:,ext:,rate:,debug -- "$@") ||exit 1
args=$(${getopt} -o hvu:d:x:r:D --long=help,verbose,url,download:,ext:,rate:,debug -- "$@") ||exit 1

# replace the arguments with the parsed values
eval set -- "${args}"
Expand All @@ -61,11 +59,6 @@ do
shift 2
;;

-s|--suffix)
suffix=$2
shift 2
;;

-x|--ext)
ext=$2
shift 2
Expand Down Expand Up @@ -106,10 +99,14 @@ fi
# process each lang{:type} argument
for arg in $*
do
lang=${arg%:*}
typename=$(TypeToName "${arg#*:}")
[ -z "${typename}" ] && typename="${suffix}"
url="${RootURL}/${lang}${typename}"
arg="${arg}:::::::::::"
lang="${arg%%:*}"
arg="${arg#*:}"
typename="${arg%%:*}"
[ -z "${typename}" ] && typename=pedia
file="${lang}$(TypeToFilename "${typename}")"
wikiname="${file%%-*}"
url="${RootURL}/${wikiname}"

stamp=$(lynx -dump "${url}" | awk '
BEGIN {
Expand All @@ -126,8 +123,8 @@ END{
}
}')

file="${arg}${suffix}-${stamp}-${ext}"
remote="${RootURL}/${arg}${suffix}/${stamp}/${file}"
file="${wikiname}-${stamp}-${ext}"
remote="${RootURL}/${wikiname}/${stamp}/${file}"
echo Download: ${remote}
if [ -n "${download}" ]
then
Expand Down

0 comments on commit 5c0f6fd

Please sign in to comment.