Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

[offline-renderer,scripts] split prefix for renderer

Modify the renderer so that the idx-tmp files are placed in the
work directory.

Update Copy script to match.

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information...
commit 42e78e870e78e992f4dd24945b4423f8f951cb67 1 parent 891eb1a
@hxw hxw authored
View
11 host-tools/offline-renderer/ArticleRenderer.py
@@ -187,7 +187,8 @@ def usage(message):
print(' --test=file Output the uncompressed file for testing')
print(' --font-path=dir Path to font files (*.bmf) [fonts]')
print(' --article-index=file Article index dictionary input [articles.db]')
- print(' --prefix=name Device file name portion for .dat/.idx-tmp [pedia]')
+ print(' --data-prefix=name Directory and file name portion for .dat files [pedia]')
+ print(' --index-prefix=name Directory and file name portion for .idx-tmp files [pedia]')
print(' --languages-links=<YN> Turn on/off inter-wiki links [YES]')
print(' --images=<YN> Turn on/off in-line math images [YES]')
print(' --articles=<N> Articles per block [32]')
@@ -207,12 +208,13 @@ def main():
try:
opts, args = getopt.getopt(sys.argv[1:],
- 'hvwn:p:i:t:f:l:a:b:',
+ 'hvwn:d:p:i:t:f:l:a:b:',
['help',
'verbose',
'warnings',
'number=',
- 'prefix=',
+ 'data-prefix=',
+ 'index-prefix=',
'article-index=',
'test=',
'font-path=',
@@ -255,8 +257,9 @@ def main():
file_number = int(arg)
except ValueError:
usage('"{0:s}={1:s}" is not numeric'.format(opt, arg))
- elif opt in ('-p', '--prefix'):
+ elif opt in ('-d', '--data-prefix'):
data_file = arg + '{0:d}.dat'
+ elif opt in ('-p', '--index-prefix'):
index_file = arg + '{0:d}.idx-tmp'
elif opt in ('-f', '--font-path'):
font_path = arg
View
10 host-tools/offline-renderer/Makefile
@@ -55,7 +55,8 @@ ARTICLE_BLOCK_SIZE ?= 262144
WORKDIR_PATH := $(shell ${RESOLVEPATH} ${WORKDIR}/${WIKI_LANGUAGE}${WIKI_DIR_SUFFIX})
TEMPDIR_PATH := $(shell ${RESOLVEPATH} ${TEMPDIR}/${WIKI_LANGUAGE}${WIKI_DIR_SUFFIX})
-PREFIX := $(shell ${RESOLVEPATH} ${DESTDIR}/${WIKI_LANGUAGE}${WIKI_DIR_SUFFIX}/${WIKI_FILE_PREFIX})
+DATA_PREFIX := $(shell ${RESOLVEPATH} ${DESTDIR}/${WIKI_LANGUAGE}${WIKI_DIR_SUFFIX}/${WIKI_FILE_PREFIX})
+INDEX_PREFIX := $(shell ${RESOLVEPATH} ${WORKDIR}/${WIKI_LANGUAGE}${WIKI_DIR_SUFFIX}/${WIKI_FILE_PREFIX})
ARTICLES := $(shell ${RESOLVEPATH} ${WORKDIR_PATH}/articles.db)
COUNTS_FILE := $(shell ${RESOLVEPATH} ${WORKDIR_PATH}/counts.text)
TEMPLATE_FILE := $(shell ${RESOLVEPATH} ${WORKDIR_PATH}/templates.db)
@@ -128,7 +129,7 @@ index: check-dirs check-xml stamp-RedirectedTo.py stamp-PinyinTable.py
--article-counts="${COUNTS_FILE}" \
--templates="${TEMPLATE_FILE}" \
--language="${WIKI_LANGUAGE}" \
- --prefix="${PREFIX}" ${TRUNCATE_ARG} ${XML_FILES}
+ --prefix="${DATA_PREFIX}" ${TRUNCATE_ARG} ${XML_FILES}
HTML_FILES_COUNT := $(words $(wildcard ${WORKDIR_PATH}/*.html))
.PHONY: merge
@@ -183,7 +184,8 @@ render: check-dirs check-fonts check-html stamp-PinyinTable.py
./ArticleRenderer.py ${VERBOSE_ARG} --number="${RENDER_BLOCK}" \
--font-path="${FONT_PATH}" \
--article-index="${ARTICLES}" \
- --prefix="${PREFIX}" \
+ --data-prefix="${DATA_PREFIX}" \
+ --index-prefix="${INDEX_PREFIX}" \
--language-links="${ENABLE_LANGUAGES_LINKS}" \
--images="${ENABLE_IMAGES}" \
--articles="${ARTICLES_PER_BLOCK}" \
@@ -192,7 +194,7 @@ render: check-dirs check-fonts check-html stamp-PinyinTable.py
.PHONY: combine
combine: check-dirs
- ./combine_idx.py ${VERBOSE_ARG} --prefix="${PREFIX}"
+ ./combine_idx.py ${VERBOSE_ARG} --prefix="${INDEX_PREFIX}" --output="${DATA_PREFIX}.idx"
# sudo apt-get install python-dev
.PHONY: pylzma
View
8 host-tools/offline-renderer/combine_idx.py
@@ -22,7 +22,8 @@ def usage(message):
print('usage: {0:s} <options>'.format(os.path.basename(__file__)))
print(' --help This message')
print(' --verbose Enable verbose output')
- print(' --prefix=name Device file name portion for .idx [pedia]')
+ print(' --prefix=name Directory and file name portion for .idx-tmp files [pedia]')
+ print(' --output=name Directory and file name portion for .idx file [pedia.idx]')
exit(1)
def main():
@@ -31,7 +32,7 @@ def main():
global UINT32_SIZE
try:
- opts, args = getopt.getopt(sys.argv[1:], 'hvo:f:p:', ['help', 'verbose', 'out=', 'offsets=', 'prefix='])
+ opts, args = getopt.getopt(sys.argv[1:], 'hvo:f:p:', ['help', 'verbose', 'output=', 'prefix='])
except getopt.GetoptError, err:
usage(err)
@@ -47,7 +48,8 @@ def main():
off_name = arg
elif opt in ('-p', '--prefix'):
in_format = arg + '{0:d}.idx-tmp'
- out_name = arg + '.idx'
+ elif opt in ('-o', '--output'):
+ out_name = arg
else:
usage('unhandled option: ' + opt)
View
6 scripts/Copy
@@ -106,7 +106,7 @@ work="${work}/${language}${suffix}"
dest="${dest}/${language}${suffix}"
# extract numeric suffix from host name
-# expect that the rendering hosts are numbered from zero
+# expect that the rendering hosts are numbered from one
this_host=$(hostname --short)
this_host_prefix=${this_host%%[0-9]}
this_id=${this_host##*[^0-9]}
@@ -117,6 +117,7 @@ this_id=${this_host##*[^0-9]}
if [ X"${this_host}" != X"${to}" ]
then
d="${to}:${base_dir}/${dest}"
+ w="${to}:${base_dir}/${work}"
case "${verbose}" in
[yY][eE][sS])
echo copying files to: ${d}
@@ -124,7 +125,8 @@ then
*)
;;
esac
- rsync -cavHx --progress "${dest}"/*.dat "${dest}"/*.idx-tmp "${d}/"
+ rsync -cavHx --progress "${dest}"/*.dat "${d}/"
+ rsync -cavHx --progress "${work}"/*.idx-tmp "${w}/"
else
echo warning: transfer to self ignored
fi
Please sign in to comment.
Something went wrong with that request. Please try again.