Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

[Makefile,scripts] Allow re-index operation

Since parsing is very slow allow the parse stamps to be retouched (if
they already exist).  This is so that it is possible to re run the
index process and then the rendering process without having to regenerate
the HTML files.

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information...
commit 77b2b07d6022618b2cb1d1fa7758982673af1ad3 1 parent 4450b7d
@hxw hxw authored
View
9 Makefile
@@ -495,6 +495,12 @@ stamp-clean${1}: stamp-parse-clean${1} stamp-render-clean${1}
stamp-parse-clean${1}:
${RM} "${PARSE_STAMP}${1}"
+# this is to reuse the existing parsed html after a re-index
+# (but only if the stamps already exist)
+.PHONY: stamp-parse-touch${1}
+stamp-parse-touch${1}:
+ [ -e "${PARSE_STAMP}${1}" ] && ${TOUCH} "${PARSE_STAMP}${1}"
+
.PHONY: stamp-render-clean${1}
stamp-render-clean${1}:
${RM} "${RENDER_STAMP}${1}"
@@ -523,6 +529,9 @@ farm${1}-clean: farm${1}-parse-clean farm${1}-render-clean
.PHONY: farm${1}-parse-clean
farm${1}-parse-clean: $$(foreach i,${2},stamp-parse-clean$$(strip $${i}))
+.PHONY: farm${1}-parse-touch
+farm${1}-parse-touch: $$(foreach i,${2},stamp-parse-touch$$(strip $${i}))
+
.PHONY: farm${1}-render-clean
farm${1}-render-clean: $$(foreach i,${2},stamp-render-clean$$(strip $${i}))
View
7 host-tools/offline-renderer/Makefile
@@ -85,11 +85,12 @@ ifeq (YES,$(strip ${VERBOSE}))
VERBOSE_ARG = --verbose
endif
-TRUNCATE_ARG =
+TRUNCATE_ARG = --truncate-title
ifneq (yes,$(strip ${ENABLE_LANGUAGE_LINKS}))
-ifneq (YES,$(strip ${ENABLE_LANGUAGE_LINKS}))
-TRUNCATE_ARG += --truncate-title
+TRUNCATE_ARG =
endif
+ifneq (YES,$(strip ${ENABLE_LANGUAGE_LINKS}))
+TRUNCATE_ARG =
endif
View
32 scripts/Run
@@ -24,7 +24,7 @@ USAGE()
echo ' --machines=m -m <m> set machine count [${MACHINE_COUNT} or 9]'
echo ' --parallel=n -j <n> set make -j value [${PARALLEL_BUILD} or 3]'
echo ' --language=xx -l <xx> set language [en]'
- echo ' --suffix=<s> -s <s> set the language suffix [pedia]'
+ echo ' --suffix=<s> -s <s> set the directory suffix [pedia]'
echo ' --prefix=name -p <name> set data file prefix [wiki]'
echo ' --work=dir -w <dir> workdir [work]'
echo ' --dest=dir -d <dir> destdir [image]'
@@ -109,6 +109,7 @@ do
-i|--index-only)
IndexOnly=yes
+ clear=index
shift
;;
@@ -242,15 +243,44 @@ common_opts="${common_opts} WIKI_LANGUAGE='${language}'"
common_opts="${common_opts} WIKI_DIR_SUFFIX='${suffix}'"
common_opts="${common_opts} XML_FILES='${xml}'"
common_opts="${common_opts} VERBOSE='${verbose}'"
+case "${language}" in
+ en)
+ # low compression for 8GB card
+ #common_opts="${common_opts} ENABLE_LANGUAGES_LINKS=YES"
+ #common_opts="${common_opts} ARTICLES_PER_BLOCK=32"
+ #common_opts="${common_opts} ARTICLE_BLOCK_SIZE=131072"
+ # Best version for 4GB card
+ common_opts="${common_opts} ENABLE_LANGUAGES_LINKS=NO"
+
+ ## ok
+ #common_opts="${common_opts} ARTICLES_PER_BLOCK=64"
+ #common_opts="${common_opts} ARTICLE_BLOCK_SIZE=250000"
+
+ ## ok
+ common_opts="${common_opts} ARTICLES_PER_BLOCK=64"
+ common_opts="${common_opts} ARTICLE_BLOCK_SIZE=200000"
+
+ ;;
+
+ *)
+ common_opts="${common_opts} ENABLE_LANGUAGES_LINKS=YES"
+ common_opts="${common_opts} ARTICLES_PER_BLOCK=16"
+ common_opts="${common_opts} ARTICLE_BLOCK_SIZE=131072"
+ ;;
+esac
# clean up
case "${clear}" in
[yY]|[yY][eE][sS])
RUN make index-clean "${farm}-clean" cleandirs ${common_opts}
;;
+ [iI][nN][dD][eE][xX])
+ RUN make index-clean ${common_opts}
+ ;;
[rR][eE][nN][dD][eE][rR])
RUN make "${farm}-render-clean" ${common_opts}
+ RUN make "${farm}-parse-touch" ${common_opts}
;;
[pP][aA][rR][sS][eE])
RUN make "${farm}-clean" ${common_opts}
View
9 scripts/progress
@@ -25,6 +25,7 @@ USAGE()
echo ' --language=<xx> -L <xx> process for language [en]'
echo ' --suffix=<s> -S <s> set the language suffix [pedia]'
echo ' --run -r start the Run script in a screen {index, parse, render}'
+ echo ' --re-index -I start the Run script in a screen {index-only}'
echo ' --re-render -R start the Run script in a screen {re-render}'
echo ' --copy -c run the copy process i.e. {Copy --to=8 -l <lang>}'
echo ' --combine -k run the combine process'
@@ -48,7 +49,7 @@ base_dir=samo
getopt=/usr/local/bin/getopt
[ -x "${getopt}" ] || getopt=getopt
-args=$(${getopt} -o hvb:swigp::t::aml:L:S:rRckG4Nfx --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,suffix:,run,re-render,copy,combine,get,sim4,new-session,fetch-log,xml-link -- "$@") || exit 1
+args=$(${getopt} -o hvb:swigp::t::aml:L:S:rRIckG4Nfx --long=help,verbose,base:,stamps,work,image,git,process::,tail::,analyse,memory,list:,language:,suffix:,run,re-render,re-index,copy,combine,get,sim4,new-session,fetch-log,xml-link -- "$@") || exit 1
# replace the arguments with the parsed values
eval set -- "${args}"
@@ -95,6 +96,12 @@ do
shift
;;
+ -I|--re-index)
+ c="scripts/Run --index-only --language='${language}' --suffix='${suffix}'"
+ RenderDo --yes ${list} --brief "byobu -X eval chdir 'chdir ${base_dir}' 'exec ${c}'"
+ shift
+ ;;
+
-c|--copy)
c="scripts/Copy --base='${base_dir}' --to='${combine_host}' --language='${language}' --suffix='${suffix}'"
RenderDo --yes ${list} --brief "byobu -X eval chdir 'chdir ${base_dir}' 'exec ${c}'"
Please sign in to comment.
Something went wrong with that request. Please try again.