Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

[Makefiles] add merge operation

This is an optional step between parse and render.
it reduces the number of article html files to the value specified
by TOTAL_HTML_FILES on the make command line.

This allows for the possibliity of having more than 64 parsing
instances thne merging the generated files back to 64 or less
for the rendering stage.

Signed-off-by: Christopher Hall <hsw@openmoko.com>
  • Loading branch information...
commit dda9f1e196304e8bbeac12bf01fb06fcc754935a 1 parent 4edf800
@hxw hxw authored
Showing with 54 additions and 1 deletion.
  1. +13 −0 Makefile
  2. +41 −1 host-tools/offline-renderer/Makefile
View
13 Makefile
@@ -277,12 +277,14 @@ WIKI_LANGUAGE ?= en
WIKI_FILE_PREFIX ?= wiki
XML_FILES_PATH := $(foreach f,${XML_FILES},$(shell ${REALPATH} "${f}"))
RENDER_BLOCK ?= 0
+TOTAL_HTML_FILES ?= 27
.PHONY: index
index: validate-destdir
cd ${HOST_TOOLS}/offline-renderer && ${MAKE} index \
WIKI_LANGUAGE="${WIKI_LANGUAGE}" WIKI_FILE_PREFIX="${WIKI_FILE_PREFIX}" \
XML_FILES="${XML_FILES_PATH}" RENDER_BLOCK="${RENDER_BLOCK}" \
+ TOTAL_HTML_FILES="${TOTAL_HTML_FILES}" \
WORKDIR="${WORKDIR_PATH}" DESTDIR="${DESTDIR_PATH}"
.PHONY: parse
@@ -290,6 +292,15 @@ parse: validate-destdir
cd ${HOST_TOOLS}/offline-renderer && ${MAKE} parse \
WIKI_LANGUAGE="${WIKI_LANGUAGE}" WIKI_FILE_PREFIX="${WIKI_FILE_PREFIX}" \
XML_FILES="${XML_FILES_PATH}" RENDER_BLOCK="${RENDER_BLOCK}" \
+ TOTAL_HTML_FILES="${TOTAL_HTML_FILES}" \
+ WORKDIR="${WORKDIR_PATH}" DESTDIR="${DESTDIR_PATH}"
+
+.PHONY: merge
+merge: validate-destdir
+ cd ${HOST_TOOLS}/offline-renderer && ${MAKE} merge \
+ WIKI_LANGUAGE="${WIKI_LANGUAGE}" WIKI_FILE_PREFIX="${WIKI_FILE_PREFIX}" \
+ XML_FILES="${XML_FILES_PATH}" RENDER_BLOCK="${RENDER_BLOCK}" \
+ TOTAL_HTML_FILES="${TOTAL_HTML_FILES}" \
WORKDIR="${WORKDIR_PATH}" DESTDIR="${DESTDIR_PATH}"
.PHONY: render
@@ -297,6 +308,7 @@ render: fonts validate-destdir
cd ${HOST_TOOLS}/offline-renderer && ${MAKE} render \
WIKI_LANGUAGE="${WIKI_LANGUAGE}" WIKI_FILE_PREFIX="${WIKI_FILE_PREFIX}" \
XML_FILES="${XML_FILES_PATH}" RENDER_BLOCK="${RENDER_BLOCK}" \
+ TOTAL_HTML_FILES="${TOTAL_HTML_FILES}" \
WORKDIR="${WORKDIR_PATH}" DESTDIR="${DESTDIR_PATH}"
.PHONY: combine
@@ -304,6 +316,7 @@ combine: validate-destdir
cd ${HOST_TOOLS}/offline-renderer && ${MAKE} combine \
WIKI_LANGUAGE="${WIKI_LANGUAGE}" WIKI_FILE_PREFIX="${WIKI_FILE_PREFIX}" \
XML_FILES="${XML_FILES_PATH}" RENDER_BLOCK="${RENDER_BLOCK}" \
+ TOTAL_HTML_FILES="${TOTAL_HTML_FILES}" \
WORKDIR="${WORKDIR_PATH}" DESTDIR="${DESTDIR_PATH}"
.PHONY: hash
View
42 host-tools/offline-renderer/Makefile
@@ -39,6 +39,7 @@ include ${MK_DIR}/definitions.mk
RENDER_BLOCK ?= 0
START ?= 1
COUNT ?= all
+TOTAL_HTML_FILES ?= 64
DESTDIR ?= .
FONT_PATH ?= $(shell ${REALPATH} ${HOST_TOOLS}/fonts)
WORKDIR ?= /tmp
@@ -50,7 +51,8 @@ ARTICLES := $(shell ${RESOLVEPATH} ${WORKDIR}/articles.db)
COUNTS_FILE := $(shell ${RESOLVEPATH} ${WORKDIR}/counts.text)
TEMPLATE_FILE := $(shell ${RESOLVEPATH} ${WORKDIR}/templates.db)
OFFSETS := $(shell ${RESOLVEPATH} ${WORKDIR}/offsets.db)
-HTML_ARTICLES := $(shell ${RESOLVEPATH} ${WORKDIR}/articles-${RENDER_BLOCK}.html)
+HTML_ARTICLES_PREFIX := $(shell ${RESOLVEPATH} ${WORKDIR}/articles-)
+HTML_ARTICLES := $(shell ${RESOLVEPATH} ${HTML_ARTICLES_PREFIX}${RENDER_BLOCK}.html)
PYLZMA_DIR := pylzma-0.3.0
MATH_DIR := mediawiki-offline/math
@@ -106,6 +108,44 @@ index: check-dirs check-xml RedirectedTo.py
--templates="${TEMPLATE_FILE}" \
--prefix="${PREFIX}" ${XML_FILES}
+HTML_FILES_COUNT := $(words $(wildcard ${WORKDIR}/*.html))
+.PHONY: merge
+merge:
+ifeq (0,$(strip $(shell [ "${HTML_FILES_COUNT}" -gt "${TOTAL_HTML_FILES}" ] ; echo $$?)))
+ @echo merging: ${HTML_FILES_COUNT} to: ${TOTAL_HTML_FILES}
+ @${RM} ${HTML_ARTICLES_PREFIX}*.html-UNMERGED
+ @i=0 ; \
+ while [ $${i} -lt ${HTML_FILES_COUNT} ] ; \
+ do \
+ f=${HTML_ARTICLES_PREFIX}$${i}.html ; \
+ mv "$${f}" "$${f}-UNMERGED" ; \
+ i=$$(($${i} + 1)) ; \
+ done
+ @i=0 ; in=0 ; out=0 ; \
+ total=${TOTAL_HTML_FILES} ; count=${HTML_FILES_COUNT} ; \
+ while [ $${out} -lt ${TOTAL_HTML_FILES} ] ; \
+ do \
+ step=$$(expr '(' 2 '*' $${count} / $${total} + 1 ')' / 2) ; \
+ count=$$(($${count} - $${step})) ; \
+ f_out=${HTML_ARTICLES_PREFIX}$${out}.html ; \
+ ${RM} $${f_out} ; \
+ echo -n generating: $${f_out} ; \
+ while [ $${step} -gt 0 ] ; \
+ do \
+ f=${HTML_ARTICLES_PREFIX}$${in}.html-UNMERGED ; \
+ echo -n " $${in}" ; \
+ cat "$${f}" >> "$${f_out}" ; \
+ in=$$(($${in} + 1)) ; \
+ step=$$(($${step} - 1)) ; \
+ done ; \
+ echo ; \
+ out=$$(($${out} + 1)) ; \
+ total=$$(($${total} - 1)) ; \
+ done
+else
+ @echo Merge Ignored: insufficient files: ${HTML_FILES_COUNT} '<=' ${TOTAL_HTML_FILES}
+endif
+
.PHONY: parse
parse: check-dirs check-html math RedirectedTo.py
./ArticleParser.py ${VERBOSE_ARG} --xhtml="${HTML_ARTICLES}" \
Please sign in to comment.
Something went wrong with that request. Please try again.