Merge pull request #243 from hwhw/master

use libjpeg-turbo, use common libs with crengine

.gitmodules

@@ -22,9 +22,6 @@
 [submodule "freetype2"]
 	path = freetype2
 	url = git://git.sv.gnu.org/freetype/freetype2.git
-[submodule "jpeg"]
-	path = jpeg
-	url = git://git.ghostscript.com/thirdparty/jpeg.git
 [submodule "luasocket"]
 	path = luasocket
 	url = https://github.com/diegonehab/luasocket

.travis.yml

@@ -14,13 +14,14 @@ before_install:
     - sudo apt-get update
 
 install:
-    - sudo apt-get install libsdl1.2-dev luarocks
+    - sudo apt-get install libsdl1.2-dev luarocks nasm
     - git clone https://github.com/Olivine-Labs/busted/
     - cd busted && git checkout v1.10.0 && sudo luarocks make && cd ..
 
 script:
-    - make fetchthirdparty
-    - make
+    - make fetchthirdparty all
+
+after_success:
     - wget https://tesseract-ocr.googlecode.com/files/tesseract-ocr-3.02.eng.tar.gz
     - tar zxf tesseract-ocr-3.02.eng.tar.gz
     - cd build/* && export TESSDATA_PREFIX=`pwd`/data && mkdir -p data/tessdata

Makefile

@@ -78,14 +78,14 @@ $(FREETYPE_LIB):
 	-$(MAKE) -C $(FREETYPE_DIR)/build --silent install
 	cp -fL $(FREETYPE_DIR)/build/$(if $(WIN32),bin,lib)/$(notdir $(FREETYPE_LIB)) $@
 
-# libjpeg, fetched via GIT as a submodule
+# libjpeg-turbo, fetched via subversion
 $(JPEG_LIB):
 	cd $(JPEG_DIR) && \
-		CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" \
-		CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" \
-		./configure -q --disable-static --enable-shared \
-			--host=$(CHOST)
-	$(MAKE) -j$(PROCESSORS) -C $(JPEG_DIR) --silent
+		CC="$(CC)" CXX="$(CXX)" CPPFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" \
+		./configure -q --prefix=$(CURDIR)/$(JPEG_DIR) \
+			--host=$(if $(ANDROID),"arm-linux",$(CHOST)) \
+			--disable-static --enable-shared --with-jpeg8
+	$(MAKE) -j$(PROCESSORS) -C $(JPEG_DIR) --silent install
 	cp -fL $(JPEG_DIR)/.libs/$(notdir $(JPEG_LIB)) $@
 
 # libpng, fetched via GIT as a submodule
@@ -109,7 +109,7 @@ $(MUPDF_LIB): $(JPEG_LIB) $(FREETYPE_LIB)
 		OS=$(if $(WIN32),,Other) verbose=1
 	$(MAKE) -j$(PROCESSORS) -C mupdf \
 		LDFLAGS="$(LDFLAGS) -L../$(OUTPUT_DIR)" \
-		XCFLAGS="$(CFLAGS) -DNOBUILTINFONT -I../jpeg -I../$(FREETYPE_DIR)/include" \
+		XCFLAGS="$(CFLAGS) -DNOBUILTINFONT -I../$(JPEG_DIR)/include -I../$(FREETYPE_DIR)/include" \
 		CC="$(CC)" \
 		build="release" MUDRAW= MUTOOL= CURL_LIB= \
 		OS=$(if $(WIN32),,Other) verbose=1 \
@@ -144,13 +144,19 @@ $(DJVULIBRE_LIB): $(JPEG_LIB)
 		$(DJVULIBRE_LIB)
 
 # crengine, fetched via GIT as a submodule
-$(CRENGINE_LIB): $(ZLIB) $(PNG_LIB) $(FREETYPE_LIB)
+$(CRENGINE_LIB): $(ZLIB) $(PNG_LIB) $(FREETYPE_LIB) $(JPEG_LIB)
 	test -e $(CRENGINE_WRAPPER_DIR)/build \
 	|| mkdir $(CRENGINE_WRAPPER_DIR)/build
 	cd $(CRENGINE_WRAPPER_DIR)/build \
-	&& CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
-		LDFLAGS="$(LDFLAGS) -L$(CURDIR)/$(FREETYPE_DIR)/build/lib -L$(CURDIR)/$(ZLIB_DIR) -L$(CURDIR)/$(PNG_DIR)/lib" \
-		cmake -DCMAKE_BUILD_TYPE=Release ..
+	&& CC="$(CC)" CXX="$(CXX)" RC="$(RC)" \
+		CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
+		JPEG_LIB="$(CURDIR)/$(JPEG_LIB)" \
+		PNG_LIB="$(CURDIR)/$(PNG_LIB)" \
+		FREETYPE_LIB="$(CURDIR)/$(FREETYPE_LIB)" \
+		ZLIB="$(CURDIR)/$(ZLIB)" \
+		LIBS_DIR="$(CURDIR)/$(OUTPUT_DIR)/libs" \
+		cmake -DCMAKE_BUILD_TYPE=Release \
+		$(if $(WIN32),-DCMAKE_SYSTEM_NAME=Windows,) ..
 	cd $(CRENGINE_WRAPPER_DIR)/build &&  $(MAKE)
 	cp -fL $(CRENGINE_WRAPPER_DIR)/build/$(notdir $(CRENGINE_LIB)) \
 		$(CRENGINE_LIB)
@@ -251,10 +257,8 @@ $(OUTPUT_DIR)/libs/libkoreader-cre.so: cre.cpp \
 
 $(OUTPUT_DIR)/libs/libwrap-mupdf.so: wrap-mupdf.c \
 			$(MUPDF_LIB)
-	# Bionic's C library comes with its own pthread implementation
-	# So we need not to load pthread library for Android build
 	$(CC) -I$(MUPDF_DIR)/include $(DYNLIB_CFLAGS) \
-		-o $@ $^ $(if $(ANDROID),,-lpthread)
+		-o $@ $^
 
 # ===========================================================================
 # the attachment extraction tool:
@@ -559,7 +563,11 @@ fetchthirdparty:
 	[ `md5sum libpng-1.6.12.tar.gz |cut -d\  -f1` != 297388a6746a65a2127ecdeb1c6e5c82 ] \
 		&& rm libpng-1.6.12.tar.gz && wget http://download.sourceforge.net/libpng/libpng-1.6.12.tar.gz || true
 	tar zxf libpng-1.6.12.tar.gz
-
+	# download libjpeg-turbo
+	[ ! -f libjpeg-turbo-1.3.1.tar.gz ] \
+		&& wget http://download.sourceforge.net/libjpeg-turbo/libjpeg-turbo-1.3.1.tar.gz || true
+	[ `md5sum libjpeg-turbo-1.3.1.tar.gz |cut -d\  -f1` != 2c3a68129dac443a72815ff5bb374b05 ] \
+		&& rm libpng-1.6.12.tar.gz && false || tar zxf libjpeg-turbo-1.3.1.tar.gz
 
 # ===========================================================================
 clean:
@@ -570,7 +578,7 @@ clean:
 	-$(MAKE) -C $(MUPDF_DIR) build="release" clean
 	-$(MAKE) -C $(POPEN_NOSHELL_DIR) clean
 	-$(MAKE) -C $(K2PDFOPT_DIR) clean
-	-$(MAKE) -C $(JPEG_DIR) distclean
+	-$(MAKE) -C $(JPEG_DIR) clean uninstall
 	-$(MAKE) -C $(SDCV_DIR) clean
 	-$(MAKE) -C $(PNG_DIR) clean uninstall
 	-$(MAKE) -C $(GLIB_DIR) clean uninstall

Makefile.defs

@@ -67,6 +67,7 @@ endif
 STRIP:=$(CHOST)-strip
 AR:=$(CHOST)-ar
 LD:=$(CHOST)-ld
+RC:=$(CHOST)-windres
 RANLIB:=$(CHOST)-ranlib
 
 HOSTAR:=ar
@@ -245,13 +246,13 @@ MUPDF_JS_LIB_STATIC=$(MUPDF_LIB_DIR)/libmujs.a
 MUPDF_THIRDPARTY_LIBS=$(MUPDF_LIB_DIR)/libopenjpeg.a \
 			$(MUPDF_LIB_DIR)/libjbig2dec.a \
 			$(Z_LIB)
-MUPDF_LIB=$(OUTPUT_DIR)/libs/$(if $(WIN32),libmupdf.dll,libmupdf.so)
+MUPDF_LIB=$(OUTPUT_DIR)/libs/libmupdf.$(if $(WIN32),dll,so)
 
 DJVULIBRE_DIR=djvulibre
 DJVULIBRE_LIB=$(OUTPUT_DIR)/libs/libdjvulibre$(if $(WIN32),-21.dll,.so.21)
 DJVULIBRE_LIB_DIR=$(DJVULIBRE_DIR)/build/libdjvu/.libs/
 
-CRENGINE_LIB=$(OUTPUT_DIR)/libs/libcrengine.so
+CRENGINE_LIB=$(OUTPUT_DIR)/libs/libcrengine.$(if $(WIN32),dll,so)
 
 LUA_DIR=luajit-2.0
 LUAJIT=$(OUTPUT_DIR)/$(if $(WIN32),luajit.exe,luajit)
@@ -270,8 +271,8 @@ K2PDFOPT_CFLAGS=-I$(K2PDFOPT_DIR)/willuslib \
 FREETYPE_LIB=$(OUTPUT_DIR)/libs/libfreetype$(if $(WIN32),-6.dll,.so.6)
 FREETYPE_DIR=freetype2
 
-JPEG_LIB=$(OUTPUT_DIR)/libs/libjpeg$(if $(WIN32),-9.dll,.so.9)
-JPEG_DIR=jpeg
+JPEG_LIB=$(OUTPUT_DIR)/libs/libjpeg$(if $(WIN32),-8.dll,.so.8)
+JPEG_DIR=libjpeg-turbo-1.3.1
 
 PNG_LIB=$(OUTPUT_DIR)/libs/libpng16$(if $(WIN32),-16.dll,.so.16)
 PNG_DIR=libpng-1.6.12

ffi/mupdf.lua

@@ -10,16 +10,16 @@ LuaJIT's FFI.
 
 local ffi = require("ffi")
 require("ffi/mupdf_h")
-require("ffi/pthread_h")
 
 local BlitBuffer = require("ffi/blitbuffer")
 
 local M
 if ffi.os == "Windows" then
-    M = ffi.load("libs/libwrap-mupdf.dll")
+    M = ffi.load("libs/libmupdf.dll")
 else
-    M = ffi.load("libs/libwrap-mupdf.so")
+    M = ffi.load("libs/libmupdf.so")
 end
+local W = ffi.load("libs/libwrap-mupdf.so")
 
 local mupdf = {
     debug_memory = false,
@@ -44,8 +44,8 @@ mupdf.debug = print_debug
 local function merror(message)
     if ctx ~= nil then
         error(string.format("%s: %s (%d)", message,
-            ffi.string(M.mupdf_error_message(context())),
-            M.mupdf_error_code(context())))
+            ffi.string(W.mupdf_error_message(context())),
+            W.mupdf_error_code(context())))
     else
         error(message)
     end
@@ -57,7 +57,7 @@ local function context()
     if save_ctx ~= nil then return save_ctx end
 
     local context = M.fz_new_context_imp(
-        mupdf.debug_memory and M.mupdf_get_my_alloc_context() or nil,
+        mupdf.debug_memory and W.mupdf_get_my_alloc_context() or nil,
         nil,
         mupdf.cache_size, FZ_VERSION)
 
@@ -83,7 +83,7 @@ function mupdf.openDocument(filename, cache_size)
     M.fz_register_document_handlers(context())
 
     local mupdf_doc = {
-        doc = M.mupdf_open_document(context(), filename),
+        doc = W.mupdf_open_document(context(), filename),
         filename = filename,
     }
 
@@ -136,7 +136,7 @@ function document_mt.__index:getPages()
     -- cache number of pages
     if self.number_of_pages then return self.number_of_pages end
 
-    local pages = M.mupdf_count_pages(context(), self.doc)
+    local pages = W.mupdf_count_pages(context(), self.doc)
     if pages == -1 then
 	    merror("cannot access page tree")
     end
@@ -176,7 +176,7 @@ Returns an empty table when there is no ToC
 --]]
 function document_mt.__index:getToc()
     local toc = {}
-    local outline = M.mupdf_load_outline(context(), self.doc)
+    local outline = W.mupdf_load_outline(context(), self.doc)
     if outline ~= nil then
         ffi.gc(outline, wrap_free_outline)
         toc_walker(toc, outline, 1)
@@ -189,7 +189,7 @@ open a page, return page object
 --]]
 function document_mt.__index:openPage(number)
     local mupdf_page = {
-        page = M.mupdf_load_page(context(), self.doc, number-1),
+        page = W.mupdf_load_page(context(), self.doc, number-1),
         number = number,
         doc = self,
     }
@@ -207,7 +207,7 @@ This will return sensible values only when the debug_memory flag is set
 --]]
 function document_mt.__index:getCacheSize()
     if mupdf.debug_memory then
-        return M.mupdf_get_cache_size()
+        return W.mupdf_get_cache_size()
     else
         return 0
     end
@@ -228,7 +228,7 @@ function document_mt.__index:writeDocument(filename)
 	opts[0].do_garbage = 0
 	opts[0].do_linear = 0
 	opts[0].continue_on_error = 1
-	local ok = M.mupdf_write_document(self.doc, filename, opts)
+	local ok = W.mupdf_write_document(self.doc, filename, opts)
     if ok == nil then merror("could not write document") end
 end
 
@@ -272,10 +272,10 @@ check which part of the page actually contains content
 function page_mt.__index:getUsedBBox()
     local result = ffi.new("fz_rect[1]")
 
-    local dev = M.mupdf_new_bbox_device(context(), result)
+    local dev = W.mupdf_new_bbox_device(context(), result)
     if dev == nil then merror("cannot allocate bbox_device") end
     ffi.gc(dev, M.fz_free_device)
-	local ok = M.mupdf_run_page(context(), self.doc.doc, self.page, dev, M.fz_identity, nil)
+	local ok = W.mupdf_run_page(context(), self.doc.doc, self.page, dev, M.fz_identity, nil)
     if ok == nil then merror("cannot calculate bbox for page") end
 
     return result[0].x0, result[0].y0, result[0].x1, result[0].y1
@@ -365,17 +365,17 @@ will return an empty table if we have no text
 --]]
 function page_mt.__index:getPageText()
     -- first, we run the page through a special device, the text_device
-    local text_page = M.mupdf_new_text_page(context())
+    local text_page = W.mupdf_new_text_page(context())
     if text_page == nil then merror("cannot alloc text_page") end
     ffi.gc(text_page, wrap_free_text_page)
-    local text_sheet = M.mupdf_new_text_sheet(context())
+    local text_sheet = W.mupdf_new_text_sheet(context())
     if text_sheet == nil then merror("cannot alloc text_sheet") end
     ffi.gc(text_sheet, wrap_free_text_sheet)
-    local tdev = M.mupdf_new_text_device(context(), text_sheet, text_page)
+    local tdev = W.mupdf_new_text_device(context(), text_sheet, text_page)
     if tdev == nil then merror("cannot alloc text device") end
     ffi.gc(tdev, M.fz_free_device)
 
-    if M.mupdf_run_page(context(), self.doc.doc, self.page, tdev, M.fz_identity, nil) == nil then
+    if W.mupdf_run_page(context(), self.doc.doc, self.page, tdev, M.fz_identity, nil) == nil then
         merror("cannot run page through text device")
     end
 
@@ -457,7 +457,7 @@ end
 Get a list of the Hyperlinks on a page
 --]]
 function page_mt.__index:getPageLinks()
-	local page_links = M.mupdf_load_links(context(), self.doc.doc, self.page)
+	local page_links = W.mupdf_load_links(context(), self.doc.doc, self.page)
     -- do not error out when page_links == NULL, since there might
     -- simply be no links present.
 
@@ -491,11 +491,11 @@ end
 local function run_page(page, pixmap, ctm)
 	M.fz_clear_pixmap_with_value(context(), pixmap, 0xff)
 
-	local dev = M.mupdf_new_draw_device(context(), pixmap)
+	local dev = W.mupdf_new_draw_device(context(), pixmap)
     if dev == nil then merror("cannot create draw device") end
     ffi.gc(dev, M.fz_free_device)
 
-	local ok = M.mupdf_run_page(context(), page.doc.doc, page.page, dev, ctm, nil)
+	local ok = W.mupdf_run_page(context(), page.doc.doc, page.page, dev, ctm, nil)
     if ok == nil then merror("could not run page") end
 end
 --[[
@@ -531,7 +531,7 @@ function page_mt.__index:draw_new(draw_context, width, height, offset_x, offset_
 
     local colorspace = mupdf.color and M.fz_device_rgb(context())
         or M.fz_device_gray(context())
-	local pix = M.mupdf_new_pixmap_with_bbox_and_data(
+	local pix = W.mupdf_new_pixmap_with_bbox_and_data(
         context(), colorspace, bbox, ffi.cast("unsigned char*", bb.data))
     if pix == nil then merror("cannot allocate pixmap") end
     ffi.gc(pix, wrap_drop_pixmap)
@@ -578,12 +578,12 @@ function page_mt.__index:addMarkupAnnotation(points, n, type)
 	local doc = M.pdf_specifics(self.doc.doc)
     if doc == nil then merror("could not get pdf_specifics") end
 
-    local annot = M.mupdf_pdf_create_annot(context(), self.doc.doc, self.page, type)
+    local annot = W.mupdf_pdf_create_annot(context(), self.doc.doc, self.page, type)
     if annot == nil then merror("could not create annotation") end
 
-    local ok = M.mupdf_pdf_set_markup_annot_quadpoints(context(), self.doc.doc, annot, pts, n)
+    local ok = W.mupdf_pdf_set_markup_annot_quadpoints(context(), self.doc.doc, annot, pts, n)
     if ok == nil then merror("could not set markup annot quadpoints") end
-    local ok = M.mupdf_pdf_set_markup_appearance(context(), self.doc.doc, annot, color, alpha, line_thickness, line_height)
+    local ok = W.mupdf_pdf_set_markup_appearance(context(), self.doc.doc, annot, color, alpha, line_thickness, line_height)
     if ok == nil then merror("could not set markup appearance") end
 end
 
@@ -597,12 +597,12 @@ end
 render image data
 --]]
 function mupdf.renderImage(data, size, width, height)
-    local image = M.mupdf_new_image_from_data(context(),
+    local image = W.mupdf_new_image_from_data(context(),
                     ffi.cast("unsigned char*", data), size)
     if image == nil then merror("could not load image data") end
     M.fz_keep_image(context(), image)
     ffi.gc(image, wrap_drop_image)
-    local pixmap = M.mupdf_new_pixmap_from_image(context(),
+    local pixmap = W.mupdf_new_pixmap_from_image(context(),
                     image, width or -1, height or -1)
     if pixmap == nil then merror("could not create pixmap from image") end
     ffi.gc(pixmap, wrap_drop_pixmap)
@@ -647,6 +647,25 @@ local function get_k2pdfopt()
     return k2pdfopt
 end
 
+-- lazily load libpthread
+local pthread
+local function get_pthread()
+    if pthread then return pthread end
+
+    local util = require("ffi/util")
+
+    require("ffi/pthread_h")
+
+    if ffi.os == "Windows" then
+        return ffi.load("libwinpthread-1.dll")
+    elseif util.isAndroid() then
+        -- pthread directives are in the default namespace on Android
+        return ffi.C
+    else
+        return ffi.load("pthread")
+    end
+end
+
 --[[
 the following function is a reimplementation of what can be found
 in libk2pdfopt/willuslib/bmpmupdf.c
@@ -713,7 +732,7 @@ local function render_for_kopt(bmp, page, scale, bounds)
 
     local colorspace = mupdf.color and M.fz_device_rgb(context())
         or M.fz_device_gray(context())
-	local pix = M.mupdf_new_pixmap_with_bbox(context(), colorspace, bbox)
+	local pix = W.mupdf_new_pixmap_with_bbox(context(), colorspace, bbox)
     if pix == nil then merror("could not allocate pixmap") end
     ffi.gc(pix, wrap_drop_pixmap)
 
@@ -746,12 +765,13 @@ function page_mt.__index:reflow(kopt_context)
     render_for_kopt(kopt_context.src, self, scale, bounds)
 
 	if kopt_context.precache ~= 0 then
-		local rf_thread = ffi.new("pthread_t[1]")
+        local pthread = get_pthread()
+        local rf_thread = ffi.new("pthread_t[1]")
         local attr = ffi.new("pthread_attr_t")
-        M.pthread_attr_init(attr)
-        M.pthread_attr_setdetachstate(attr, M.PTHREAD_CREATE_DETACHED)
-        M.pthread_create(rf_thread, attr, k2pdfopt.k2pdfopt_reflow_bmp, ffi.cast("void*", kopt_context))
-        M.pthread_attr_destroy(attr)
+        pthread.pthread_attr_init(attr)
+        pthread.pthread_attr_setdetachstate(attr, pthread.PTHREAD_CREATE_DETACHED)
+        pthread.pthread_create(rf_thread, attr, k2pdfopt.k2pdfopt_reflow_bmp, ffi.cast("void*", kopt_context))
+        pthread.pthread_attr_destroy(attr)
 	else
 		k2pdfopt.k2pdfopt_reflow_bmp(kopt_context)
 	end