From ca151427adf59a3c54a0a65fe7c7f65c26dec2cc Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Fri, 8 Aug 2014 13:16:22 -0400 Subject: [PATCH] utf8proc -> libmojibake (closes #7656, fixes #7582) --- .gitmodules | 3 ++ Make.inc | 8 ++--- deps/Makefile | 67 +++++++++++++++++++----------------- deps/Versions.make | 1 - deps/libmojibake | 1 + deps/utf8proc_Makefile.patch | 19 ---------- deps/utf8proc_msvc.patch | 40 --------------------- src/Makefile | 2 +- src/flisp/Makefile | 2 +- src/flisp/Windows.mk | 10 +++--- src/flisp/julia_extensions.c | 2 +- test/core.jl | 3 ++ 12 files changed, 55 insertions(+), 103 deletions(-) create mode 160000 deps/libmojibake delete mode 100644 deps/utf8proc_Makefile.patch delete mode 100644 deps/utf8proc_msvc.patch diff --git a/.gitmodules b/.gitmodules index 54688a32a6a74..fae922d6fee39 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,6 @@ path = doc/juliadoc url = git://github.com/JuliaLang/JuliaDoc.git +[submodule "deps/libmojibake"] + path = deps/libmojibake + url = git://github.com/JuliaLang/libmojibake.git diff --git a/Make.inc b/Make.inc index df7ce23fb2577..dd8fd2dabc9b6 100644 --- a/Make.inc +++ b/Make.inc @@ -29,7 +29,7 @@ USE_SYSTEM_ZLIB=0 USE_SYSTEM_GRISU=0 USE_SYSTEM_RMATH=0 USE_SYSTEM_LIBUV=0 -USE_SYSTEM_UTF8PROC=0 +USE_SYSTEM_MOJIBAKE=0 # Link to the LLVM shared library USE_LLVM_SHLIB = 0 @@ -531,10 +531,10 @@ else LIBUV_INC = $(JULIAHOME)/deps/libuv/include endif -ifeq ($(USE_SYSTEM_UTF8PROC), 1) - LIBUTF8PROC = -lutf8proc +ifeq ($(USE_SYSTEM_MOJIBAKE), 1) + LIBMOJIBAKE = -lmojibake else - LIBUTF8PROC = $(build_libdir)/libutf8proc.a + LIBMOJIBAKE = $(build_libdir)/libmojibake.a endif # OS specific stuff diff --git a/deps/Makefile b/deps/Makefile index 53a86c813abeb..33372ab818eb7 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -19,7 +19,7 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC)" CXX="$(CXX)" MAKE_COMMON = DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$(build_libdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) #autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf uv -#custom Makefile rules: openlibm Rmath double-conversion dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc +#custom Makefile rules: openlibm Rmath double-conversion dsfmt suitesparse-wrapper suitesparse lapack openblas mojibake # prevent installing libs into usr/lib64 on opensuse unexport CONFIG_SITE @@ -111,8 +111,8 @@ ifeq ($(USE_SYSTEM_SUITESPARSE), 0) STAGE2_DEPS += suitesparse endif -ifeq ($(USE_SYSTEM_UTF8PROC), 0) -STAGE2_DEPS += utf8proc +ifeq ($(USE_SYSTEM_MOJIBAKE), 0) +STAGE2_DEPS += mojibake endif # Only compile standalone LAPACK if we are not using OpenBLAS. @@ -144,7 +144,7 @@ install: $(addprefix install-, $(LIBS)) cleanall: $(addprefix clean-, $(LIBS)) distcleanall: $(addprefix distclean-, $(LIBS)) rm -rf $(build_prefix) -getall: get-llvm get-uv get-pcre get-double-conversion get-openlibm get-openspecfun get-dsfmt get-Rmath get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-zlib get-patchelf get-utf8proc +getall: get-llvm get-uv get-pcre get-double-conversion get-openlibm get-openspecfun get-dsfmt get-Rmath get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-zlib get-patchelf get-mojibake ## PATHS ## # sort is used to remove potential duplicates @@ -1203,42 +1203,47 @@ compile-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET) check-fftw-double: fftw-$(FFTW_VER)-double/checked install-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET) -## UTF8PROC ## +## MOJIBAKE ## -UTF8PROC_OBJ_SOURCE = utf8proc-v$(UTF8PROC_VER)/libutf8proc.a -UTF8PROC_OBJ_LIB = $(build_libdir)/libutf8proc.a -UTF8PROC_OBJ_HEADER = $(build_includedir)/utf8proc.h -UTF8PROC_OBJ_TARGET = $(UTF8PROC_OBJ_LIB) $(UTF8PROC_OBJ_HEADER) +MOJIBAKE_SRC_TARGET = libmojibake/libmojibake.a +MOJIBAKE_OBJ_LIB = $(build_libdir)/libmojibake.a +MOJIBAKE_OBJ_HEADER = $(build_includedir)/mojibake.h +MOJIBAKE_OBJ_TARGET = $(MOJIBAKE_OBJ_LIB) $(MOJIBAKE_OBJ_HEADER) -utf8proc-v$(UTF8PROC_VER).tar.gz: - $(JLDOWNLOAD) $@ http://www.public-software-group.org/pub/projects/utf8proc/v$(UTF8PROC_VER)/$@ -utf8proc-v$(UTF8PROC_VER)/Makefile: utf8proc-v$(UTF8PROC_VER).tar.gz - $(JLCHECKSUM) $< - $(TAR) -xzf $< - patch $@ < utf8proc_Makefile.patch - cd utf8proc-v$(UTF8PROC_VER) && patch < ../utf8proc_msvc.patch +libmojibake/Makefile: + (cd .. && git submodule init && git submodule update) +ifeq (exists, $(shell [ -d libmojibake/.git ] && echo exists )) +$(MOJIBAKE_SRC_TARGET): libmojibake/.git/HEAD +endif +ifeq (exists, $(shell [ -d $(JULIAHOME)/.git/modules/deps/libmojibake ] && echo exists )) +$(MOJIBAKE_SRC_TARGET): $(JULIAHOME)/.git/modules/deps/libmojibake/HEAD +endif +$(MOJIBAKE_SRC_TARGET): libmojibake/Makefile + $(MAKE) -C libmojibake cc="$(CC) -O2 -std=c99 $(fPIC)" AR="$(AR)" libmojibake.a touch -c $@ +libmojibake/checked: $(MOJIBAKE_SRC_TARGET) +ifeq ($(OS),$(BUILD_OS)) + -$(MAKE) -C libmojibake check +endif + echo 1 > $@ -$(UTF8PROC_OBJ_SOURCE): utf8proc-v$(UTF8PROC_VER)/Makefile - $(MAKE) -C utf8proc-v$(UTF8PROC_VER) cc="$(CC) -O2 -std=c99 $(fPIC)" AR="$(AR)" libutf8proc.a - -$(UTF8PROC_OBJ_LIB): $(UTF8PROC_OBJ_SOURCE) +$(MOJIBAKE_OBJ_LIB): $(MOJIBAKE_SRC_TARGET) cp -f $< $@ -$(UTF8PROC_OBJ_HEADER): utf8proc-v$(UTF8PROC_VER)/Makefile - cp -f utf8proc-v$(UTF8PROC_VER)/utf8proc.h $@ +$(MOJIBAKE_OBJ_HEADER): libmojibake/Makefile + cp -f libmojibake/mojibake.h $@ -clean-utf8proc: - -$(MAKE) -C utf8proc-v$(UTF8PROC_VER) clean -distclean-utf8proc: clean-utf8proc - -rm -rf utf8proc-v$(UTF8PROC_VER).tar.gz utf8proc-v$(UTF8PROC_VER) +clean-mojibake: + -$(MAKE) -C libmojibake clean + -rm -rf $(build_libdir)/libmojibake.a $(build_includedir)/mojibake.h +distclean-mojibake: clean-mojibake -get-utf8proc: utf8proc-v$(UTF8PROC_VER).tar.gz -configure-utf8proc: get-utf8proc -compile-utf8proc: $(UTF8PROC_OBJ_SOURCE) -check-utf8proc: -install-utf8proc: $(UTF8PROC_OBJ_TARGET) +get-mojibake: libmojibake/Makefile +configure-mojibake: get-mojibake +compile-mojibake: $(MOJIBAKE_SRC_TARGET) +check-mojibake: libmojibake/checked +install-mojibake: $(MOJIBAKE_OBJ_TARGET) ## SUITESPARSE ## diff --git a/deps/Versions.make b/deps/Versions.make index 1a73a47bdcb78..754c26f0b4650 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -16,4 +16,3 @@ MPFR_VER=3.1.2 ZLIB_VER = 1.2.8 PATCHELF_VER = 0.8 GIT_VER = 1.8.2.3 -UTF8PROC_VER = 1.1.6 diff --git a/deps/libmojibake b/deps/libmojibake new file mode 160000 index 0000000000000..bc357b276f1fd --- /dev/null +++ b/deps/libmojibake @@ -0,0 +1 @@ +Subproject commit bc357b276f1fd7124bbb31a4e212a30e57520eee diff --git a/deps/utf8proc_Makefile.patch b/deps/utf8proc_Makefile.patch deleted file mode 100644 index 2e29e92ee8539..0000000000000 --- a/deps/utf8proc_Makefile.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- Makefile.old 2014-01-23 20:06:09.000000000 -0800 -+++ Makefile 2014-01-23 20:06:11.000000000 -0800 -@@ -5,6 +5,7 @@ - - cflags = -O2 -std=c99 -pedantic -Wall -fpic $(CFLAGS) - cc = $(CC) $(cflags) -+AR = ar - - - # meta targets -@@ -31,7 +32,7 @@ - - libutf8proc.a: utf8proc.o - rm -f libutf8proc.a -- ar rs libutf8proc.a utf8proc.o -+ $(AR) rs libutf8proc.a utf8proc.o - - libutf8proc.so: utf8proc.o - $(cc) -shared -o libutf8proc.so utf8proc.o diff --git a/deps/utf8proc_msvc.patch b/deps/utf8proc_msvc.patch deleted file mode 100644 index 1fc887177b44a..0000000000000 --- a/deps/utf8proc_msvc.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/utf8proc.c b/utf8proc.c -index ef2d433..e6c51f7 100644 ---- a/utf8proc.c -+++ b/utf8proc.c -@@ -536,7 +536,7 @@ ssize_t utf8proc_map( - *dstptr = NULL; - result = utf8proc_decompose(str, strlen, NULL, 0, options); - if (result < 0) return result; -- buffer = malloc(result * sizeof(int32_t) + 1); -+ buffer = (int32_t *) malloc(result * sizeof(int32_t) + 1); - if (!buffer) return UTF8PROC_ERROR_NOMEM; - result = utf8proc_decompose(str, strlen, buffer, result, options); - if (result < 0) { -@@ -550,7 +550,7 @@ ssize_t utf8proc_map( - } - { - int32_t *newptr; -- newptr = realloc(buffer, (size_t)result+1); -+ newptr = (int32_t *) realloc(buffer, (size_t)result+1); - if (newptr) buffer = newptr; - } - *dstptr = (uint8_t *)buffer; -diff --git a/utf8proc.h b/utf8proc.h -index 24a891b..304e227 100644 ---- a/utf8proc.h -+++ b/utf8proc.h -@@ -65,8 +65,13 @@ typedef int int32_t; - #else - #define ssize_t int - #endif -+#ifdef __cplusplus -+typedef unsigned char _bool; -+enum {_false, _true}; -+#else - typedef unsigned char bool; - enum {false, true}; -+#endif - #else - #include - #include diff --git a/src/Makefile b/src/Makefile index 4a4c5d87561f0..f6209366e571a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -25,7 +25,7 @@ ifeq ($(USE_LLVM_SHLIB),1) LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM-$(call exec,$(LLVM_CONFIG) --version) endif -COMMON_LIBS = -L$(build_shlibdir) -L$(build_libdir) $(LIBUV) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LLVMLINK) $(OSLIBS) +COMMON_LIBS = -L$(build_shlibdir) -L$(build_libdir) $(LIBUV) $(LIBMOJIBAKE) $(NO_WHOLE_ARCHIVE) $(LLVMLINK) $(OSLIBS) DEBUG_LIBS = $(WHOLE_ARCHIVE) $(JULIAHOME)/src/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(JULIAHOME)/src/support/libsupport-debug.a $(COMMON_LIBS) RELEASE_LIBS = $(WHOLE_ARCHIVE) $(JULIAHOME)/src/flisp/libflisp.a $(WHOLE_ARCHIVE) $(JULIAHOME)/src/support/libsupport.a $(COMMON_LIBS) diff --git a/src/flisp/Makefile b/src/flisp/Makefile index b6f11b899e351..d1cab79953edd 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -18,7 +18,7 @@ endif OBJS = $(SRCS:%.c=%.o) DOBJS = $(SRCS:%.c=%.do) LLTDIR = ../support -LLT = $(LLTDIR)/libsupport.a $(LIBUV) $(LIBUTF8PROC) +LLT = $(LLTDIR)/libsupport.a $(LIBUV) $(LIBMOJIBAKE) FLAGS = -I$(LLTDIR) $(CFLAGS) $(HFILEDIRS:%=-I%) \ -I$(LIBUV_INC) -I$(build_includedir) $(LIBDIRS:%=-L%) \ diff --git a/src/flisp/Windows.mk b/src/flisp/Windows.mk index 0abecca80c46f..444ab2aa1e8c6 100644 --- a/src/flisp/Windows.mk +++ b/src/flisp/Windows.mk @@ -31,17 +31,17 @@ OBJECTS = \ basename.obj LIBUV = $(MAKEDIR)\..\..\deps\libuv\libuv.lib -LIBUTF8PROC = $(MAKEDIR)\..\..\deps\utf8proc-v1.1.6\libutf8proc.lib +LIBMOJIBAKE = $(MAKEDIR)\..\..\deps\libmojibake\libmojibake.lib LIBSUPPORT = $(MAKEDIR)\..\support\libsupport.lib -INCLUDE = $(INCLUDE);$(MAKEDIR)\..\..\deps\libuv\include;$(MAKEDIR)\..\..\deps\utf8proc-v1.1.6;$(MAKEDIR)\..\support +INCLUDE = $(INCLUDE);$(MAKEDIR)\..\..\deps\libuv\include;$(MAKEDIR)\..\..\deps\libmojibake;$(MAKEDIR)\..\support CFLAGS = $(CFLAGS) /Qstd=c99 -D_CRT_SECURE_NO_WARNINGS -DLIBRARY_EXPORTS LFLAGS = $(LFLAGS) kernel32.lib ws2_32.lib psapi.lib advapi32.lib iphlpapi.lib default: $(NAME).exe -$(NAME).exe: lib$(NAME).lib flmain.obj $(LIBSUPPORT) $(LIBUV) $(LIBUTF8PROC) +$(NAME).exe: lib$(NAME).lib flmain.obj $(LIBSUPPORT) $(LIBUV) $(LIBMOJIBAKE) $(LINK) $(LFLAGS) /OUT:$(NAME).exe /PDB:$(NAME).pdb /MAP $** $(LIBSUPPORT): @@ -50,8 +50,8 @@ $(LIBSUPPORT): $(LIBUV): PUSHD $(MAKEDIR)\..\..\deps\libuv && $(MAKE) /NOLOGO /F Windows.mk && POPD -$(LIBUTF8PROC): - PUSHD $(MAKEDIR)\..\..\deps\utf8proc-v1.1.6 && cl -nologo /c utf8proc.c && $(AR) /OUT:libutf8proc.lib utf8proc.obj && POPD +$(LIBMOJIBAKE): + PUSHD $(MAKEDIR)\..\..\deps\libmojibake && cl -nologo /c utf8proc.c && $(AR) /OUT:libmojibake.lib utf8proc.obj && POPD lib$(NAME).lib: $(OBJECTS) $(AR) /OUT:lib$(NAME).lib $** diff --git a/src/flisp/julia_extensions.c b/src/flisp/julia_extensions.c index a2e8e568fccc7..704498232c6f2 100644 --- a/src/flisp/julia_extensions.c +++ b/src/flisp/julia_extensions.c @@ -3,7 +3,7 @@ #include #include #include "flisp.h" -#include "utf8proc.h" +#include "mojibake.h" #ifdef __cplusplus extern "C" { diff --git a/test/core.jl b/test/core.jl index f26a3d1fb5d4d..a04bf1443ac20 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1826,3 +1826,6 @@ sa = sub(a,4:6) try issue7897!(sa, zeros(10)) end + +# issue #7582 +aₜ = "a variable using Unicode 6"