Merge pull request #153 from xerial/revert-152-revert-137-SupportBitS…

…huffle

Revert "Revert "[WIP] Add bit-shuffling interfaces to improve LZ performance""

Makefile

@@ -7,17 +7,50 @@ SBT:=./sbt
 all: snappy
 
 SNAPPY_OUT:=$(TARGET)/$(snappy)-$(os_arch)
-SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(VERSION).tar.gz 
+SNAPPY_ARCHIVE:=$(TARGET)/snappy-$(SNAPPY_VERSION).tar.gz
 SNAPPY_CC:=snappy-sinksource.cc snappy-stubs-internal.cc snappy.cc
-SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(VERSION)
+SNAPPY_SRC_DIR:=$(TARGET)/snappy-$(SNAPPY_VERSION)
 SNAPPY_SRC:=$(addprefix $(SNAPPY_SRC_DIR)/,$(SNAPPY_CC))
-SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) SnappyNative.o)
-
-SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
 SNAPPY_GIT_REPO_URL:=https://github.com/google/snappy
 SNAPPY_GIT_REV:=2b9152d9c5bed71dffb7f7f6c7a3ec48b058ff2d # 1.1.3 with autogen.sh fix
+SNAPPY_UNPACKED:=$(TARGET)/snappy-extracted.log
+SNAPPY_GIT_UNPACKED:=$(TARGET)/snappy-git-extracted.log
+
+BITSHUFFLE_ARCHIVE:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION).tar.gz
+BITSHUFFLE_C:=bitshuffle_core.c iochain.c
+BITSHUFFLE_SRC_DIR:=$(TARGET)/bitshuffle-$(BITSHUFFLE_VERSION)/src
+BITSHUFFLE_SRC:=$(addprefix $(BITSHUFFLE_SRC_DIR)/,$(BITSHUFFLE_C))
+BITSHUFFLE_UNPACKED:=$(TARGET)/bitshuffle-extracted.log
+
+$(BITSHUFFLE_ARCHIVE):
+	@mkdir -p $(@D)
+	curl -L -o$@ https://github.com/kiyo-masui/bitshuffle/archive/$(BITSHUFFLE_VERSION).tar.gz
 
-CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR)
+$(BITSHUFFLE_UNPACKED): $(BITSHUFFLE_ARCHIVE)
+	$(TAR) xvfz $< -C $(TARGET)
+	touch $@
+
+$(BITSHUFFLE_SRC): $(BITSHUFFLE_UNPACKED)
+
+$(SNAPPY_OUT)/%.o: $(BITSHUFFLE_SRC_DIR)/%.c
+	@mkdir -p $(@D)
+	$(CXX) $(CXXFLAGS) $(CXXFLAGS_BITSHUFFLE) -c $< -o $@
+
+SNAPPY_OBJ:=$(addprefix $(SNAPPY_OUT)/,$(patsubst %.cc,%.o,$(SNAPPY_CC)) $(patsubst %.c,%.o,$(BITSHUFFLE_C)) SnappyNative.o BitShuffleNative.o)
+
+CXXFLAGS:=$(CXXFLAGS) -I$(SNAPPY_SRC_DIR) -I$(BITSHUFFLE_SRC_DIR)
+
+ifndef CXXFLAGS_BITSHUFFLE
+  ifeq ($(OS_NAME)-$(OS_ARCH),Linux-x86_64)
+	# SSE2 is supported in all the x86_64 platforms and AVX2 is only supported
+        # in the small part of them. gcc in linux/x86_64 typically enables SSE2 by default though,
+	# we explicitly set flags below to make this precondition clearer.
+	CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -msse2
+  else
+	# Undefined macros to generate a platform-independent binary
+	CXXFLAGS_BITSHUFFLE:=-U__AVX2__ -U__SSE2__
+  endif
+endif
 
 ifeq ($(OS_NAME),SunOS)
 	TAR:= gtar
@@ -27,7 +60,12 @@ endif
 
 $(SNAPPY_ARCHIVE):
 	@mkdir -p $(@D)
-	curl -L -o$@ https://github.com/google/snappy/releases/download/$(VERSION)/snappy-$(VERSION).tar.gz
+	curl -L -o$@ https://github.com/google/snappy/releases/download/$(SNAPPY_VERSION)/snappy-$(SNAPPY_VERSION).tar.gz
+
+$(SNAPPY_UNPACKED): $(SNAPPY_ARCHIVE)
+	$(TAR) xvfz $< -C $(TARGET)
+	touch $@
+	cd  $(SNAPPY_SRC_DIR) && ./configure
 
 $(SNAPPY_GIT_UNPACKED):
 	rm -rf $(SNAPPY_SRC_DIR)
@@ -37,31 +75,41 @@ $(SNAPPY_GIT_UNPACKED):
 	cd $(SNAPPY_SRC_DIR) && ./autogen.sh && ./configure
 	touch $@
 
-jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h
+jni-header: $(SRC)/org/xerial/snappy/SnappyNative.h $(SRC)/org/xerial/snappy/BitShuffleNative.h
 
-$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class : $(SRC)/org/xerial/snappy/SnappyNative.java
+$(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class: $(SRC)/org/xerial/snappy/SnappyNative.java
 	@mkdir -p $(TARGET)/jni-classes
 	$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
 
 $(SRC)/org/xerial/snappy/SnappyNative.h: $(TARGET)/jni-classes/org/xerial/snappy/SnappyNative.class
 	$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.SnappyNative
 
+$(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class: $(SRC)/org/xerial/snappy/BitShuffleNative.java
+	@mkdir -p $(TARGET)/jni-classes
+	$(JAVAC) -source 1.6 -target 1.6 -d $(TARGET)/jni-classes -sourcepath $(SRC) $<
+
+$(SRC)/org/xerial/snappy/BitShuffleNative.h: $(TARGET)/jni-classes/org/xerial/snappy/BitShuffleNative.class
+	$(JAVAH) -force -classpath $(TARGET)/jni-classes -o $@ org.xerial.snappy.BitShuffleNative
+
 $(SNAPPY_SRC): $(SNAPPY_GIT_UNPACKED)
 
-$(SNAPPY_OUT)/%.o : $(SNAPPY_SRC_DIR)/%.cc
+$(SNAPPY_OUT)/%.o: $(SNAPPY_SRC_DIR)/%.cc
 	@mkdir -p $(@D)
-	$(CXX) $(CXXFLAGS) -c $< -o $@ 
+	$(CXX) $(CXXFLAGS) -c $< -o $@
 
-$(SNAPPY_OUT)/SnappyNative.o : $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h  
+$(SNAPPY_OUT)/SnappyNative.o: $(SRC)/org/xerial/snappy/SnappyNative.cpp $(SRC)/org/xerial/snappy/SnappyNative.h
 	@mkdir -p $(@D)
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+$(SNAPPY_OUT)/BitShuffleNative.o: $(SRC)/org/xerial/snappy/BitShuffleNative.cpp $(SRC)/org/xerial/snappy/BitShuffleNative.h
+	@mkdir -p $(@D)
+	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 $(SNAPPY_OUT)/$(LIBNAME): $(SNAPPY_OBJ)
-	$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS) 
+	$(CXX) $(CXXFLAGS) -o $@ $+ $(LINKFLAGS)
 	$(STRIP) $@
 
-clean-native: 
+clean-native:
 	rm -rf $(SNAPPY_OUT)
 
 clean:
@@ -76,7 +124,7 @@ snappy-jar-version:=snappy-java-$(shell perl -npe "s/version in ThisBuild\s+:=\s
 native: $(SNAPPY_GIT_UNPACKED) $(NATIVE_DLL)
 snappy: native $(TARGET)/$(snappy-jar-version).jar
 
-$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME) 
+$(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
 	@mkdir -p $(@D)
 	cp $< $@
 	@mkdir -p $(NATIVE_TARGET_DIR)
@@ -85,20 +133,20 @@ $(NATIVE_DLL): $(SNAPPY_OUT)/$(LIBNAME)
 
 package: $(TARGET)/$(snappy-jar-version).jar
 
-$(TARGET)/$(snappy-jar-version).jar: 
-	$(SBT) package 
+$(TARGET)/$(snappy-jar-version).jar:
+	$(SBT) package
 
 test: $(NATIVE_DLL)
 	$(SBT) test
 
-win32: 
+win32:
 	$(MAKE) native CROSS_PREFIX=i686-w64-mingw32- OS_NAME=Windows OS_ARCH=x86
 
 # for cross-compilation on Ubuntu, install the g++-mingw-w64-x86-64 package
 win64:
 	$(MAKE) native CROSS_PREFIX=x86_64-w64-mingw32- OS_NAME=Windows OS_ARCH=x86_64
 
-mac32: 
+mac32:
 	$(MAKE) native OS_NAME=Mac OS_ARCH=x86
 
 linux32:
@@ -133,17 +181,16 @@ install-m2:
 
 googlecode-upload: googlecode-lib-upload googlecode-src-upload
 
-googlecode-lib-upload: $(TARGET)/snappy-java-$(VERSION)-lib.upload
-googlecode-src-upload: $(TARGET)/snappy-java-$(VERSION)-src.upload
+googlecode-lib-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload
+googlecode-src-upload: $(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload
 
 GOOGLECODE_USER:=leo@xerial.org
 
-$(TARGET)/snappy-java-$(VERSION)-lib.upload:
-	./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).jar 
+$(TARGET)/snappy-java-$(SNAPPY_VERSION)-lib.upload:
+	./googlecode_upload.py -s "library for all platforms" -p snappy-java -l "Type-Executable,Featured,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).jar
 	touch $@
 
-$(TARGET)/snappy-java-$(VERSION)-src.upload:
-	./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(VERSION).tar.gz 
+$(TARGET)/snappy-java-$(SNAPPY_VERSION)-src.upload:
+	./googlecode_upload.py -s "source code archive" -p snappy-java -l "Type-Source,OpSys-All" -u "$(GOOGLECODE_USER)" target/snappy-java-$(SNAPPY_VERSION).tar.gz
 	touch $@
 
-

Makefile.common

@@ -8,13 +8,13 @@ endif
 
 
 JAVA  := "$$JAVA_HOME/bin/java"
-JAVAC := "$$JAVA_HOME/bin/javac" 
+JAVAC := "$$JAVA_HOME/bin/javac"
 JAVAH := "$$JAVA_HOME/bin/javah"
 
 OSINFO_CLASS := org.xerial.snappy.OSInfo
 OSINFO_PROG := lib/org/xerial/snappy/OSInfo.class
 
-## building OSInfo.java 
+## building OSInfo.java
 #$(info compiling OSInfo.java)
 #$(shell mkdir -p lib)
 #$(shell $(JAVAC) src/main/java/org/xerial/snappy/OSInfo.java -d lib)
@@ -31,7 +31,7 @@ else
 	sep := :
 endif
 
-snappy := snappy-$(VERSION)
+snappy := snappy-$(SNAPPY_VERSION)
 
 
 jni_md := $(shell find -L "$(JAVA_HOME)" -name jni_md.h | head -1)
@@ -70,7 +70,7 @@ Default_STRIP        := $(CROSS_PREFIX)strip
 Default_CXXFLAGS     := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
 Default_LINKFLAGS    := -shared -static
 Default_LIBNAME      := libsnappyjava.so
-Default_SNAPPY_FLAGS := 
+Default_SNAPPY_FLAGS :=
 
 Linux-x86_CXX       := $(CROSS_PREFIX)g++
 Linux-x86_STRIP     := $(CROSS_PREFIX)strip
@@ -81,18 +81,18 @@ else
 endif
 Linux-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
 Linux-x86_LIBNAME   := libsnappyjava.so
-Linux-x86_SNAPPY_FLAGS:= 
+Linux-x86_SNAPPY_FLAGS:=
 
-Linux-x86_64_CXX       := $(CROSS_PREFIX)g++ 
+Linux-x86_64_CXX       := $(CROSS_PREFIX)g++
 Linux-x86_64_STRIP     := $(CROSS_PREFIX)strip
 ifeq ($(IBM_JDK_7),)
-  Linux-x86_64_CXXFLAGS  := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 
+  Linux-x86_64_CXXFLAGS  := -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
 else
-  Linux-x86_64_CXXFLAGS  := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 
+  Linux-x86_64_CXXFLAGS  := -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
 endif
 Linux-x86_64_LINKFLAGS := -shared -static-libgcc -static-libstdc++
 Linux-x86_64_LIBNAME   := libsnappyjava.so
-Linux-x86_64_SNAPPY_FLAGS  := 
+Linux-x86_64_SNAPPY_FLAGS  :=
 
 Linux-ppc_CXX         := g++
 Linux-ppc_STRIP       := strip
@@ -108,7 +108,7 @@ Linux-ppc_SNAPPY_FLAGS  :=
 Linux-ppc64_CXX       := g++
 Linux-ppc64_STRIP     := strip
 ifeq ($(IBM_JDK_7),)
-  Linux-ppc64_CXXFLAGS  := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64 
+  Linux-ppc64_CXXFLAGS  := -DHAVE_CONFIG_H -Ilib/inc_linux -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden -m64
 else
   Linux-ppc64_CXXFLAGS  := -DHAVE_CONFIG_H -include $(IBM_JDK_LIB)/jni_md.h -include $(IBM_JDK_LIB)/jniport.h -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux -O2 -fPIC -m64
 endif
@@ -160,19 +160,19 @@ Linux-s390x_LINKFLAGS := -shared -static-libgcc -static-libstdc++
 Linux-s390x_LIBNAME   := libsnappyjava.so
 Linux-s390x_SNAPPY_FLAGS  :=
 
-SunOS-x86_CXX       := g++ 
+SunOS-x86_CXX       := g++
 SunOS-x86_STRIP     := strip
 SunOS-x86_CXXFLAGS  := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
 SunOS-x86_LINKFLAGS := -shared -static-libgcc -static-libstdc++
 SunOS-x86_LIBNAME   := libsnappyjava.so
-SunOS-x86_SNAPPY_FLAGS  := 
+SunOS-x86_SNAPPY_FLAGS  :=
 
-SunOS-sparc_CXX       := g++ 
+SunOS-sparc_CXX       := g++
 SunOS-sparc_STRIP     := strip
 SunOS-sparc_CXXFLAGS  := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden
 SunOS-sparc_LINKFLAGS := -shared -static-libgcc -static-libstdc++
 SunOS-sparc_LIBNAME   := libsnappyjava.so
-SunOS-sparc_SNAPPY_FLAGS  := 
+SunOS-sparc_SNAPPY_FLAGS  :=
 
 SunOS-x86_64_CXX       := g++
 SunOS-x86_64_STRIP     := strip
@@ -191,14 +191,14 @@ Linux-arm_STRIP     := $(CROSS_PREFIX)strip
 Linux-arm_CXXFLAGS  := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=softfp
 Linux-arm_LINKFLAGS := -shared -static-libgcc
 Linux-arm_LIBNAME   := libsnappyjava.so
-Linux-arm_SNAPPY_FLAGS:= 
+Linux-arm_SNAPPY_FLAGS:=
 
 Linux-armhf_CXX       := $(CROSS_PREFIX)g++
 Linux-armhf_STRIP     := $(CROSS_PREFIX)strip
 Linux-armhf_CXXFLAGS  := -include lib/inc_linux/jni_md.h -I$(JAVA_HOME)/include -O2 -fPIC -fvisibility=hidden -mfloat-abi=hard
 Linux-armhf_LINKFLAGS := -shared -static-libgcc
 Linux-armhf_LIBNAME   := libsnappyjava.so
-Linux-armhf_SNAPPY_FLAGS:= 
+Linux-armhf_SNAPPY_FLAGS:=
 
 Linux-aarch64_CXX       := $(CROSS_PREFIX)g++
 Linux-aarch64_STRIP     := $(CROSS_PREFIX)strip
@@ -214,47 +214,46 @@ Mac-x86_LINKFLAGS := -dynamiclib -static-libgcc
 Mac-x86_LIBNAME   := libsnappyjava.jnilib
 Mac-x86_SNAPPY_FLAGS  :=
 
-Mac-x86_64_CXX       := g++ -arch $(OS_ARCH) 
+Mac-x86_64_CXX       := g++ -arch $(OS_ARCH)
 Mac-x86_64_STRIP     := strip -x
 Mac-x86_64_CXXFLAGS  := -Ilib/inc_mac -I$(JAVA_HOME)/include -O2 -fPIC -mmacosx-version-min=10.5 -fvisibility=hidden
-Mac-x86_64_LINKFLAGS := -dynamiclib 
-Mac-x86_64_LIBNAME   := libsnappyjava.jnilib 
-Mac-x86_64_SNAPPY_FLAGS  := 
+Mac-x86_64_LINKFLAGS := -dynamiclib
+Mac-x86_64_LIBNAME   := libsnappyjava.jnilib
+Mac-x86_64_SNAPPY_FLAGS  :=
 
 FreeBSD-x86_64_CXX          := $(CROSS_PREFIX)g++
 FreeBSD-x86_64_STRIP        := $(CROSS_PREFIX)strip
 FreeBSD-x86_64_CXXFLAGS     := -I$(JAVA_HOME)/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
 FreeBSD-x86_64_LINKFLAGS    := -shared -static-libgcc
 FreeBSD-x86_64_LIBNAME      := libsnappyjava.so
-FreeBSD-x86_64_SNAPPY_FLAGS := 
+FreeBSD-x86_64_SNAPPY_FLAGS :=
 
 Windows-x86_CXX          := $(CROSS_PREFIX)g++
 Windows-x86_STRIP        := $(CROSS_PREFIX)strip
-Windows-x86_CXXFLAGS     := -Ilib/inc_win -O2 
+Windows-x86_CXXFLAGS     := -Ilib/inc_win -O2
 Windows-x86_LINKFLAGS    := -Wl,--kill-at -shared -static
 Windows-x86_LIBNAME      := snappyjava.dll
-Windows-x86_SNAPPY_FLAGS := 
+Windows-x86_SNAPPY_FLAGS :=
 
 Windows-x86_64_CXX          := $(CROSS_PREFIX)g++
 Windows-x86_64_STRIP        := $(CROSS_PREFIX)strip
-Windows-x86_64_CXXFLAGS     := -Ilib/inc_win -O2 
-Windows-x86_64_LINKFLAGS    := -Wl,--kill-at -shared -static 
+Windows-x86_64_CXXFLAGS     := -Ilib/inc_win -O2
+Windows-x86_64_LINKFLAGS    := -Wl,--kill-at -shared -static
 Windows-x86_64_LIBNAME      := snappyjava.dll
-Windows-x86_64_SNAPPY_FLAGS := 
+Windows-x86_64_SNAPPY_FLAGS :=
 
 
 CXX        := $($(os_arch)_CXX)
 STRIP     := $($(os_arch)_STRIP)
-CXXFLAGS    := $($(os_arch)_CXXFLAGS) 
-LINKFLAGS := $($(os_arch)_LINKFLAGS) 
+CXXFLAGS    := $($(os_arch)_CXXFLAGS)
+LINKFLAGS := $($(os_arch)_LINKFLAGS)
 LIBNAME   := $($(os_arch)_LIBNAME)
 SNAPPY_FLAGS := $($(os_arch)_SNAPPY_FLAGS)
 
 
 
-CXXFLAGS := $(CXXFLAGS) -Ilib/include  
+CXXFLAGS := $(CXXFLAGS) -Ilib/include
 ifneq ($(jni_include),)
 CXXFLAGS := $(CXXFLAGS) -I"$(jni_include)"
 endif
 
-

README.md

@@ -143,6 +143,12 @@ Older snapshots of snappy contain a buggy config.h.in that does not work properl
 	$ export JAVA_HOME=/usr/lib/jvm/java-1.7.1-ibm-1.7.1.2.10-1jpp.3.el7_0.s390x
 	$ make USE_GIT=1 GIT_REPO_URL=https://github.com/google/snappy.git GIT_SNAPPY_BRANCH=master IBM_JDK_7=1
 
+## Activating SSE2/AVX2 instructions in BitShuffle
+
+The most of the native libraries that snappy-java contains disable SSE2/AVX2 instructions in terms of portability (SSE2 is enabled only in Linux/x86_64 platforms). To enable AVX2 instructions, you need to compile as follows:
+
+	$ make CXXFLAGS_BITSHUFFLE=-mavx2  # -msse2 for SSE2 instructions
+
 ## Cross-compiling for other platforms
 The Makefile contains rules for cross-compiling the native library for other platforms so that the snappy-java JAR can support multiple platforms. For example, to build the native libraries for x86 Linux, x86 and x86-64 Windows, and soft- and hard-float ARM: