Permalink
Browse files

Made GPLNativeCodeLoader use the embedded native binaries.

  • Loading branch information...
1 parent 4672bde commit e4e9136b21282f839e01491aa1a14390646ba7bb @sjlee sjlee committed Oct 3, 2013
Showing with 127 additions and 5 deletions.
  1. +15 −3 pom.xml
  2. +112 −2 src/main/java/com/hadoop/compression/lzo/GPLNativeCodeLoader.java
View
@@ -184,6 +184,7 @@
</exec>
<property name="build.platform" value="${nonspace.os}-${os.arch}-${sun.arch.data.model}"/>
<property name="build.native" value="${project.build.directory}/native/${build.platform}"/>
+ <property name="build.native.target" value="${project.build.outputDirectory}/native/${build.platform}"/>
<property name="native.src.dir" value="${basedir}/src/main/native"/>
<property name="test.build.dir" value="${project.build.directory}/test-classes"/>
<property name="test.log.dir" value="${test.build.dir}/logs"/>
@@ -200,6 +201,7 @@
<exportAntProperties>true</exportAntProperties>
<target name="set-props-win" description="sets key properties that are used throughout" if="windows">
<property name="build.native" value="${project.build.directory}/native/${env.OS}-${env.PLATFORM}"/>
+ <property name="build.native.target" value="${project.build.outputDirectory}/native/${build.platform}"/>
<property name="native.src.dir" value="${basedir}/src/main/native"/>
<property name="test.build.dir" value="${project.build.directory}/test-classes"/>
<property name="test.log.dir" value="${test.build.dir}/logs"/>
@@ -269,7 +271,7 @@
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="check-native-uptodate-non-win" description="checks if native binaries should be rebuilt" if="non-windows">
- <uptodate property="native.uptodate" targetfile="${build.native}/lib/libgplcompression.la">
+ <uptodate property="native.uptodate" targetfile="${build.native.target}/lib/libgplcompression.la">
<srcfiles dir="${native.src.dir}" includes="**/*"/>
</uptodate>
</target>
@@ -284,7 +286,7 @@
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="check-native-uptodate-win" description="checks if native binaries should be rebuilt" if="windows">
- <uptodate property="native.uptodate" targetfile="${build.native}/lib/gplcompression.dll">
+ <uptodate property="native.uptodate" targetfile="${build.native.target}/lib/gplcompression.dll">
<srcfiles dir="${native.src.dir}" includes="**/*"/>
</uptodate>
</target>
@@ -305,6 +307,7 @@
</condition>
<mkdir dir="${build.native}/lib"/>
+ <mkdir dir="${build.native.target}/lib"/>
<mkdir dir="${build.native}/src/com/hadoop/compression/lzo"/>
<javah classpath="${build.classes}"
@@ -334,6 +337,10 @@
<exec dir="${build.native}" executable="sh" failonerror="true">
<arg line="${build.native}/libtool --mode=install cp ${build.native}/libgplcompression.la ${build.native}/lib"/>
</exec>
+
+ <copy todir="${build.native.target}/lib">
+ <fileset dir="${build.native}/lib"/>
+ </copy>
</target>
</configuration>
</execution>
@@ -348,6 +355,7 @@
<property name="build.classes" refid="maven.compile.classpath"/>
<mkdir dir="${build.native}/lib"/>
+ <mkdir dir="${build.native.target}/lib"/>
<mkdir dir="${build.native}/src/com/hadoop/compression/lzo"/>
<javah classpath="${build.classes}"
@@ -367,6 +375,10 @@
</exec>
<copy file="${build.native}/gplcompression.dll" todir="${build.native}/lib" />
+
+ <copy todir="${build.native.target}/lib">
+ <fileset dir="${build.native}/lib"/>
+ </copy>
</target>
</configuration>
</execution>
@@ -405,7 +417,7 @@
<version>2.14.1</version>
<configuration>
<reuseForks>false</reuseForks>
- <argLine>-Djava.library.path=${build.native}/lib</argLine>
+ <argLine>-Djava.library.path=${build.native.target}/lib</argLine>
<systemPropertyVariables>
<test.build.data>${project.build.directory}/test-classes/data</test.build.data>
<hadoop.log.dir>${test.log.dir}</hadoop.log.dir>
@@ -18,20 +18,47 @@
package com.hadoop.compression.lzo;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class GPLNativeCodeLoader {
+ public static final String LIBRARY_NAME = "gplcompression";
+ /**
+ * The system property that causes hadoop-lzo to ignore the embedded native
+ * library and load it from the normal library path instead. It is false by
+ * default (i.e. it loads the embedded native library if found over the
+ * library path).
+ */
+ public static final String USE_BINARIES_ON_LIB_PATH =
+ "com.hadoop.compression.lzo.use.libpath";
private static final Log LOG = LogFactory.getLog(GPLNativeCodeLoader.class);
private static boolean nativeLibraryLoaded = false;
static {
try {
//try to load the lib
- System.loadLibrary("gplcompression");
+ if (!useBinariesOnLibPath()) {
+ File unpackedFile = unpackBinaries();
+ if (unpackedFile != null) { // the file was successfully unpacked
+ String path = unpackedFile.getAbsolutePath();
+ System.load(path);
+ LOG.info("Loaded native gpl library from the embedded binaries");
+ } else { // fall back
+ System.loadLibrary(LIBRARY_NAME);
+ LOG.info("Loaded native gpl library from the library path");
+ }
+ } else {
+ System.loadLibrary(LIBRARY_NAME);
+ LOG.info("Loaded native gpl library from the library path");
+ }
nativeLibraryLoaded = true;
- LOG.info("Loaded native gpl library");
} catch (Throwable t) {
LOG.error("Could not load native gpl library", t);
nativeLibraryLoaded = false;
@@ -46,4 +73,87 @@ public static boolean isNativeCodeLoaded() {
return nativeLibraryLoaded;
}
+ private static boolean useBinariesOnLibPath() {
+ return Boolean.getBoolean(USE_BINARIES_ON_LIB_PATH);
+ }
+
+ /**
+ * Locates the native library in the jar (loadble by the classloader really),
+ * unpacks it in a temp location, and returns that file. If the native library
+ * is not found by the classloader, returns null.
+ */
+ private static File unpackBinaries() {
+ // locate the binaries inside the jar
+ String fileName = System.mapLibraryName(LIBRARY_NAME);
+ String directory = getDirectoryLocation();
+ // use the current defining classloader to load the resource
+ InputStream is =
+ GPLNativeCodeLoader.class.getResourceAsStream(directory + "/" + fileName);
+ if (is == null) {
+ // specific to mac
+ // on mac the filename can be either .dylib or .jnilib: try again with the
+ // alternate name
+ if (getOsName().contains("Mac")) {
+ if (fileName.endsWith(".dylib")) {
+ fileName = fileName.replace(".dylib", ".jnilib");
+ } else if (fileName.endsWith(".jnilib")) {
+ fileName = fileName.replace(".jnilib", ".dylib");
+ }
+ is = GPLNativeCodeLoader.class.getResourceAsStream(directory + "/" + fileName);
+ }
+ // the OS-specific library was not found: fall back on the library path
+ if (is == null) {
+ return null;
+ }
+ }
+
+ // write the file
+ byte[] buffer = new byte[8192];
+ OutputStream os = null;
+ try {
+ // prepare the unpacked file location
+ File unpackedFile = File.createTempFile("unpacked-", "-" + fileName);
+ // ensure the file gets cleaned up
+ unpackedFile.deleteOnExit();
+
+ os = new FileOutputStream(unpackedFile);
+ int read = 0;
+ while ((read = is.read(buffer)) != -1) {
+ os.write(buffer, 0, read);
+ }
+
+ // set the execution permission
+ unpackedFile.setExecutable(true, false);
+ LOG.debug("temporary unpacked path: " + unpackedFile);
+ // return the file
+ return unpackedFile;
+ } catch (IOException e) {
+ LOG.error("could not unpack the binaries", e);
+ return null;
+ } finally {
+ try { is.close(); } catch (IOException ignore) {}
+ if (os != null) {
+ try { os.close(); } catch (IOException ignore) {}
+ }
+ }
+ }
+
+ private static String getDirectoryLocation() {
+ String osName = getOsName().replace(' ', '_');
+ boolean windows = osName.toLowerCase().contains("windows");
+ if (!windows) {
+ String location = "/native/" + osName + "-" + System.getProperty("os.arch") + "-" +
+ System.getProperty("sun.arch.data.model") + "/lib";
+ LOG.debug("location: " + location);
+ return location;
+ } else {
+ String location = "/native/" + System.getenv("OS") + "-" + System.getenv("PLATFORM") + "/lib";
+ LOG.debug("location: " + location);
+ return location;
+ }
+ }
+
+ private static String getOsName() {
+ return System.getProperty("os.name");
+ }
}

0 comments on commit e4e9136

Please sign in to comment.