Skip to content

Commit

Permalink
Add idea projects to .gitignore, add build properties to jar, and log…
Browse files Browse the repository at this point in the history
… build's git revision once upon static load.
  • Loading branch information
kevinweil committed Nov 1, 2009
1 parent 0f43411 commit 5f0333b
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 19 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Expand Up @@ -20,3 +20,6 @@
.svn .svn
build/ build/
bin/ bin/
*.ipr
*.iml
*.iws
31 changes: 29 additions & 2 deletions build.xml
Expand Up @@ -89,7 +89,7 @@


<property name="make.cmd" value="make"/> <property name="make.cmd" value="make"/>


<!-- IVY properteis set here --> <!-- IVY properties set here -->
<property name="ivy.dir" location="ivy" /> <property name="ivy.dir" location="ivy" />
<loadproperties srcfile="${ivy.dir}/libraries.properties"/> <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
<property name="ivy.jar" location="${ivy.dir}/ivy-${ivy.version}.jar"/> <property name="ivy.jar" location="${ivy.dir}/ivy-${ivy.version}.jar"/>
Expand Down Expand Up @@ -176,6 +176,33 @@
<mkdir dir="${test.build.classes}"/> <mkdir dir="${test.build.classes}"/>
</target> </target>


<!-- ====================================================== -->
<!-- Build properties file -->
<!-- ====================================================== -->
<target name="buildinfo">
<tstamp>
<format property="build_time" pattern="MM/dd/yyyy hh:mm aa" timezone="GMT"/>
</tstamp>
<exec executable="git" outputproperty="build_revision">
<arg value="rev-parse"/>
<arg value="HEAD"/>
</exec>
<exec executable="whoami" outputproperty="build_author"/>
<exec executable="uname" outputproperty="build_os">
<arg value="-a"/>
</exec>

<propertyfile file="${build.classes}/build.properties"
comment="This file is automatically generated - DO NOT EDIT">
<entry key="build_time" value="${build_time}"/>
<entry key="build_revision" value="${build_revision}"/>
<entry key="build_author" value="${build_author}"/>
<entry key="build_version" value="${version}"/>
<entry key="build_os" value="${build_os}"/>
</propertyfile>
</target>


<!-- ====================================================== --> <!-- ====================================================== -->
<!-- Compile the Java files --> <!-- Compile the Java files -->
<!-- ====================================================== --> <!-- ====================================================== -->
Expand Down Expand Up @@ -243,7 +270,7 @@
<!-- ================================================================== --> <!-- ================================================================== -->
<!-- --> <!-- -->
<!-- ================================================================== --> <!-- ================================================================== -->
<target name="jar" depends="compile-java" description="Make jar"> <target name="jar" depends="compile-java, buildinfo" description="Make jar">
<jar jarfile="${build.dir}/${final.name}.jar" <jar jarfile="${build.dir}/${final.name}.jar"
basedir="${build.classes}"> basedir="${build.classes}">
<manifest> <manifest>
Expand Down
45 changes: 28 additions & 17 deletions src/java/com/hadoop/compression/lzo/LzoCodec.java
Expand Up @@ -15,12 +15,13 @@
* along with Hadoop-Gpl-Compression. If not, see * along with Hadoop-Gpl-Compression. If not, see
* <http://www.gnu.org/licenses/>. * <http://www.gnu.org/licenses/>.
*/ */

package com.hadoop.compression.lzo; package com.hadoop.compression.lzo;


import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.Properties;


import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
Expand All @@ -38,10 +39,9 @@
* A {@link org.apache.hadoop.io.compress.CompressionCodec} for a streaming * A {@link org.apache.hadoop.io.compress.CompressionCodec} for a streaming
* <b>lzo</b> compression/decompression pair. * <b>lzo</b> compression/decompression pair.
* http://www.oberhumer.com/opensource/lzo/ * http://www.oberhumer.com/opensource/lzo/
* *
*/ */
public class LzoCodec implements Configurable, CompressionCodec { public class LzoCodec implements Configurable, CompressionCodec {

private static final Log LOG = LogFactory.getLog(LzoCodec.class.getName()); private static final Log LOG = LogFactory.getLog(LzoCodec.class.getName());


private Configuration conf; private Configuration conf;
Expand All @@ -62,7 +62,7 @@ public Configuration getConf() {
LzoDecompressor.isNativeLzoLoaded(); LzoDecompressor.isNativeLzoLoaded();


if (nativeLzoLoaded) { if (nativeLzoLoaded) {
LOG.info("Successfully loaded & initialized native-lzo library"); LOG.info("Successfully loaded & initialized native-lzo library [hadoop-lzo rev " + getRevisionHash() + "]");
} else { } else {
LOG.error("Failed to load/initialize native-lzo library"); LOG.error("Failed to load/initialize native-lzo library");
} }
Expand All @@ -73,7 +73,7 @@ public Configuration getConf() {


/** /**
* Check if native-lzo library is loaded & initialized. * Check if native-lzo library is loaded & initialized.
* *
* @param conf configuration * @param conf configuration
* @return <code>true</code> if native-lzo library is loaded & initialized; * @return <code>true</code> if native-lzo library is loaded & initialized;
* else <code>false</code> * else <code>false</code>
Expand All @@ -82,12 +82,23 @@ public static boolean isNativeLzoLoaded(Configuration conf) {
return nativeLzoLoaded && conf.getBoolean("hadoop.native.lib", true); return nativeLzoLoaded && conf.getBoolean("hadoop.native.lib", true);
} }


public CompressionOutputStream createOutputStream(OutputStream out) public static String getRevisionHash() {
try {
Properties p = new Properties();
p.load(LzoCodec.class.getResourceAsStream("/build.properties"));
return p.getProperty("build_revision");
} catch (IOException e) {
LOG.error("Could not find build properties file with revision hash");
return "UNKNOWN";
}
}

public CompressionOutputStream createOutputStream(OutputStream out)
throws IOException { throws IOException {
return createOutputStream(out, createCompressor()); return createOutputStream(out, createCompressor());
} }


public CompressionOutputStream createOutputStream(OutputStream out, public CompressionOutputStream createOutputStream(OutputStream out,
Compressor compressor) throws IOException { Compressor compressor) throws IOException {
// Ensure native-lzo library is loaded & initialized // Ensure native-lzo library is loaded & initialized
if (!isNativeLzoLoaded(conf)) { if (!isNativeLzoLoaded(conf)) {
Expand All @@ -102,20 +113,20 @@ public CompressionOutputStream createOutputStream(OutputStream out,
* LZO will expand incompressible data by a little amount. * LZO will expand incompressible data by a little amount.
* I still haven't computed the exact values, but I suggest using * I still haven't computed the exact values, but I suggest using
* these formulas for a worst-case expansion calculation: * these formulas for a worst-case expansion calculation:
* *
* Algorithm LZO1, LZO1A, LZO1B, LZO1C, LZO1F, LZO1X, LZO1Y, LZO1Z: * Algorithm LZO1, LZO1A, LZO1B, LZO1C, LZO1F, LZO1X, LZO1Y, LZO1Z:
* ---------------------------------------------------------------- * ----------------------------------------------------------------
* output_block_size = input_block_size + (input_block_size / 16) + 64 + 3 * output_block_size = input_block_size + (input_block_size / 16) + 64 + 3
* *
* This is about 106% for a large block size. * This is about 106% for a large block size.
* *
* Algorithm LZO2A: * Algorithm LZO2A:
* ---------------- * ----------------
* output_block_size = input_block_size + (input_block_size / 8) + 128 + 3 * output_block_size = input_block_size + (input_block_size / 8) + 128 + 3
*/ */


// Create the lzo output-stream // Create the lzo output-stream
LzoCompressor.CompressionStrategy strategy = LzoCompressor.CompressionStrategy strategy =
LzoCompressor.CompressionStrategy.valueOf( LzoCompressor.CompressionStrategy.valueOf(
conf.get("io.compression.codec.lzo.compressor", conf.get("io.compression.codec.lzo.compressor",
LzoCompressor.CompressionStrategy.LZO1X_1.name())); LzoCompressor.CompressionStrategy.LZO1X_1.name()));
Expand Down Expand Up @@ -143,7 +154,7 @@ public Compressor createCompressor() {
throw new RuntimeException("native-lzo library not available"); throw new RuntimeException("native-lzo library not available");
} }


LzoCompressor.CompressionStrategy strategy = LzoCompressor.CompressionStrategy strategy =
LzoCompressor.CompressionStrategy.valueOf( LzoCompressor.CompressionStrategy.valueOf(
conf.get("io.compression.codec.lzo.compressor", conf.get("io.compression.codec.lzo.compressor",
LzoCompressor.CompressionStrategy.LZO1X_1.name())); LzoCompressor.CompressionStrategy.LZO1X_1.name()));
Expand All @@ -158,14 +169,14 @@ public CompressionInputStream createInputStream(InputStream in)
return createInputStream(in, createDecompressor()); return createInputStream(in, createDecompressor());
} }


public CompressionInputStream createInputStream(InputStream in, public CompressionInputStream createInputStream(InputStream in,
Decompressor decompressor) Decompressor decompressor)
throws IOException { throws IOException {
// Ensure native-lzo library is loaded & initialized // Ensure native-lzo library is loaded & initialized
if (!isNativeLzoLoaded(conf)) { if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available"); throw new RuntimeException("native-lzo library not available");
} }
return new BlockDecompressorStream(in, decompressor, return new BlockDecompressorStream(in, decompressor,
conf.getInt("io.compression.codec.lzo.buffersize", 64*1024)); conf.getInt("io.compression.codec.lzo.buffersize", 64*1024));
} }


Expand All @@ -183,14 +194,14 @@ public Decompressor createDecompressor() {
throw new RuntimeException("native-lzo library not available"); throw new RuntimeException("native-lzo library not available");
} }


LzoDecompressor.CompressionStrategy strategy = LzoDecompressor.CompressionStrategy strategy =
LzoDecompressor.CompressionStrategy.valueOf( LzoDecompressor.CompressionStrategy.valueOf(
conf.get("io.compression.codec.lzo.decompressor", conf.get("io.compression.codec.lzo.decompressor",
LzoDecompressor.CompressionStrategy.LZO1X.name())); LzoDecompressor.CompressionStrategy.LZO1X.name()));
int bufferSize = int bufferSize =
conf.getInt("io.compression.codec.lzo.buffersize", 64*1024); conf.getInt("io.compression.codec.lzo.buffersize", 64*1024);


return new LzoDecompressor(strategy, bufferSize); return new LzoDecompressor(strategy, bufferSize);
} }


/** /**
Expand Down

0 comments on commit 5f0333b

Please sign in to comment.