From f447021c30b8dc77c1add6cb7b122c6ca1e33493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Le=20Lay?= Date: Fri, 27 Apr 2012 22:33:56 +0200 Subject: [PATCH] Add LZO support. --- project.clj | 3 ++- src/jvm/backtype/hadoop/pail/SequenceFileFormat.java | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index bf08cb9..84e9d23 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject backtype/dfs-datastores "1.1.3-SNAPSHOT" +(defproject org.clojars.lelayf/dfs-datastores "1.1.3-SNAPSHOT" :source-path "src/clj" :test-path "test/clj" :java-source-path "src/jvm" @@ -16,3 +16,4 @@ [lein-javac "1.3.0"] [lein-junit "1.0.0"] [junit "4.7"]]) + diff --git a/src/jvm/backtype/hadoop/pail/SequenceFileFormat.java b/src/jvm/backtype/hadoop/pail/SequenceFileFormat.java index 5acdda6..a2a8e62 100644 --- a/src/jvm/backtype/hadoop/pail/SequenceFileFormat.java +++ b/src/jvm/backtype/hadoop/pail/SequenceFileFormat.java @@ -17,6 +17,8 @@ import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.hadoop.io.compress.LzoCodec ; +//import org.apache.hadoop.io.compress.SnappyCodec ; import org.apache.hadoop.mapred.*; import org.apache.log4j.Logger; @@ -38,6 +40,8 @@ public class SequenceFileFormat implements PailFormat { public static final String CODEC_ARG_DEFAULT = "default"; public static final String CODEC_ARG_GZIP = "gzip"; public static final String CODEC_ARG_BZIP2 = "bzip2"; + public static final String CODEC_ARG_LZO = "lzo"; + // public static final String CODEC_ARG_SNAPPY = "snappy"; private static final Map TYPES = new HashMap() {{ put(TYPE_ARG_RECORD, CompressionType.RECORD); @@ -48,6 +52,8 @@ public class SequenceFileFormat implements PailFormat { put(CODEC_ARG_DEFAULT, new DefaultCodec()); put(CODEC_ARG_GZIP, new GzipCodec()); put(CODEC_ARG_BZIP2, new BZip2Codec()); + put(CODEC_ARG_LZO, new LzoCodec()); + // put(CODEC_ARG_LZO, new SnappyCodec()); }}; private String _typeArg; @@ -56,7 +62,7 @@ public class SequenceFileFormat implements PailFormat { public SequenceFileFormat(Map args) { args = new KeywordArgParser() .add(TYPE_ARG, null, true, TYPE_ARG_RECORD, TYPE_ARG_BLOCK) - .add(CODEC_ARG, CODEC_ARG_DEFAULT, false, CODEC_ARG_DEFAULT, CODEC_ARG_GZIP, CODEC_ARG_BZIP2) + .add(CODEC_ARG, CODEC_ARG_DEFAULT, false, CODEC_ARG_DEFAULT, CODEC_ARG_GZIP, CODEC_ARG_BZIP2, CODEC_ARG_LZO) .parse(args); _typeArg = (String) args.get(TYPE_ARG); _codecArg = (String) args.get(CODEC_ARG);