Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

added travis support. Fixed filesystem permissions issues in consolid…

…ator.
  • Loading branch information...
commit 0378be078be668f12118db9120d35c57a2164eef 1 parent 604eccd
@sritchie sritchie authored
View
10 .travis.yml
@@ -0,0 +1,10 @@
+# See this guide for more settings:
+#
+# http://about.travis-ci.org/docs/user/build-configuration/
+
+language: clojure
+script: lein javac, junit
+branches:
+ only:
+ - develop
+ - master
View
105 build.xml
@@ -1,105 +0,0 @@
-<project name="dfs_datastores" default="jar">
- <property name="src.dir" location="${basedir}/src/jvm"/>
- <property name="final.name" value="dfs_datastores"/>
- <property name="javac.deprecation" value="true"/>
-
- <property name="build.dir" location="${basedir}/build"/>
- <property name="build.classes" location="${build.dir}/classes"/>
- <property name="build.test" location="${build.dir}/test"/>
- <property name="build.encoding" value="ISO-8859-1"/>
-
- <property name="test.junit.output.format" value="plain"/>
- <property name="test.timeout" value="600000"/>
-
- <property name="src.test" location="test/jvm"/>
-
- <property environment="env" />
-
- <fileset id="deps.jars" dir="${basedir}" includes="lib/**/*.jar"/>
- <path id="classpath.id">
- <pathelement location="${basedir}/build"/>
- <fileset refid="deps.jars"/>
- </path>
-
- <path id="test.build.classpath.id">
- <path refid="classpath.id"/>
- <pathelement location="build/${final.name}.jar"/>
- </path>
-
- <path id="test.run.classpath.id">
- <path refid="test.build.classpath.id"/>
- <pathelement location="${build.test}"/>
- </path>
-
- <target name="classpath">
- <property name="cp" refid="classpath.id"/>
- <echo message="Classpath: ${cp}"/>
- </target>
-
- <target name="init">
- <mkdir dir="${build.dir}"/>
- <mkdir dir="${build.classes}"/>
- </target>
-
- <target name="init-test">
- <mkdir dir="${build.dir}/test"/>
- <mkdir dir="${build.dir}/test/data"/>
- <mkdir dir="${build.dir}/test/data/sandbox"/>
- <mkdir dir="${build.test}"/>
- </target>
-
- <target name="compile" depends="init">
- <javac
- encoding="${build.encoding}"
- srcdir="${src.dir};"
- includes="**/*.java"
- destdir="${build.classes}"
- debug="true"
- deprecation="${javac.deprecation}">
- <classpath refid="classpath.id"/>
- </javac>
- </target>
-
- <target name="jar" depends="compile" description="Build jar">
- <jar jarfile="${build.dir}/${final.name}.jar" basedir="${build.classes}/" />
- </target>
-
- <target name="clean" description="Destroy all build artifacts">
- <delete dir="${build.dir}"/>
- </target>
-
- <target name="test" description="Runs all tests" depends="jar,init-test,compile-test">
- <delete dir="build/test/log"/>
- <mkdir dir="build/test/log"/>
- <junit
- printsummary="yes" showoutput="${test.output}"
- haltonfailure="no" fork="yes" maxmemory="512m"
- errorProperty="tests.failed" failureProperty="tests.failed"
- timeout="${test.timeout}">
-
- <sysproperty key="build.test" value="${build.test}"/>
- <classpath refid="test.run.classpath.id"/>
- <formatter type="${test.junit.output.format}" />
- <batchtest todir="build/test/log" unless="testcase">
- <fileset dir="${src.test}" includes="**/*Test.java" />
- </batchtest>
- <batchtest todir="build/test/log" if="testcase">
- <fileset dir="${src.test}" includes="**/${testcase}.java"/>
- </batchtest>
- </junit>
- <fail if="tests.failed">Tests failed!</fail>
- </target>
-
- <target name="compile-test" depends="init-test,jar">
- <javac
- encoding="${build.encoding}"
- srcdir="test/jvm;"
- includes="**/*.java"
- destdir="${build.test}"
- debug="true"
- deprecation="${javac.deprecation}">
- <classpath refid="test.build.classpath.id"/>
- </javac>
- </target>
-
-</project>
View
8 project.clj
@@ -4,8 +4,14 @@
:java-source-path "src/jvm"
:java-test-path "test/jvm"
:javac-options {:debug "true" :fork "true"}
+ :javac-source-path [["src"] ["test"]]
+ :junit [["classes"]]
+ :junit-options {:fork "off" :haltonfailure "on"}
:dependencies [[jvyaml "1.0.0"]
[com.google.guava/guava "r09"]]
:dev-dependencies [[org.apache.hadoop/hadoop-core "0.20.2-dev"]
[midje "1.3.0"]
- [junit/junit "3.8.2"]])
+ [lein-midje "1.0.8"]
+ [lein-javac "1.3.0"]
+ [lein-junit "1.0.0"]
+ [junit "4.7"]])
View
8 src/jvm/backtype/hadoop/Consolidator.java
@@ -201,10 +201,10 @@ public void map(ArrayWritable sourcesArr, Text target, OutputCollector<NullWrita
}
@Override
- public void configure(JobConf job) {
- args = (ConsolidatorArgs) Utils.getObject(job, ARGS);
+ public void configure(JobConf conf) {
+ args = (ConsolidatorArgs) Utils.getObject(conf, ARGS);
try {
- fs = Utils.getFS(args.fsUri);
+ fs = Utils.getFS(args.fsUri, conf);
} catch(IOException e) {
throw new RuntimeException(e);
}
@@ -354,7 +354,7 @@ public int compare(InputSplit o1, InputSplit o2) {
List<String> dirs = args.dirs;
List<InputSplit> ret = new ArrayList<InputSplit>();
for(String dir: dirs) {
- FileSystem fs = Utils.getFS(dir);
+ FileSystem fs = Utils.getFS(dir, conf);
ret.addAll(createSplits(fs, lister.getFiles(fs,dir),
dir, args.targetSizeBytes, args.extension));
}
View
6 src/jvm/backtype/support/Utils.java
@@ -191,7 +191,11 @@ public static String getScheme(String path) {
}
public static FileSystem getFS(String path) throws IOException {
- return new Path(path).getFileSystem(new Configuration());
+ return getFS(path, new Configuration());
+ }
+
+ public static FileSystem getFS(String path, Configuration conf) throws IOException {
+ return new Path(path).getFileSystem(conf);
}
public static String stripExtension(String str, String extension) {
Please sign in to comment.
Something went wrong with that request. Please try again.