Skip to content

Commit

Permalink
Merge pull request #1 from lintool/master
Browse files Browse the repository at this point in the history 
Minor Hadoop fixes
  • Loading branch information
Don Metzler committed Feb 7, 2012
2 parents c44cbed + 0620a11 commit 7e325f5
Show file tree
Hide file tree
Showing 13 changed files with 67 additions and 4 deletions.
51 changes: 51 additions & 0 deletions .classpath
View file
Expand Up @@ -3,5 +3,56 @@
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
<classpathentry kind="con" path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?ivyXmlPath=ivy%2Fivy.xml&amp;confs=*&amp;ivySettingsPath=%24%7Bworkspace_loc%3Amavuno%2Fivy%2Fivysettings.xml%7D&amp;loadSettingsOnDemand=false&amp;propertyFiles="/>
<classpathentry kind="lib" path="lib/activation-1.1.jar"/>
<classpathentry kind="lib" path="lib/ant-1.6.5.jar"/>
<classpathentry kind="lib" path="lib/commons-cli-1.2.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-el-1.0.jar"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.0.1.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/commons-net-1.4.1.jar"/>
<classpathentry kind="lib" path="lib/core-3.1.1.jar"/>
<classpathentry kind="lib" path="lib/fanseparser-0.2.2.jar"/>
<classpathentry kind="lib" path="lib/geronimo-jms_1.1_spec-1.0.jar"/>
<classpathentry kind="lib" path="lib/guava-r09-javadoc.jar"/>
<classpathentry kind="lib" path="lib/guava-r09-sources.jar"/>
<classpathentry kind="lib" path="lib/guava-r09.jar"/>
<classpathentry kind="lib" path="lib/hadoop-core-0.20.2.jar"/>
<classpathentry kind="lib" path="lib/hsqldb-1.8.0.10.jar"/>
<classpathentry kind="lib" path="lib/jasper-compiler-5.5.12.jar"/>
<classpathentry kind="lib" path="lib/jasper-runtime-5.5.12.jar"/>
<classpathentry kind="lib" path="lib/jets3t-0.7.1.jar"/>
<classpathentry kind="lib" path="lib/jetty-6.1.14.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-6.1.14.jar"/>
<classpathentry kind="lib" path="lib/joda-convert-1.1.jar"/>
<classpathentry kind="lib" path="lib/joda-time-2.0-javadoc.jar"/>
<classpathentry kind="lib" path="lib/joda-time-2.0-sources.jar"/>
<classpathentry kind="lib" path="lib/joda-time-2.0.jar"/>
<classpathentry kind="lib" path="lib/jsp-2.1-6.1.14.jar"/>
<classpathentry kind="lib" path="lib/jsp-api-2.1-6.1.14.jar"/>
<classpathentry kind="lib" path="lib/jsr305-1.3.7.jar"/>
<classpathentry kind="lib" path="lib/junit-4.5.jar"/>
<classpathentry kind="lib" path="lib/jwnl-1.3.3.jar"/>
<classpathentry kind="lib" path="lib/kfs-0.3.jar"/>
<classpathentry kind="lib" path="lib/log4j-1.2.16-javadoc.jar"/>
<classpathentry kind="lib" path="lib/log4j-1.2.16-sources.jar"/>
<classpathentry kind="lib" path="lib/log4j-1.2.16.jar"/>
<classpathentry kind="lib" path="lib/mail-1.4.1.jar"/>
<classpathentry kind="lib" path="lib/mavuno-models-0.2.jar"/>
<classpathentry kind="lib" path="lib/opennlp-maxent-3.0.1-incubating.jar"/>
<classpathentry kind="lib" path="lib/opennlp-tools-1.5.1-incubating-javadoc.jar"/>
<classpathentry kind="lib" path="lib/opennlp-tools-1.5.1-incubating-sources.jar"/>
<classpathentry kind="lib" path="lib/opennlp-tools-1.5.1-incubating.jar"/>
<classpathentry kind="lib" path="lib/oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/servlet-api-2.5-6.1.14.jar"/>
<classpathentry kind="lib" path="lib/stanford-corenlp-2012-01-08.jar"/>
<classpathentry kind="lib" path="lib/stanford-corenlp-models-2011-12-27.jar"/>
<classpathentry kind="lib" path="lib/xalan-2.7.0.jar"/>
<classpathentry kind="lib" path="lib/xercesImpl-2.8.0.jar"/>
<classpathentry kind="lib" path="lib/xml-apis-1.3.03.jar"/>
<classpathentry kind="lib" path="lib/xmlenc-0.52.jar"/>
<classpathentry kind="lib" path="lib/xom-1.2.5-javadoc.jar"/>
<classpathentry kind="lib" path="lib/xom-1.2.5-sources.jar"/>
<classpathentry kind="lib" path="lib/xom-1.2.5.jar"/>
<classpathentry kind="output" path="build"/>
</classpath>
5 changes: 3 additions & 2 deletions src/edu/isi/mavuno/app/util/ExamplesToSequenceFile.java
View file
@@ -1,11 +1,11 @@
/*
* Mavuno: A Hadoop-Based Text Mining Toolkit
*
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You may
* obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
Expand Down Expand Up @@ -93,6 +93,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("ExamplesToSequenceFile");
job.setJarByClass(ExamplesToSequenceFile.class);

FileInputFormat.addInputPath(job, new Path(contextPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/app/util/SequenceFileToText.java
View file
Expand Up @@ -81,6 +81,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("SequenceFileToText");
job.setJarByClass(SequenceFileToText.class);

FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/extract/CombineGlobalStats.java
View file
Expand Up @@ -92,6 +92,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("CombineGlobalStats");
job.setJarByClass(CombineGlobalStats.class);

for(int split = 0; split < numSplits; split++) {
FileInputFormat.addInputPath(job, new Path(inputPath + "/" + split));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/extract/CombineSplits.java
View file
Expand Up @@ -156,6 +156,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("CombineSplits");
job.setJarByClass(CombineSplits.class);

for(int split = 0; split < numSplits; split++) {
FileInputFormat.addInputPath(job, new Path(examplesPath + "/" + split));
Expand Down
3 changes: 2 additions & 1 deletion src/edu/isi/mavuno/extract/Extract.java
View file
Expand Up @@ -316,7 +316,7 @@ public int run(String[] args) throws ClassNotFoundException, InterruptedExceptio
return run();
}

@SuppressWarnings({ "unchecked", "rawtypes" })
@SuppressWarnings("unchecked")
public int run() throws ClassNotFoundException, InterruptedException, IOException {
Configuration conf = getConf();

Expand All @@ -341,6 +341,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("Extract");
job.setJarByClass(Extract.class);

MavunoUtils.recursivelyAddInputPaths(job, corpusPath);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
3 changes: 2 additions & 1 deletion src/edu/isi/mavuno/extract/ExtractGlobalStats.java
View file
Expand Up @@ -270,7 +270,7 @@ public int run(String[] args) throws ClassNotFoundException, InterruptedExceptio
return run();
}

@SuppressWarnings({ "unchecked", "rawtypes" })
@SuppressWarnings("unchecked")
public int run() throws ClassNotFoundException, InterruptedException, IOException {
Configuration conf = getConf();

Expand Down Expand Up @@ -311,6 +311,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("ExtractGlobalStats");
job.setJarByClass(ExtractGlobalStats.class);

MavunoUtils.recursivelyAddInputPaths(job, corpusPath);
FileOutputFormat.setOutputPath(job, new Path(outputPath + "/../split/" + split));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/extract/Split.java
View file
Expand Up @@ -184,6 +184,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("Split");
job.setJarByClass(Split.class);

MavunoUtils.recursivelyAddInputPaths(job, inputPath);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/score/CombineScores.java
View file
Expand Up @@ -112,6 +112,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("CombineScores");
job.setJarByClass(CombineScores.class);

MavunoUtils.recursivelyAddInputPaths(job, inputPath);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/score/GetTopResults.java
View file
Expand Up @@ -170,6 +170,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("GetTopResults");
job.setJarByClass(GetTopResults.class);

FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/score/ScoreContexts.java
View file
Expand Up @@ -205,6 +205,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("ScoreContexts");
job.setJarByClass(ScoreContexts.class);

FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/score/ScorePatterns.java
View file
Expand Up @@ -205,6 +205,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("ScorePatterns");
job.setJarByClass(ScorePatterns.class);

FileInputFormat.addInputPath(job, new Path(inputPath));
FileOutputFormat.setOutputPath(job, new Path(outputPath));
Expand Down
1 change: 1 addition & 0 deletions src/edu/isi/mavuno/score/UpdateWeights.java
View file
Expand Up @@ -150,6 +150,7 @@ public int run() throws ClassNotFoundException, InterruptedException, IOExceptio

Job job = new Job(conf);
job.setJobName("UpdateWeights");
job.setJarByClass(UpdateWeights.class);

FileInputFormat.addInputPath(job, new Path(statsPath));
FileInputFormat.addInputPath(job, new Path(scoresPath));
Expand Down

0 comments on commit 7e325f5

Please sign in to comment.