Skip to content
This repository has been archived by the owner on May 6, 2018. It is now read-only.

Commit

Permalink
Fixed broken build.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jimmy0017
jimmy0017 committed Dec 11, 2014
1 parent cb919a9 commit 42d06ed
Show file tree
Hide file tree
Showing 33 changed files with 298 additions and 298 deletions.
24 changes: 12 additions & 12 deletions .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,25 @@
<classpathentry kind="lib" path="lib/avro-1.7.6-cdh5.2.1.jar"/>
<classpathentry kind="lib" path="lib/bliki-core-3.0.16.jar"/>
<classpathentry kind="lib" path="lib/cglib-2.2.1-v20090111.jar"/>
<classpathentry kind="lib" path="lib/cloud9-2.0.0-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="lib/cloud9-2.0.0.jar"/>
<classpathentry kind="lib" path="lib/collections-generic-4.01.jar"/>
<classpathentry kind="lib" path="lib/colt-1.2.0.jar"/>
<classpathentry kind="lib" path="lib/commons-cli-1.2.jar"/>
<classpathentry kind="lib" path="lib/commons-codec-1.4.jar"/>
<classpathentry kind="lib" path="lib/commons-collections-20040616.jar"/>
<classpathentry kind="lib" path="lib/commons-collections-3.2.1.jar"/>
<classpathentry kind="lib" path="lib/commons-compress-1.4.1.jar"/>
<classpathentry kind="lib" path="lib/commons-configuration-1.8.jar"/>
<classpathentry kind="lib" path="lib/commons-el-1.0.jar"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.4.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
<classpathentry kind="lib" path="lib/commons-logging-1.1.3.jar"/>
<classpathentry kind="lib" path="lib/commons-math3-3.3.jar"/>
<classpathentry kind="lib" path="lib/commons-math3-3.1.1.jar"/>
<classpathentry kind="lib" path="lib/commons-net-3.1.jar"/>
<classpathentry kind="lib" path="lib/concurrent-1.3.4.jar"/>
<classpathentry kind="lib" path="lib/core-3.1.1.jar"/>
<classpathentry kind="lib" path="lib/dsiutils-2.2.2.jar"/>
<classpathentry kind="lib" path="lib/fastutil-6.5.16.jar"/>
<classpathentry kind="lib" path="lib/dsiutils-2.0.15.jar"/>
<classpathentry kind="lib" path="lib/fastutil-6.5.4.jar"/>
<classpathentry kind="lib" path="lib/findbugs-annotations-1.3.9-1.jar"/>
<classpathentry kind="lib" path="lib/gson-2.3.1.jar"/>
<classpathentry kind="lib" path="lib/guava-18.0.jar"/>
Expand Down Expand Up @@ -96,11 +96,11 @@
<classpathentry kind="lib" path="lib/jython-standalone-2.5.2.jar"/>
<classpathentry kind="lib" path="lib/kamikaze-3.0.3.jar"/>
<classpathentry kind="lib" path="lib/kfs-0.3.jar"/>
<classpathentry kind="lib" path="lib/lintools-datatypes-1.0.0-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="lib/lintools-datatypes-fastutil-1.0.0-SNAPSHOT.jar"/>
<classpathentry kind="lib" path="lib/lintools-datatypes-1.0.0.jar"/>
<classpathentry kind="lib" path="lib/lintools-datatypes-fastutil-1.0.0.jar"/>
<classpathentry kind="lib" path="lib/log4j-1.2.17.jar"/>
<classpathentry kind="lib" path="lib/logback-classic-1.1.2.jar"/>
<classpathentry kind="lib" path="lib/logback-core-1.1.2.jar"/>
<classpathentry kind="lib" path="lib/logback-classic-1.0.9.jar"/>
<classpathentry kind="lib" path="lib/logback-core-1.0.9.jar"/>
<classpathentry kind="lib" path="lib/lucene-analyzers-3.5.0.jar"/>
<classpathentry kind="lib" path="lib/lucene-core-3.5.0.jar"/>
<classpathentry kind="lib" path="lib/mail-1.4.5.jar"/>
Expand All @@ -112,12 +112,12 @@
<classpathentry kind="lib" path="lib/protobuf-java-2.5.0.jar"/>
<classpathentry kind="lib" path="lib/servlet-api-2.5-6.1.14.jar"/>
<classpathentry kind="lib" path="lib/servlet-api-2.5.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.7.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.5.jar"/>
<classpathentry kind="lib" path="lib/slf4j-log4j12-1.7.5.jar"/>
<classpathentry kind="lib" path="lib/snappy-java-1.0.4.1.jar"/>
<classpathentry kind="lib" path="lib/stanford-chinese-segmenter-2011-09-14-ivory.jar"/>
<classpathentry kind="lib" path="lib/stax-api-1.0-2.jar"/>
<classpathentry kind="lib" path="lib/sux4j-3.2.1.jar"/>
<classpathentry kind="lib" path="lib/sux4j-3.0.8.jar"/>
<classpathentry kind="lib" path="lib/tools-1.5.0.jar"/>
<classpathentry kind="lib" path="lib/xmlenc-0.52.jar"/>
<classpathentry kind="lib" path="lib/xz-1.0.jar"/>
Expand Down
10 changes: 5 additions & 5 deletions src/java/integration/ivory/integration/wikipedia/SearchSequenceFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import tl.lin.data.map.HMapSFW;
import tl.lin.data.map.HMapStFW;

/**
* Read sequence files, output key-value pairs that match specified key.
Expand Down Expand Up @@ -59,14 +59,14 @@ public SearchSequenceFiles() {
}

static class MyMapperTerm extends MapReduceBase implements
Mapper<IntWritable, HMapSFW, IntWritable, HMapSFW> {
Mapper<IntWritable, HMapStFW, IntWritable, HMapStFW> {
private String[] keys;

public void configure(JobConf job) {
keys = job.get("keys").split(",");
}

public void map(IntWritable key, HMapSFW value, OutputCollector<IntWritable, HMapSFW> output,
public void map(IntWritable key, HMapStFW value, OutputCollector<IntWritable, HMapStFW> output,
Reporter reporter) throws IOException {
for (String compareKey : keys) {
int k = Integer.parseInt(compareKey);
Expand Down Expand Up @@ -131,8 +131,8 @@ public int run(String[] args) throws Exception {

if (valueClassName.contains("HMapSFW")) {
job.setMapperClass(MyMapperTerm.class);
job.setMapOutputValueClass(HMapSFW.class);
job.setOutputValueClass(HMapSFW.class);
job.setMapOutputValueClass(HMapStFW.class);
job.setOutputValueClass(HMapStFW.class);
} else {
job.setMapperClass(MyMapperInt.class);
job.setMapOutputValueClass(WeightedIntDocVector.class);
Expand Down
12 changes: 6 additions & 6 deletions src/java/integration/ivory/integration/wikipedia/VerifyWikipediaProcessingCrosslingual.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.junit.Test;

import tl.lin.data.map.HMapIFW;
import tl.lin.data.map.HMapSFW;
import tl.lin.data.map.HMapStFW;
import tl.lin.data.map.MapIF;
import tl.lin.data.map.MapKF;

Expand Down Expand Up @@ -233,7 +233,7 @@ public void runBuildIndexEnSide() throws Exception {
"-input=" + enwikiEn + "/wt-term-doc-vectors",
"-output=" + enwikiEn + "/test_wt-term-doc-vectors",
"-keys=" + enTermDocVector1Id + "," + enTermDocVector2Id,
"-valueclass=" + HMapSFW.class.getCanonicalName()};
"-valueclass=" + HMapStFW.class.getCanonicalName()};
IntegrationUtils.exec(Joiner.on(" ").join(args));

args = new String[] { "hadoop jar", IntegrationUtils.getJar("dist", "ivory"),
Expand All @@ -252,7 +252,7 @@ public void verifyTermDocVectorsEn() throws Exception {

SequenceFile.Reader reader;
IntWritable key = new IntWritable();
HMapSFW value = new HMapSFW();
HMapStFW value = new HMapStFW();

reader = new SequenceFile.Reader(fs.getConf(),
SequenceFile.Reader.file(new Path(enwikiEn + "/test_wt-term-doc-vectors/part-00000")));
Expand Down Expand Up @@ -365,7 +365,7 @@ public void runBuildIndexDeSide() throws Exception {
"-input=" + dewikiEn + "/wt-term-doc-vectors",
"-output=" + dewikiEn + "/test_wt-term-doc-vectors",
"-keys=" + deTermDocVector1Id + "," + deTermDocVector2Id,
"-valueclass=" + HMapSFW.class.getCanonicalName()};
"-valueclass=" + HMapStFW.class.getCanonicalName()};
IntegrationUtils.exec(Joiner.on(" ").join(args));

args = new String[] { "hadoop jar", IntegrationUtils.getJar("dist", "ivory"),
Expand All @@ -384,7 +384,7 @@ public void verifyTermDocVectorsDe() throws Exception {

SequenceFile.Reader reader;
IntWritable key = new IntWritable();
HMapSFW value = new HMapSFW();
HMapStFW value = new HMapStFW();

reader = new SequenceFile.Reader(fs.getConf(),
SequenceFile.Reader.file(new Path(dewikiEn + "/test_wt-term-doc-vectors/part-00000")));
Expand Down Expand Up @@ -434,7 +434,7 @@ public void verifyIntDocVectorsDe() throws Exception {
reader.close();
}

private void verifyTermDocVector(Map<String, Float> doc, HMapSFW value) {
private void verifyTermDocVector(Map<String, Float> doc, HMapStFW value) {
assertTrue(value != null);
for (Map.Entry<String, Float> entry : doc.entrySet()) {
assertTrue(value.containsKey(entry.getKey()));
Expand Down
12 changes: 6 additions & 6 deletions src/java/integration/ivory/integration/wikipedia/VerifyWikipediaProcessingMonolingual.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.junit.Test;

import tl.lin.data.map.HMapIFW;
import tl.lin.data.map.HMapSFW;
import tl.lin.data.map.HMapStFW;
import tl.lin.data.map.MapIF;
import tl.lin.data.map.MapKF;

Expand Down Expand Up @@ -213,7 +213,7 @@ public void runBuildIndexGalago() throws Exception {
"-input=" + galagoIndex + "/wt-term-doc-vectors",
"-output=" + galagoIndex + "/test_wt-term-doc-vectors",
"-keys=" + galagoTermDocVector1Id + "," + galagoTermDocVector2Id,
"-valueclass=" + HMapSFW.class.getCanonicalName() };
"-valueclass=" + HMapStFW.class.getCanonicalName() };
IntegrationUtils.exec(Joiner.on(" ").join(args));

args = new String[] { "hadoop jar", IntegrationUtils.getJar("dist", "ivory"),
Expand All @@ -226,7 +226,7 @@ public void runBuildIndexGalago() throws Exception {

System.out.println("verifyTermDocVectorsGalago");
IntWritable key1 = new IntWritable();
HMapSFW value1 = new HMapSFW();
HMapStFW value1 = new HMapStFW();

SequenceFile.Reader reader1 = new SequenceFile.Reader(fs.getConf(),
SequenceFile.Reader.file(new Path(galagoIndex + "/test_wt-term-doc-vectors/part-00000")));
Expand Down Expand Up @@ -327,7 +327,7 @@ public void runBuildIndexOpennlp() throws Exception {
"-input=" + opennlpIndex + "/wt-term-doc-vectors",
"-output=" + opennlpIndex + "/test_wt-term-doc-vectors",
"-keys=" + opennlpTermDocVector1Id + "," + opennlpTermDocVector2Id,
"-valueclass=" + HMapSFW.class.getCanonicalName() };
"-valueclass=" + HMapStFW.class.getCanonicalName() };
IntegrationUtils.exec(Joiner.on(" ").join(args));

args = new String[] { "hadoop jar", IntegrationUtils.getJar("dist", "ivory"),
Expand All @@ -340,7 +340,7 @@ public void runBuildIndexOpennlp() throws Exception {

System.out.println("verifyTermDocVectorsOpennlp");
IntWritable key1 = new IntWritable();
HMapSFW value1 = new HMapSFW();
HMapStFW value1 = new HMapStFW();

SequenceFile.Reader reader1 = new SequenceFile.Reader(fs.getConf(),
SequenceFile.Reader.file(new Path(opennlpIndex + "/test_wt-term-doc-vectors/part-00000")));
Expand Down Expand Up @@ -385,7 +385,7 @@ public void runBuildIndexOpennlp() throws Exception {
reader2.close();
}

private void verifyTermDocVector(Map<String, Float> doc, HMapSFW value) {
private void verifyTermDocVector(Map<String, Float> doc, HMapStFW value) {
assertTrue(value != null);
for (Map.Entry<String, Float> entry : doc.entrySet()) {
System.out.println("checking " + entry.getKey() + ": expected = " + entry.getValue() + ", actual = " + value.get(entry.getKey()));
Expand Down
6 changes: 3 additions & 3 deletions src/java/main/ivory/core/preprocess/BuildTargetLangWeightedIntDocVectors.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import org.apache.log4j.Logger;

import tl.lin.data.map.HMapIFW;
import tl.lin.data.map.HMapSFW;
import tl.lin.data.map.HMapStFW;
import tl.lin.data.map.MapKF;
import edu.umd.cloud9.util.PowerTool;
import edu.umd.hooka.Vocab;
Expand Down Expand Up @@ -72,7 +72,7 @@ protected static enum Terms{
}

private static class MyMapper extends MapReduceBase implements
Mapper<IntWritable, HMapSFW, IntWritable, WeightedIntDocVector> {
Mapper<IntWritable, HMapStFW, IntWritable, WeightedIntDocVector> {

static IntWritable mDocno = new IntWritable();
private boolean normalize = false;
Expand Down Expand Up @@ -102,7 +102,7 @@ public void configure(JobConf conf){
HMapIFW weightedVector = new HMapIFW();

float sum2;
public void map(IntWritable docno, HMapSFW doc,
public void map(IntWritable docno, HMapStFW doc,
OutputCollector<IntWritable, WeightedIntDocVector> output, Reporter reporter)
throws IOException {
mDocno.set(docno.get());
Expand Down
12 changes: 6 additions & 6 deletions src/java/main/ivory/core/preprocess/BuildTranslatedTermDocVectors.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.log4j.Logger;

import tl.lin.data.map.HMapIFW;
import tl.lin.data.map.HMapSFW;
import tl.lin.data.map.HMapStFW;
import tl.lin.data.map.MapIF;

import com.google.common.collect.Maps;
Expand Down Expand Up @@ -66,7 +66,7 @@ protected static enum Docs { DBG, ZERO, SHORT, SHORTAfterTranslation, Total };
protected static enum DF { TransDf, NoDf }

private static class MyMapperTrans extends MapReduceBase implements
Mapper<IntWritable, TermDocVector, IntWritable, HMapSFW> {
Mapper<IntWritable, TermDocVector, IntWritable, HMapStFW> {

private ScoringModel model;
// eVocabSrc is the English vocabulary for probability table e2f_Probs.
Expand Down Expand Up @@ -209,7 +209,7 @@ public void configure(JobConf conf) {
}

public void map(IntWritable docno, TermDocVector doc,
OutputCollector<IntWritable, HMapSFW> output, Reporter reporter) throws IOException {
OutputCollector<IntWritable, HMapStFW> output, Reporter reporter) throws IOException {
if (docno.get() % SAMPLING != 0) {
return; // for generating sample document vectors. no sampling if SAMPLING=1
}
Expand All @@ -236,7 +236,7 @@ public void map(IntWritable docno, TermDocVector doc,
int docLen = CLIRUtils.translateTFs(doc, tfS, eVocabSrc, eVocabTrg, fVocabSrc, fVocabTrg,
e2f_Probs, f2e_Probs, tokenizer, LOG);

HMapSFW v = CLIRUtils.createTermDocVector(docLen, tfS, eVocabTrg, model, dict, dfTable,
HMapStFW v = CLIRUtils.createTermDocVector(docLen, tfS, eVocabTrg, model, dict, dfTable,
isNormalize, LOG);

// If no translation of any word is in the target vocab, remove document i.e., our model
Expand Down Expand Up @@ -354,9 +354,9 @@ public int runTool() throws Exception {

conf.setInputFormat(SequenceFileInputFormat.class);
conf.setMapOutputKeyClass(IntWritable.class);
conf.setMapOutputValueClass(HMapSFW.class);
conf.setMapOutputValueClass(HMapStFW.class);
conf.setOutputKeyClass(IntWritable.class);
conf.setOutputValueClass(HMapSFW.class);
conf.setOutputValueClass(HMapStFW.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);

conf.setMapperClass(MyMapperTrans.class);
Expand Down
12 changes: 6 additions & 6 deletions src/java/main/ivory/core/preprocess/BuildWeightedTermDocVectors.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.log4j.Logger;

import tl.lin.data.map.HMapSFW;
import tl.lin.data.map.HMapStFW;
import tl.lin.data.map.MapKF;

import com.google.common.collect.Maps;
Expand All @@ -61,7 +61,7 @@ public class BuildWeightedTermDocVectors extends PowerTool {
protected static enum Docs { Total, ZERO, SHORT }

private static class MyMapper extends MapReduceBase implements
Mapper<IntWritable, LazyTermDocVector, IntWritable, HMapSFW> {
Mapper<IntWritable, LazyTermDocVector, IntWritable, HMapStFW> {

static IntWritable mDocno = new IntWritable();
private static DocLengthTable mDLTable;
Expand All @@ -72,7 +72,7 @@ private static class MyMapper extends MapReduceBase implements
private boolean normalize = false;
DefaultFrequencySortedDictionary dict;
DfTableArray dfTable;
HMapSFW weightedVector = new HMapSFW();
HMapStFW weightedVector = new HMapStFW();
String term;
float wt, sum2;

Expand Down Expand Up @@ -162,7 +162,7 @@ public void configure(JobConf conf){
}

public void map(IntWritable docno, LazyTermDocVector doc,
OutputCollector<IntWritable, HMapSFW> output, Reporter reporter)
OutputCollector<IntWritable, HMapStFW> output, Reporter reporter)
throws IOException {
mDocno.set(docno.get());
int docLen = mDLTable.getDocLength(mDocno.get());
Expand Down Expand Up @@ -288,10 +288,10 @@ public int runTool() throws Exception {
FileOutputFormat.setOutputPath(conf, weightedVectorsPath);
conf.setInputFormat(SequenceFileInputFormat.class);
conf.setMapOutputKeyClass(IntWritable.class);
conf.setMapOutputValueClass(HMapSFW.class);
conf.setMapOutputValueClass(HMapStFW.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.setOutputKeyClass(IntWritable.class);
conf.setOutputValueClass(HMapSFW.class);
conf.setOutputValueClass(HMapStFW.class);

LOG.info("Running job: "+conf.getJobName());

Expand Down
Loading

0 comments on commit 42d06ed

Please sign in to comment.