Skip to content

Commit

Permalink
update MR tests (the old API ones)
Browse files Browse the repository at this point in the history 
relates to elastic#69
  • Loading branch information
@costin
costin committed Oct 31, 2013
1 parent 398ef9c commit 30acd93
Show file tree
Hide file tree
Showing 2 changed files with 164 additions and 9 deletions.
113 changes: 108 additions & 5 deletions src/test/java/org/elasticsearch/hadoop/integration/mr/MROldApiSaveTest.java
View file
Expand Up @@ -35,8 +35,11 @@
import org.elasticsearch.hadoop.integration.HdpBootstrap;
import org.elasticsearch.hadoop.mr.ESOutputFormat;
import org.elasticsearch.hadoop.util.WritableUtils;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;

@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class MROldApiSaveTest {

public static class JsonMapper extends MapReduceBase implements Mapper {
Expand All @@ -58,7 +61,7 @@ public void map(Object key, Object value, OutputCollector output, Reporter repor
}

@Test
public void testBasicSave() throws Exception {
public void testBasicIndex() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
Expand All @@ -69,13 +72,13 @@ public void testBasicSave() throws Exception {
conf.setBoolean("mapred.used.genericoptionsparser", true);

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));
conf.set("es.resource", "mroldapi/save");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/save");

JobClient.runJob(conf);
}

@Test
public void testSaveWithId() throws Exception {
public void testBasicIndexWithId() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
Expand All @@ -84,10 +87,110 @@ public void testSaveWithId() throws Exception {
conf.setMapperClass(JsonMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.set("es.mapping.id", "number");
conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/savewithid");

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));
conf.set("es.resource", "mroldapi/savewithid");

JobClient.runJob(conf);
}

@Test
public void testCreateWithId() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(ESOutputFormat.class);
conf.setMapOutputValueClass(MapWritable.class);
conf.setMapperClass(JsonMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));

conf.set(ConfigurationOptions.ES_OPERATION, "create");
conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");

JobClient.runJob(conf);
}

@Test(expected = IOException.class)
public void testCreateWithIdShouldFailOnDuplicate() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(ESOutputFormat.class);
conf.setMapOutputValueClass(MapWritable.class);
conf.setMapperClass(JsonMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));

conf.set(ConfigurationOptions.ES_OPERATION, "create");
conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid");

JobClient.runJob(conf);
}

@Test(expected = IllegalArgumentException.class)
public void testUpdateWithoutId() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(ESOutputFormat.class);
conf.setMapOutputValueClass(MapWritable.class);
conf.setMapperClass(JsonMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));

conf.set(ConfigurationOptions.ES_OPERATION, "update");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update");

JobClient.runJob(conf);
}

@Test
public void testUpdateWithId() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(ESOutputFormat.class);
conf.setMapOutputValueClass(MapWritable.class);
conf.setMapperClass(JsonMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));

conf.set(ConfigurationOptions.ES_OPERATION, "update");
conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update");

JobClient.runJob(conf);
}

@Test(expected = IOException.class)
public void testUpdateWithoutUpsert() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(ESOutputFormat.class);
conf.setMapOutputValueClass(MapWritable.class);
conf.setMapperClass(JsonMapper.class);
conf.setReducerClass(IdentityReducer.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);

FileInputFormat.setInputPaths(conf, new Path("src/test/resources/artists.dat"));

conf.set(ConfigurationOptions.ES_OPERATION, "update");
conf.set(ConfigurationOptions.ES_MAPPING_ID, "number");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/updatewoupsert");
conf.set(ConfigurationOptions.ES_UPSERT_DOC, "false");

JobClient.runJob(conf);
}
Expand Down
60 changes: 56 additions & 4 deletions src/test/java/org/elasticsearch/hadoop/integration/mr/MROldApiSearchTest.java
View file
Expand Up @@ -35,7 +35,7 @@ public void testBasicSearch() throws Exception {
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(MapWritable.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.set("es.resource", "mroldapi/save/_search?q=*");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/save/_search?q=*");

// un-comment to print results to the console (works only in local mode)
//PrintStreamOutputFormat.stream(conf, Stream.OUT);
Expand All @@ -52,7 +52,7 @@ public void testSearchWithId() throws Exception {
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(MapWritable.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.set("es.resource", "mroldapi/savewithid/_search?q=*");
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/savewithid/_search?q=*");

// un-comment to print results to the console (works only in local mode)
//PrintStreamOutputFormat.stream(conf, Stream.OUT);
Expand All @@ -70,11 +70,63 @@ public void testSearchNonExistingIndex() throws Exception {
conf.setOutputValueClass(MapWritable.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.setBoolean(ConfigurationOptions.ES_INDEX_READ_MISSING_AS_EMPTY, true);
conf.set("es.resource", "foobar/save/_search?q=*");
conf.set(ConfigurationOptions.ES_RESOURCE, "foobar/save/_search?q=*");

// un-comment to print results to the console (works only in local mode)
//PrintStreamOutputFormat.stream(conf, Stream.OUT);

JobClient.runJob(conf);
}
}

@Test
public void testSearchCreated() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(ESInputFormat.class);
conf.setOutputFormat(PrintStreamOutputFormat.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(MapWritable.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/createwithid/_search?q=*");

// un-comment to print results to the console (works only in local mode)
//PrintStreamOutputFormat.stream(conf, Stream.OUT);

JobClient.runJob(conf);
}

@Test
public void testSearchUpdated() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(ESInputFormat.class);
conf.setOutputFormat(PrintStreamOutputFormat.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(MapWritable.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/update/_search?q=*");

// un-comment to print results to the console (works only in local mode)
//PrintStreamOutputFormat.stream(conf, Stream.OUT);

JobClient.runJob(conf);
}

@Test(expected = IllegalStateException.class)
public void testSearchUpdatedWithoutUpsertMeaningNonExistingIndex() throws Exception {
JobConf conf = HdpBootstrap.hadoopConfig();

conf.setInputFormat(ESInputFormat.class);
conf.setOutputFormat(PrintStreamOutputFormat.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(MapWritable.class);
conf.setBoolean("mapred.used.genericoptionsparser", true);
conf.setBoolean(ConfigurationOptions.ES_INDEX_READ_MISSING_AS_EMPTY, false);
conf.set(ConfigurationOptions.ES_RESOURCE, "mroldapi/updatewoupsert/_search?q=*");

// un-comment to print results to the console (works only in local mode)
//PrintStreamOutputFormat.stream(conf, Stream.OUT);

JobClient.runJob(conf);
}
}

0 comments on commit 30acd93

Please sign in to comment.