From 0c9c92c9a1ddb50858738a16c146cfacbab8be91 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 6 Nov 2018 13:00:20 -0500 Subject: [PATCH] Got most ITs working (#30) --- .../examples/cli/MapReduceClientOnRequiredTable.java | 7 +++++++ .../examples/client/CountingVerifyingReceiver.java | 2 +- .../accumulo/examples/shard/ContinuousQuery.java | 8 +++++--- .../org/apache/accumulo/examples/shard/Reverse.java | 10 ++++++---- .../java/org/apache/accumulo/examples/ExamplesIT.java | 11 ++++------- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/apache/accumulo/examples/cli/MapReduceClientOnRequiredTable.java b/src/main/java/org/apache/accumulo/examples/cli/MapReduceClientOnRequiredTable.java index 520107b..ec8f6aa 100644 --- a/src/main/java/org/apache/accumulo/examples/cli/MapReduceClientOnRequiredTable.java +++ b/src/main/java/org/apache/accumulo/examples/cli/MapReduceClientOnRequiredTable.java @@ -17,6 +17,7 @@ package org.apache.accumulo.examples.cli; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; @@ -39,6 +40,12 @@ public void setAccumuloConfigs(Job job) throws AccumuloSecurityException { final String principal = getPrincipal(), tableName = getTableName(); + ClientConfiguration cc = ClientConfiguration.create() + .withInstance(getClientInfo().getInstanceName()) + .withZkHosts(getClientInfo().getZooKeepers()); + AccumuloInputFormat.setZooKeeperInstance(job, cc); + AccumuloOutputFormat.setZooKeeperInstance(job, cc); + if (tokenFile.isEmpty()) { AuthenticationToken token = getToken(); AccumuloInputFormat.setConnectorInfo(job, principal, token); diff --git a/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java b/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java index 1806161..9b3d909 100644 --- a/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java +++ b/src/main/java/org/apache/accumulo/examples/client/CountingVerifyingReceiver.java @@ -53,7 +53,7 @@ public void receive(Key key, Value value) { + new String(expectedValue, UTF_8) + " got : " + new String(value.get(), UTF_8)); } - if (!expectedRows.containsKey(key.getRow())) { + if (!expectedRows.containsKey(key.getRow().toString())) { log.error("Got unexpected key " + key); } else { expectedRows.put(key.getRow().toString(), true); diff --git a/src/main/java/org/apache/accumulo/examples/shard/ContinuousQuery.java b/src/main/java/org/apache/accumulo/examples/shard/ContinuousQuery.java index 9911b7b..7e6f340 100644 --- a/src/main/java/org/apache/accumulo/examples/shard/ContinuousQuery.java +++ b/src/main/java/org/apache/accumulo/examples/shard/ContinuousQuery.java @@ -22,6 +22,7 @@ import java.util.Map.Entry; import java.util.Random; +import org.apache.accumulo.core.cli.Help; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; @@ -32,7 +33,6 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.IntersectingIterator; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.examples.cli.Help; import org.apache.hadoop.io.Text; import com.beust.jcommander.Parameter; @@ -47,6 +47,9 @@ public class ContinuousQuery { static class Opts extends Help { + @Parameter(names = "-c", description = "Accumulo client properties file") + String clientProps = "conf/accumulo-client.properties"; + @Parameter(names = "--shardTable", required = true, description = "name of the shard table") String tableName = null; @@ -64,8 +67,7 @@ public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(ContinuousQuery.class.getName(), args); - AccumuloClient client = Accumulo.newClient().usingProperties("conf/accumulo-client.properties") - .build(); + AccumuloClient client = Accumulo.newClient().usingProperties(opts.clientProps).build(); ArrayList randTerms = findRandomTerms( client.createScanner(opts.doc2Term, Authorizations.EMPTY), opts.numTerms); diff --git a/src/main/java/org/apache/accumulo/examples/shard/Reverse.java b/src/main/java/org/apache/accumulo/examples/shard/Reverse.java index 3db5601..bfd3534 100644 --- a/src/main/java/org/apache/accumulo/examples/shard/Reverse.java +++ b/src/main/java/org/apache/accumulo/examples/shard/Reverse.java @@ -40,10 +40,13 @@ public class Reverse { static class Opts extends Help { - @Parameter(names = "--shardTable") + @Parameter(names = "-c", description = "Accumulo client properties file") + String clientProps = "conf/accumulo-client.properties"; + + @Parameter(names = "--shardTable", description = "name of the shard table") String shardTable = "shard"; - @Parameter(names = "--doc2Term") + @Parameter(names = "--doc2Term", description = "name of the doc2Term table") String doc2TermTable = "doc2Term"; } @@ -51,8 +54,7 @@ public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(Reverse.class.getName(), args); - AccumuloClient client = Accumulo.newClient().usingProperties("conf/accumulo-client.properties") - .build(); + AccumuloClient client = Accumulo.newClient().usingProperties(opts.clientProps).build(); try (Scanner scanner = client.createScanner(opts.shardTable, Authorizations.EMPTY); BatchWriter bw = client.createBatchWriter(opts.doc2TermTable)) { diff --git a/src/test/java/org/apache/accumulo/examples/ExamplesIT.java b/src/test/java/org/apache/accumulo/examples/ExamplesIT.java index bddddff..fe81c16 100644 --- a/src/test/java/org/apache/accumulo/examples/ExamplesIT.java +++ b/src/test/java/org/apache/accumulo/examples/ExamplesIT.java @@ -192,7 +192,8 @@ public void testTrace() throws Exception { count++; } assertTrue(count > 0); - assertTrue("Output did not contain myApp@myHost", pair.getValue().contains("myApp@myHost")); + assertTrue("Output [" + pair.getValue() + "] did not contain myApp@myHost", + pair.getValue().contains("myApp@myHost")); if (ClusterType.MINI == getClusterType() && null != trace) { trace.destroy(); } @@ -437,18 +438,14 @@ public void testRowOperations() throws Exception { goodExec(RowOperations.class, "-c", getClientPropsFile()); } - @Test - public void testSequentialBatchWriter() throws Exception { - goodExec(SequentialBatchWriter.class, "-c", getClientPropsFile()); - } - @Test public void testReadWriteAndDelete() throws Exception { goodExec(ReadWriteExample.class, "-c", getClientPropsFile()); } @Test - public void testRandomBatchScanner() throws Exception { + public void testBatch() throws Exception { + goodExec(SequentialBatchWriter.class, "-c", getClientPropsFile()); goodExec(RandomBatchScanner.class, "-c", getClientPropsFile()); }