Skip to content
Permalink
Browse files
Updated examples to work with Accumulo 2.0.0-alpha-1
  • Loading branch information
mikewalch committed Oct 26, 2018
1 parent a6ccf92 commit dafe8b053d187b86bb774be7dd27f0f7bb5f0997
Showing 43 changed files with 391 additions and 410 deletions.
12 pom.xml
@@ -33,8 +33,8 @@
<description>Example code and corresponding documentation for using Apache Accumulo</description>

<properties>
<accumulo.version>2.0.0-SNAPSHOT</accumulo.version>
<hadoop.version>2.6.4</hadoop.version>
<accumulo.version>2.0.0-alpha-1</accumulo.version>
<hadoop.version>3.1.1</hadoop.version>
<slf4j.version>1.7.21</slf4j.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
@@ -45,7 +45,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
<version>26.0-jre</version>
</dependency>
</dependencies>
</dependencyManagement>
@@ -131,17 +131,17 @@
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>1.48</version>
<version>1.72</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0.1</version>
<version>26.0-jre</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.6</version>
<version>1.10</version>
</dependency>
<dependency>
<groupId>jline</groupId>
@@ -23,10 +23,11 @@
import java.util.Map.Entry;
import java.util.Random;

import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
@@ -39,13 +40,13 @@
public class BloomBatchScanner {

public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Connector connector = Connector.builder().usingProperties("conf/accumulo-client.properties").build();
AccumuloClient client = Accumulo.newClient().usingProperties("conf/accumulo-client.properties").build();

scan(connector, "bloom_test1", 7);
scan(connector, "bloom_test2", 7);
scan(client, "bloom_test1", 7);
scan(client, "bloom_test2", 7);
}

static void scan(Connector connector, String tableName, int seed) throws TableNotFoundException {
static void scan(AccumuloClient client, String tableName, int seed) throws TableNotFoundException {
Random r = new Random(seed);
HashSet<Range> ranges = new HashSet<>();
HashMap<String,Boolean> expectedRows = new HashMap<>();
@@ -61,7 +62,7 @@ static void scan(Connector connector, String tableName, int seed) throws TableNo
long lookups = ranges.size();

System.out.println("Scanning " + tableName + " with seed " + seed);
try (BatchScanner scan = connector.createBatchScanner(tableName, Authorizations.EMPTY, 20)) {
try (BatchScanner scan = client.createBatchScanner(tableName, Authorizations.EMPTY, 20)) {
scan.setRanges(ranges);
for (Entry<Key, Value> entry : scan) {
Key key = entry.getKey();
@@ -18,10 +18,11 @@

import java.util.Random;

import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -32,41 +33,41 @@
public class BloomFilters {

public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Connector connector = Connector.builder().usingProperties("conf/accumulo-client.properties").build();
AccumuloClient client = Accumulo.newClient().usingProperties("conf/accumulo-client.properties").build();
try {
System.out.println("Creating bloom_test1 and bloom_test2");
connector.tableOperations().create("bloom_test1");
connector.tableOperations().setProperty("bloom_test1", "table.compaction.major.ratio", "7");
connector.tableOperations().create("bloom_test2");
connector.tableOperations().setProperty("bloom_test2", "table.bloom.enabled", "true");
connector.tableOperations().setProperty("bloom_test2", "table.compaction.major.ratio", "7");
client.tableOperations().create("bloom_test1");
client.tableOperations().setProperty("bloom_test1", "table.compaction.major.ratio", "7");
client.tableOperations().create("bloom_test2");
client.tableOperations().setProperty("bloom_test2", "table.bloom.enabled", "true");
client.tableOperations().setProperty("bloom_test2", "table.compaction.major.ratio", "7");
} catch (TableExistsException e) {
// ignore
}

// Write a million rows 3 times flushing files to disk separately
System.out.println("Writing data to bloom_test1");
writeData(connector, "bloom_test1", 7);
connector.tableOperations().flush("bloom_test1", null, null, true);
writeData(connector, "bloom_test1", 8);
connector.tableOperations().flush("bloom_test1", null, null, true);
writeData(connector, "bloom_test1", 9);
connector.tableOperations().flush("bloom_test1", null, null, true);
writeData(client, "bloom_test1", 7);
client.tableOperations().flush("bloom_test1", null, null, true);
writeData(client, "bloom_test1", 8);
client.tableOperations().flush("bloom_test1", null, null, true);
writeData(client, "bloom_test1", 9);
client.tableOperations().flush("bloom_test1", null, null, true);

System.out.println("Writing data to bloom_test2");
writeData(connector, "bloom_test2", 7);
connector.tableOperations().flush("bloom_test2", null, null, true);
writeData(connector, "bloom_test2", 8);
connector.tableOperations().flush("bloom_test2", null, null, true);
writeData(connector, "bloom_test2", 9);
connector.tableOperations().flush("bloom_test2", null, null, true);
writeData(client, "bloom_test2", 7);
client.tableOperations().flush("bloom_test2", null, null, true);
writeData(client, "bloom_test2", 8);
client.tableOperations().flush("bloom_test2", null, null, true);
writeData(client, "bloom_test2", 9);
client.tableOperations().flush("bloom_test2", null, null, true);
}

// write a million random rows
static void writeData(Connector connector, String tableName, int seed) throws TableNotFoundException,
static void writeData(AccumuloClient client, String tableName, int seed) throws TableNotFoundException,
MutationsRejectedException{
Random r = new Random(seed);
try (BatchWriter bw = connector.createBatchWriter(tableName)) {
try (BatchWriter bw = client.createBatchWriter(tableName)) {
for (int x = 0; x < 1_000_000; x++) {
Long rowId = RandomBatchWriter.abs(r.nextLong()) % 1_000_000_000;
Mutation m = RandomBatchWriter.createMutation(rowId, 50, new ColumnVisibility());
@@ -18,30 +18,31 @@

import static org.apache.accumulo.examples.bloom.BloomFilters.writeData;

import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;

public class BloomFiltersNotFound {

public static void main(String[] args) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
Connector connector = Connector.builder().usingProperties("conf/accumulo-client.properties").build();
AccumuloClient client = Accumulo.newClient().usingProperties("conf/accumulo-client.properties").build();
try {
connector.tableOperations().create("bloom_test3");
connector.tableOperations().create("bloom_test4");
connector.tableOperations().setProperty("bloom_test4", "table.bloom.enabled", "true");
client.tableOperations().create("bloom_test3");
client.tableOperations().create("bloom_test4");
client.tableOperations().setProperty("bloom_test4", "table.bloom.enabled", "true");
} catch (TableExistsException e) {
// ignore
}
System.out.println("Writing data to bloom_test3 and bloom_test4 (bloom filters enabled)");
writeData(connector, "bloom_test3", 7);
connector.tableOperations().flush("bloom_test3", null, null, true);
writeData(connector, "bloom_test4", 7);
connector.tableOperations().flush("bloom_test4", null, null, true);
writeData(client, "bloom_test3", 7);
client.tableOperations().flush("bloom_test3", null, null, true);
writeData(client, "bloom_test4", 7);
client.tableOperations().flush("bloom_test4", null, null, true);

BloomBatchScanner.scan(connector, "bloom_test3", 8);
BloomBatchScanner.scan(connector, "bloom_test4", 8);
BloomBatchScanner.scan(client, "bloom_test3", 8);
BloomBatchScanner.scan(client, "bloom_test4", 8);
}
}
@@ -19,18 +19,14 @@
import java.io.File;
import java.time.Duration;

import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.ClientInfo;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;

import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
@@ -98,40 +94,38 @@ public File convert(String filename) {
}

@Parameter(names = {"-c", "--conf"}, required = true, converter = PropertiesConverter.class,
description = "Config file for connecting to Accumulo. See README.md for details.")
description = "Accumulo client properties file. See README.md for details.")
private File config = null;

@Parameter(names = {"-auths", "--auths"}, converter = AuthConverter.class, description = "the authorizations to use when reading or writing")
public Authorizations auths = Authorizations.EMPTY;

public Connector getConnector() {
try {
ZooKeeperInstance zki = new ZooKeeperInstance(getClientConfiguration());
return zki.getConnector(getPrincipal(), getToken());
} catch (AccumuloException | AccumuloSecurityException e) {
throw new RuntimeException(e);
private ClientInfo cachedInfo = null;
private AccumuloClient cachedAccumuloClient = null;

public AccumuloClient getAccumuloClient() {
if (cachedAccumuloClient == null) {
try {
cachedAccumuloClient = Accumulo.newClient().usingClientInfo(getClientInfo()).build();
} catch (AccumuloException|AccumuloSecurityException e) {
throw new IllegalArgumentException(e);
}
}
return cachedAccumuloClient;
}

public ClientConfiguration getClientConfiguration() {
return ClientConfiguration.fromFile(config);
public ClientInfo getClientInfo() {
if (cachedInfo == null) {
cachedInfo = Accumulo.newClient().usingProperties(config.getAbsolutePath()).info();
}
return cachedInfo;
}

public String getPrincipal() {
String user = getClientConfiguration().getString("accumulo.examples.principal");
if(user != null)
return user;

return "root";
return getClientInfo().getPrincipal();
}

public AuthenticationToken getToken() {
AuthenticationToken token = new PasswordToken("secret");
String password = getClientConfiguration().getString("accumulo.examples.password");
if(password != null){
token = new PasswordToken(password);
}

return token;
return getClientInfo().getAuthenticationToken();
}
}
@@ -16,11 +16,10 @@
*/
package org.apache.accumulo.examples.cli;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.security.SystemPermission;
@@ -36,8 +35,8 @@ public class MapReduceClientOpts extends ClientOpts {
private static final Logger log = LoggerFactory.getLogger(MapReduceClientOpts.class);

public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
AccumuloInputFormat.setZooKeeperInstance(job, this.getClientConfiguration());
AccumuloOutputFormat.setZooKeeperInstance(job, this.getClientConfiguration());
AccumuloInputFormat.setClientInfo(job, this.getClientInfo());
AccumuloInputFormat.setClientInfo(job, this.getClientInfo());
}

@Override
@@ -57,19 +56,19 @@ public AuthenticationToken getToken() {
String newPrincipal = user.getUserName();
log.info("Obtaining delegation token for {}", newPrincipal);

Connector conn = getConnector();
AccumuloClient client = getAccumuloClient();

// Do the explicit check to see if the user has the permission to get a delegation token
if (!conn.securityOperations().hasSystemPermission(conn.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
if (!client.securityOperations().hasSystemPermission(client.whoami(), SystemPermission.OBTAIN_DELEGATION_TOKEN)) {
log.error(
"{} doesn't have the {} SystemPermission neccesary to obtain a delegation token. MapReduce tasks cannot automatically use the client's"
+ " credentials on remote servers. Delegation tokens provide a means to run MapReduce without distributing the user's credentials.",
user.getUserName(), SystemPermission.OBTAIN_DELEGATION_TOKEN.name());
throw new IllegalStateException(conn.whoami() + " does not have permission to obtain a delegation token");
throw new IllegalStateException(client.whoami() + " does not have permission to obtain a delegation token");
}

// Get the delegation token from Accumulo
return conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
return client.securityOperations().getDelegationToken(new DelegationTokenConfig());
} catch (Exception e) {
final String msg = "Failed to acquire DelegationToken for use with MapReduce";
log.error(msg, e);
@@ -23,7 +23,6 @@

import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -16,7 +16,7 @@
*/
package org.apache.accumulo.examples.client;

import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.examples.cli.ClientOnRequiredTable;

/**
@@ -28,8 +28,8 @@ public static void main(String[] args) {
ClientOnRequiredTable opts = new ClientOnRequiredTable();
opts.parseArgs(Flush.class.getName(), args);
try {
Connector connector = opts.getConnector();
connector.tableOperations().flush(opts.getTableName(), null, null, true);
AccumuloClient client = opts.getAccumuloClient();
client.tableOperations().flush(opts.getTableName(), null, null, true);
} catch (Exception e) {
throw new RuntimeException(e);
}

0 comments on commit dafe8b0

Please sign in to comment.