Skip to content
Permalink
Browse files
Apply formatter changes on build
  • Loading branch information
ctubbsii committed Mar 7, 2019
1 parent 31ee5bd commit 48d255d7be9c8ba4f10a212751128954f77f7b9b
Show file tree
Hide file tree
Showing 83 changed files with 414 additions and 379 deletions.
@@ -23,20 +23,20 @@ public class TestEnv implements AutoCloseable {

public TestEnv(String[] args) {

Map<String, String> options = new HashMap<>();
Map<String,String> options = new HashMap<>();
List<String> arguments = new ArrayList<>();

for (int i = 0; i < args.length; i++) {
if(args[i].equals("-o")) {
if (args[i].equals("-o")) {
i++;
String[] tokens = args[i].split("=",2);
String[] tokens = args[i].split("=", 2);
options.put(tokens[0], tokens[1]);
} else {
arguments.add(args[i]);
}
}

if(arguments.size() != 2) {
if (arguments.size() != 2) {
throw new IllegalArgumentException("Expected <testPropsPath> <clientPropsPath> arguments.");
}

@@ -46,7 +46,7 @@ public TestEnv(String[] args) {
this.testProps = TestProps.loadFromFile(testPropsPath);
this.clientProps = Accumulo.newClientProperties().from(clientPropsPath).build();

options.forEach((k,v) -> testProps.setProperty(k, v));
options.forEach((k, v) -> testProps.setProperty(k, v));
}

public TestEnv(String testPropsPath, String clientPropsPath) {
@@ -118,8 +118,8 @@ public Configuration getHadoopConfiguration() {
hadoopConfig.set("fs.defaultFS", getHdfsRoot());
// Below is required due to bundled jar breaking default config.
// See http://stackoverflow.com/questions/17265002/hadoop-no-filesystem-for-scheme-file
hadoopConfig
.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
hadoopConfig.set("fs.hdfs.impl",
org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
hadoopConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
hadoopConfig.set("mapreduce.framework.name", "yarn");
hadoopConfig.set("yarn.resourcemanager.hostname", getYarnResourceManager());
@@ -160,8 +160,8 @@ public AccumuloClient createClient() {

public String getClientConfigFile() {
if (clientConfigFile == null) {
URL clientPropsUrl = ClientOpts.class.getClassLoader().getResource(
"accumulo-client.properties");
URL clientPropsUrl = ClientOpts.class.getClassLoader()
.getResource("accumulo-client.properties");
if (clientPropsUrl != null) {
clientConfigFile = clientPropsUrl.getFile();
}
@@ -41,8 +41,8 @@
import org.slf4j.LoggerFactory;

/**
* Bulk import a million random key value pairs. Same format as ContinuousIngest and can be
* verified by running ContinuousVerify.
* Bulk import a million random key value pairs. Same format as ContinuousIngest and can be verified
* by running ContinuousVerify.
*/
public class BulkIngest extends Configured implements Tool {
public static final int NUM_KEYS = 1_000_000;
@@ -88,8 +88,8 @@ public int run(String[] args) throws Exception {
try (AccumuloClient client = env.getAccumuloClient()) {

// make sure splits file is closed before continuing
try (PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(
splitsFile))))) {
try (PrintStream out = new PrintStream(
new BufferedOutputStream(fs.create(new Path(splitsFile))))) {
Collection<Text> splits = client.tableOperations().listSplits(tableName, 100);
for (Text split : splits) {
out.println(Base64.getEncoder().encodeToString(split.copyBytes()));
@@ -151,8 +151,8 @@ protected void setup(Context context) {
}

@Override
protected void map(LongWritable key, LongWritable value, Context context) throws IOException,
InterruptedException {
protected void map(LongWritable key, LongWritable value, Context context)
throws IOException, InterruptedException {
currentRow.set(ContinuousIngest.genRow(key.get()));

// hack since we can't pass null - don't set first val (prevRow), we want it to be null
@@ -162,16 +162,16 @@ protected void map(LongWritable key, LongWritable value, Context context) throws
}

Key outputKey = new Key(currentRow, emptyCfCq, emptyCfCq);
Value outputValue = ContinuousIngest.createValue(uuid.getBytes(), 0,
currentValue.copyBytes(), null);
Value outputValue = ContinuousIngest.createValue(uuid.getBytes(), 0, currentValue.copyBytes(),
null);

context.write(outputKey, outputValue);
}
}

/**
* Generates a million LongWritable keys. The LongWritable value points to the previous key.
* The first key value pair has a value of 1L. This is translated to null in RandomMapper
* Generates a million LongWritable keys. The LongWritable value points to the previous key. The
* first key value pair has a value of 1L. This is translated to null in RandomMapper
*/
public static class RandomInputFormat extends InputFormat {

@@ -136,8 +136,8 @@ public static void main(String[] args) throws Exception {

byte[] ingestInstanceId = UUID.randomUUID().toString().getBytes(UTF_8);

log.info(String.format("UUID %d %s", System.currentTimeMillis(), new String(ingestInstanceId,
UTF_8)));
log.info(String.format("UUID %d %s", System.currentTimeMillis(),
new String(ingestInstanceId, UTF_8)));

long count = 0;
final int flushInterval = getFlushEntries(env.getTestProperties());
@@ -38,8 +38,8 @@ public String convert(String debugLog) {
logger.setLevel(Level.TRACE);
logger.setAdditivity(false);
try {
logger.addAppender(new FileAppender(new PatternLayout(
"%d{dd HH:mm:ss,SSS} [%-8c{2}] %-5p: %m%n"), debugLog, true));
logger.addAppender(new FileAppender(
new PatternLayout("%d{dd HH:mm:ss,SSS} [%-8c{2}] %-5p: %m%n"), debugLog, true));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
@@ -88,7 +88,7 @@ public static void main(String[] args) throws Exception {
}

// System.out.println("P2 "+delta
// +" "+numToScan+" "+distance+" "+((double)numToScan/count ));
// +" "+numToScan+" "+distance+" "+((double)numToScan/count ));
}

System.out.printf("SCN %d %s %d %d%n", t1, new String(scanStart, UTF_8), (t2 - t1), count);
@@ -145,14 +145,14 @@ public int run(String[] args) throws Exception {

String tableName = env.getAccumuloTableName();

Job job = Job.getInstance(getConf(), this.getClass().getSimpleName() + "_" + tableName + "_"
+ System.currentTimeMillis());
Job job = Job.getInstance(getConf(),
this.getClass().getSimpleName() + "_" + tableName + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());

job.setInputFormatClass(AccumuloInputFormat.class);

boolean scanOffline = Boolean.parseBoolean(env
.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
boolean scanOffline = Boolean
.parseBoolean(env.getTestProperty(TestProps.CI_VERIFY_SCAN_OFFLINE));
int maxMaps = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_MAX_MAPS));
int reducers = Integer.parseInt(env.getTestProperty(TestProps.CI_VERIFY_REDUCERS));
String outputDir = env.getTestProperty(TestProps.CI_VERIFY_OUTPUT_DIR);
@@ -39,8 +39,8 @@ public static void main(String[] args) throws Exception {
System.exit(-1);
}

int numTablets = Integer.parseInt(env
.getTestProperty(TestProps.CI_COMMON_ACCUMULO_NUM_TABLETS));
int numTablets = Integer
.parseInt(env.getTestProperty(TestProps.CI_COMMON_ACCUMULO_NUM_TABLETS));

if (numTablets < 1) {
System.err.println("ERROR: numTablets < 1");
@@ -70,8 +70,8 @@ public static void main(String[] args) throws Exception {

client.tableOperations().create(tableName, ntc);

System.out.println("Created Accumulo table '" + tableName + "' with " + numTablets
+ " tablets");
System.out
.println("Created Accumulo table '" + tableName + "' with " + numTablets + " tablets");
}
}

@@ -35,7 +35,13 @@
public class TimeBinner {

enum Operation {
AVG, SUM, MIN, MAX, COUNT, CUMULATIVE, AMM, // avg,min,max
AVG,
SUM,
MIN,
MAX,
COUNT,
CUMULATIVE,
AMM, // avg,min,max
AMM_HACK1 // special case
}

@@ -16,6 +16,8 @@
*/
package org.apache.accumulo.testing.continuous;

import static java.nio.charset.StandardCharsets.UTF_8;

import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
@@ -31,7 +33,6 @@
import java.util.Map.Entry;
import java.util.TreeMap;

import com.beust.jcommander.Parameter;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.data.Key;
@@ -40,7 +41,7 @@
import org.apache.accumulo.testing.cli.ClientOpts;
import org.apache.hadoop.io.Text;

import static java.nio.charset.StandardCharsets.UTF_8;
import com.beust.jcommander.Parameter;

/**
* BUGS This code does not handle the fact that these files could include log events from previous
@@ -197,14 +198,16 @@ static class TabletHistory {

if (pos1 > 0 && pos2 > 0 && pos3 == -1) {
String tid = tablet.substring(0, pos1);
String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000" : tablet.substring(pos1 + 1, pos2);
String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000"
: tablet.substring(pos1 + 1, pos2);
String prevEndRow = tablet.charAt(pos2) == '<' ? "" : tablet.substring(pos2 + 1);
if (tid.equals(tableId)) {
// System.out.println(" "+server+" "+tid+" "+endRow+" "+prevEndRow);
Date date = sdf.parse(day + " " + time);
// System.out.println(" "+date);

assignments.add(new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime()));
assignments.add(
new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime()));

}
} else if (!tablet.startsWith("!0")) {
@@ -262,17 +265,17 @@ public static void main(String[] args) throws Exception {
}

try (AccumuloClient client = opts.createClient();
BatchScanner bscanner = client.createBatchScanner(opts.tableName, opts.auths)) {
BatchScanner bscanner = client.createBatchScanner(opts.tableName, opts.auths)) {
List<Range> refs = new ArrayList<>();

for (UndefinedNode undefinedNode : undefs)
refs.add(new Range(new Text(undefinedNode.ref)));

bscanner.setRanges(refs);

HashMap<String, List<String>> refInfo = new HashMap<>();
HashMap<String,List<String>> refInfo = new HashMap<>();

for (Entry<Key, Value> entry : bscanner) {
for (Entry<Key,Value> entry : bscanner) {
String ref = entry.getKey().getRow().toString();
List<String> vals = refInfo.computeIfAbsent(ref, k -> new ArrayList<>());
vals.add(entry.getValue().toString());
@@ -313,9 +316,11 @@ public static void main(String[] args) throws Exception {
}

if (ta == null)
System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + uuid + " " + t1 + " " + t2);
System.out.println(
undefinedNode.undef + " " + undefinedNode.ref + " " + uuid + " " + t1 + " " + t2);
else
System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + ta.tablet + " " + ta.server + " " + uuid + " " + t1 + " " + t2);
System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + ta.tablet
+ " " + ta.server + " " + uuid + " " + t1 + " " + t2);

}
} else {
@@ -39,16 +39,16 @@ static class Opts extends ClientOpts {
String failures = null;
}

public static void main(String[] args) throws IOException, AccumuloException,
AccumuloSecurityException, TableNotFoundException {
public static void main(String[] args)
throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException {
final FileSystem fs = FileSystem.get(new Configuration());
Opts opts = new Opts();
System.err
.println("Deprecated syntax for BulkImportDirectory, please use the new style (see --help)");
System.err.println(
"Deprecated syntax for BulkImportDirectory, please use the new style (see --help)");
opts.parseArgs(BulkImportDirectory.class.getName(), args);
fs.delete(new Path(opts.failures), true);
fs.mkdirs(new Path(opts.failures));
opts.createClient().tableOperations()
.importDirectory(opts.tableName, opts.source, opts.failures, false);
opts.createClient().tableOperations().importDirectory(opts.tableName, opts.source,
opts.failures, false);
}
}
@@ -31,9 +31,9 @@
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.rfile.RFileWriter;
import org.apache.accumulo.core.client.security.SecurityErrorCode;
import org.apache.accumulo.core.clientImpl.TabletServerBatchWriter;
import org.apache.accumulo.core.data.ConstraintViolationSummary;
import org.apache.accumulo.core.data.Key;
@@ -76,8 +76,7 @@ public static class Opts extends ClientOpts {
@Parameter(names = "--cols", description = "the number of columns to ingest per row")
public int cols = 1;

@Parameter(
names = "--random",
@Parameter(names = "--random",
description = "insert random rows and use the given number to seed the psuedo-random number generator")
public Integer random = null;

@@ -108,8 +107,8 @@ public static class Opts extends ClientOpts {
public FileSystem fs = null;
}

public static void createTable(AccumuloClient client, Opts args) throws AccumuloException,
AccumuloSecurityException, TableExistsException {
public static void createTable(AccumuloClient client, Opts args)
throws AccumuloException, AccumuloSecurityException, TableExistsException {
if (args.createTable) {
TreeSet<Text> splits = getSplitPoints(args.startRow, args.startRow + args.rows,
args.numsplits);
@@ -256,8 +255,8 @@ public static void ingest(AccumuloClient client, FileSystem fs, Opts opts, Confi
} else {
byte value[];
if (opts.random != null) {
value = genRandomValue(random, randomValue, opts.random.intValue(), rowid
+ opts.startRow, j);
value = genRandomValue(random, randomValue, opts.random.intValue(),
rowid + opts.startRow, j);
} else {
value = bytevals[j % bytevals.length];
}
@@ -279,8 +278,8 @@ public static void ingest(AccumuloClient client, FileSystem fs, Opts opts, Confi
} else {
byte value[];
if (opts.random != null) {
value = genRandomValue(random, randomValue, opts.random.intValue(), rowid
+ opts.startRow, j);
value = genRandomValue(random, randomValue, opts.random.intValue(),
rowid + opts.startRow, j);
} else {
value = bytevals[j % bytevals.length];
}
@@ -305,9 +304,10 @@ public static void ingest(AccumuloClient client, FileSystem fs, Opts opts, Confi
bw.close();
} catch (MutationsRejectedException e) {
if (e.getSecurityErrorCodes().size() > 0) {
for (Entry<TabletId,Set<SecurityErrorCode>> entry : e.getSecurityErrorCodes().entrySet()) {
System.err.println("ERROR : Not authorized to write to : " + entry.getKey()
+ " due to " + entry.getValue());
for (Entry<TabletId,Set<SecurityErrorCode>> entry : e.getSecurityErrorCodes()
.entrySet()) {
System.err.println("ERROR : Not authorized to write to : " + entry.getKey() + " due to "
+ entry.getValue());
}
}

@@ -325,11 +325,10 @@ public static void ingest(AccumuloClient client, FileSystem fs, Opts opts, Confi
int totalValues = opts.rows * opts.cols;
double elapsed = (stopTime - startTime) / 1000.0;

System.out
.printf(
"%,12d records written | %,8d records/sec | %,12d bytes written | %,8d bytes/sec | %6.3f secs %n",
totalValues, (int) (totalValues / elapsed), bytesWritten,
(int) (bytesWritten / elapsed), elapsed);
System.out.printf(
"%,12d records written | %,8d records/sec | %,12d bytes written | %,8d bytes/sec | %6.3f secs %n",
totalValues, (int) (totalValues / elapsed), bytesWritten, (int) (bytesWritten / elapsed),
elapsed);
}

public static void ingest(AccumuloClient c, Opts opts, Configuration conf)

0 comments on commit 48d255d

Please sign in to comment.