Skip to content
Permalink
Browse files
BulkIngest CI improvements (#101)
* Add job name
* Seed Random with strong seed
* Fix linked list to chain adjacent elements, rather than skip over
  adjacent elements to link to the next element
  • Loading branch information
ctubbsii committed Jul 23, 2019
1 parent 63af461 commit 99a9a8244b1dbd0100cb5913d32460cedcc0961b
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 5 deletions.
@@ -49,14 +49,15 @@ public class BulkIngest extends Configured implements Tool {

@Override
public int run(String[] args) throws Exception {
String ingestInstanceId = UUID.randomUUID().toString();

Job job = Job.getInstance(getConf());
job.setJobName("BulkIngest_" + ingestInstanceId);
job.setJarByClass(BulkIngest.class);
// very important to prevent guava conflicts
job.getConfiguration().set("mapreduce.job.classloader", "true");
FileSystem fs = FileSystem.get(job.getConfiguration());

String ingestInstanceId = UUID.randomUUID().toString();

log.info(String.format("UUID %d %s", System.currentTimeMillis(), ingestInstanceId));

job.setInputFormatClass(ContinuousInputFormat.class);
@@ -25,6 +25,8 @@
import java.io.DataOutput;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -128,7 +130,12 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext job) {
maxQual = job.getConfiguration().getInt(PROP_QUAL_MAX, Short.MAX_VALUE);
checksum = job.getConfiguration().getBoolean(PROP_CHECKSUM, false);

random = new Random();
try {
random = new Random(SecureRandom.getInstanceStrong().nextLong());
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(
"Seeding random from a strong secure random algorithm should never fail", e);
}
nodeCount = 0;
}

@@ -158,9 +165,8 @@ public boolean nextKeyValue() {

if (nodeCount < numNodes) {
CRC32 cksum = checksum ? new CRC32() : null;
byte[] prevRow = prevKey != null ? prevKey.getRowData().toArray() : null;

prevKey = currKey;
byte[] prevRow = prevKey != null ? prevKey.getRowData().toArray() : null;
currKey = genKey(cksum);
currValue = new Value(createValue(uuid, prevRow, cksum));

0 comments on commit 99a9a82

Please sign in to comment.