Skip to content

Commit

Permalink
Merge branch 'master' into task/StopDoubleWrappingNodes
Browse files Browse the repository at this point in the history
  • Loading branch information
@apmoriarty
apmoriarty committed Jun 2, 2021
2 parents 5c3f6ee + 3f3de71 commit 7cd19f6
Show file tree
Hide file tree
Showing 25 changed files with 265 additions and 175 deletions.
53 changes: 28 additions & 25 deletions warehouse/core/src/main/java/datawave/edge/util/EdgeValue.java
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
package datawave.edge.util;

import java.util.List;
import java.util.UUID;

import com.google.protobuf.InvalidProtocolBufferException;
import datawave.edge.protobuf.EdgeData;
import datawave.edge.protobuf.EdgeData.EdgeValue.Builder;

import org.apache.accumulo.core.data.Value;

import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.commons.lang.StringUtils;

import java.util.List;
import java.util.UUID;

/**
* Utility class for serializing edge table protocol buffer. Previously, the EdgeValueHelper class was sufficient for handling the edge values, but with the
* addition of fields beyond the counts, a more generic solution is required.
Expand All @@ -24,8 +22,7 @@ public class EdgeValue {
private final List<Long> hours;
private final List<Long> duration;
private String loadDate;
// uuidObj allows us to avoid decoding the UUID into a String unless necessary
private EdgeData.EdgeValue.UUID uuidObj;
private UUID uuidObj;
// see EdgeData.proto. hasOnlyUuidString is true if uuid_string is/should be set in the protobuf message, meaning that uuid is not set because a UUID object
// couldn't be created from the original uuid
private boolean hasOnlyUuidString;
Expand All @@ -38,7 +35,7 @@ private EdgeValue(Long count, Integer bitmask, String sourceValue, String sinkVa
}

private EdgeValue(Long count, Integer bitmask, String sourceValue, String sinkValue, List<Long> hours, List<Long> duration, String loadDate,
String uuidString, boolean hasOnlyUuidString, EdgeData.EdgeValue.UUID uuidObj, Boolean badActivityDate) {
String uuidString, boolean hasOnlyUuidString, UUID uuidObj, Boolean badActivityDate) {

if (count == 0) {
this.count = null;
Expand Down Expand Up @@ -73,7 +70,7 @@ public static class EdgeValueBuilder {
private List<Long> hours = null;
private List<Long> duration = null;
private String loadDate = null;
private EdgeData.EdgeValue.UUID uuidObj = null;
private UUID uuidObj = null;
private boolean hasOnlyUuidString = false;
private String uuidString = null;
private Boolean badActivityDate = null;
Expand Down Expand Up @@ -172,7 +169,7 @@ public void setLoadDate(String loadDate) {
this.loadDate = loadDate;
}

public EdgeData.EdgeValue.UUID getUuidObj() {
public UUID getUuidObj() {
if (this.hasOnlyUuidString) {
return null;
}
Expand All @@ -187,7 +184,7 @@ public EdgeData.EdgeValue.UUID getUuidObj() {
return uuidObj;
}

public void setUuidObj(EdgeData.EdgeValue.UUID uuidObj) {
public void setUuidObj(UUID uuidObj) {
this.uuidObj = uuidObj;
}

Expand Down Expand Up @@ -266,7 +263,7 @@ public static EdgeValue decode(Value value) throws InvalidProtocolBufferExceptio
}
if (proto.hasUuid()) {
builder.setOnlyUuidString(false);
builder.setUuidObj(proto.getUuid());
builder.setUuidObj(convertUuidObject(proto.getUuid()));
} else if (proto.hasUuidString()) {
// if there is a uuid string in the protobuf data, it means that we shouldn't have a uuid object at all
builder.setOnlyUuidString(true);
Expand Down Expand Up @@ -309,11 +306,11 @@ public Value encode() {
builder.setUuidString(this.uuidString);
}
} else if (this.uuidObj != null) { // already have the uuid object, so there's no reason to reparse the string
builder.setUuid(this.uuidObj);
builder.setUuid(convertUuidObject(this.uuidObj));
} else if (StringUtils.isNotBlank(this.uuidString)) {
try { // try to parse the uuid string to a UUID object
this.uuidObj = convertUuidStringToUuidObj(this.uuidString);
builder.setUuid(this.uuidObj);
builder.setUuid(convertUuidObject(this.uuidObj));
} catch (Exception e) {
// if it failed to parse, settle for the uuid_string
this.hasOnlyUuidString = true;
Expand All @@ -328,12 +325,8 @@ public Value encode() {
return new Value(builder.build().toByteArray());
}

public static EdgeData.EdgeValue.UUID convertUuidStringToUuidObj(String uuidString) throws IllegalArgumentException {
UUID uuidObj = UUID.fromString(uuidString);
EdgeData.EdgeValue.UUID.Builder uuidBuilder = EdgeData.EdgeValue.UUID.newBuilder();
uuidBuilder.setLeastSignificantBits(uuidObj.getLeastSignificantBits());
uuidBuilder.setMostSignificantBits(uuidObj.getMostSignificantBits());
return uuidBuilder.build();
public static UUID convertUuidStringToUuidObj(String uuidString) {
return UUID.fromString(uuidString);
}

public Long getCount() {
Expand Down Expand Up @@ -388,7 +381,7 @@ public String getLoadDate() {
return loadDate;
}

public EdgeData.EdgeValue.UUID getUuidObject() {
public UUID getUuidObject() {
if (this.hasOnlyUuidString) {
return null;
}
Expand All @@ -411,9 +404,19 @@ public String getUuid() {
return this.uuidString;
}

private static String convertUuidObjectToString(EdgeData.EdgeValue.UUID rawUuid) {
UUID uuidObj = new UUID(rawUuid.getMostSignificantBits(), rawUuid.getLeastSignificantBits());
return uuidObj.toString();
public static String convertUuidObjectToString(UUID rawUuid) {
return rawUuid.toString();
}

public static UUID convertUuidObject(EdgeData.EdgeValue.UUID rawUuid) {
return new UUID(rawUuid.getLeastSignificantBits(), rawUuid.getMostSignificantBits());
}

public static EdgeData.EdgeValue.UUID convertUuidObject(UUID rawUuid) {
EdgeData.EdgeValue.UUID.Builder builder = EdgeData.EdgeValue.UUID.newBuilder();
builder.setLeastSignificantBits(rawUuid.getLeastSignificantBits());
builder.setMostSignificantBits(rawUuid.getMostSignificantBits());
return builder.build();
}

public Boolean isBadActivityDate() {
Expand Down
4 changes: 1 addition & 3 deletions warehouse/ingest-core/src/main/java/datawave/iterators/EdgeCombiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,9 +193,7 @@ private void useEarliestUuid(EdgeValueBuilder builder, EdgeData.EdgeValue protoE
// the value corresponding to the key with the most recent timestamp will come first
// the value corresponding to the key with the oldest timestamp will come last
if (protoEdgeValue.hasUuid()) {
// previously, we took uuid from proto.EdgeValue, converted it to UUID, then to String
// then when encoding we converted it back to a UUID and then into a UUID builder
builder.setUuidObj(protoEdgeValue.getUuid());
builder.setUuidObj(EdgeValue.convertUuidObject(protoEdgeValue.getUuid()));
builder.setOnlyUuidString(false);
} else if (protoEdgeValue.hasUuidString()) {
builder.setOnlyUuidString(true);
Expand Down
2 changes: 1 addition & 1 deletion .../query-core/src/main/java/datawave/query/jexl/visitors/ConjunctionEliminationVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ private boolean isDisjunction(JexlNode node) {
// Return whether or not the two JEXL queries are equivalent.
private boolean isEquivalent(JexlNode node, ASTJexlScript script) throws ParseException {
ASTJexlScript nodeScript = getScript(node);
return TreeEqualityVisitor.isEqual(nodeScript, script, new TreeEqualityVisitor.Reason());
return TreeEqualityVisitor.isEqual(nodeScript, script);
}

// Return the Jexl node as a script.
Expand Down
2 changes: 1 addition & 1 deletion .../query-core/src/main/java/datawave/query/jexl/visitors/DisjunctionEliminationVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ private boolean isDisjunction(JexlNode node) {
// Return whether or not the two JEXL queries are equivalent.
private boolean isEquivalent(JexlNode node, ASTJexlScript script) throws ParseException {
ASTJexlScript nodeScript = getScript(node);
return TreeEqualityVisitor.isEqual(nodeScript, script, new TreeEqualityVisitor.Reason());
return TreeEqualityVisitor.isEqual(nodeScript, script);
}

// Return the JEXL node as a script.
Expand Down
65 changes: 54 additions & 11 deletions warehouse/query-core/src/main/java/datawave/query/jexl/visitors/TreeEqualityVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,27 +62,70 @@
import datawave.webservice.common.logging.ThreadConfigurableLogger;

/**
* Determine whether two trees are equivalent, accounting for arbitrary order within subtrees
* Determine whether two trees are equivalent, accounting for arbitrary order within subtrees.
*/
public class TreeEqualityVisitor implements ParserVisitor {
private static final Logger log = ThreadConfigurableLogger.getLogger(TreeEqualityVisitor.class);

private boolean equal = true;

public TreeEqualityVisitor() {
this.equal = true;
public final static class Comparison {

private static final Comparison IS_EQUAL = new Comparison(true, null);

private static Comparison notEqual(String reason) {
return new Comparison(false, reason);
}

private final boolean equal;
private final String reason;

private Comparison(boolean equal, String reason) {
this.equal = equal;
this.reason = reason;
}

public boolean isEqual() {
return equal;
}

public String getReason() {
return reason;
}
}

public static class Reason {
public String reason = null;
/**
* Return whether or not the provided query trees are considered equivalent.
*
* @param first
* the first query tree
* @param second
* the second query tree
* @return true if the query trees are considered equivalent, or false otherwise
*/
public static boolean isEqual(JexlNode first, JexlNode second) {
return checkEquality(first, second).isEqual();
}

public static boolean isEqual(ASTJexlScript script1, ASTJexlScript script2, Reason reason) {
TreeEqualityVisitor visitor = new TreeEqualityVisitor();

reason.reason = (String) (script1.jjtAccept(visitor, script2));

return visitor.equal;
/**
* Compare the provided query trees for equivalency and return the resulting comparison.
*
* @param first
* the first query tree
* @param second
* the second query tree
* @return the comparison result
*/
public static Comparison checkEquality(JexlNode first, JexlNode second) {
if (first != null && second != null) {
TreeEqualityVisitor visitor = new TreeEqualityVisitor();
String reason = (String) first.jjtAccept(visitor, second);
return visitor.equal ? Comparison.IS_EQUAL : Comparison.notEqual(reason);
} else if (first == null && second == null) {
return Comparison.IS_EQUAL;
} else {
return Comparison.notEqual("One tree is null: " + first + " vs " + second);
}
}

/**
Expand Down
12 changes: 4 additions & 8 deletions warehouse/query-core/src/test/java/datawave/query/index/lookup/IndexInfoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,7 @@ public void testIntersection_NestedOrTermHasDocIdsInfiniteToSmall() {
assertEquals(1, merged.uids().size());
assertEquals(expectedSorted, merged.uids());

assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode()),
new TreeEqualityVisitor.Reason()));
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode())));
}

@Test
Expand Down Expand Up @@ -149,8 +148,7 @@ public void testIntersection_NestedOrTermHasDocIdsSmallToInfinite() {

assertEquals(1, merged.uids().size());
assertEquals(expectedSorted, merged.uids());
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode()),
new TreeEqualityVisitor.Reason()));
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode())));
}

@Test
Expand Down Expand Up @@ -184,8 +182,7 @@ public void testIntersection_NestedOrTermHasDocIdsInfiniteToSmallWithDelayed() {

assertEquals(1, merged.uids().size());
assertEquals(expectedSorted, merged.uids());
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode()),
new TreeEqualityVisitor.Reason()));
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode())));
}

@Test
Expand Down Expand Up @@ -219,8 +216,7 @@ public void testIntersection_NestedOrTermHasDocIdsSmallToInfiniteWithDelay() {

assertEquals(1, merged.uids().size());
assertEquals(expectedSorted, merged.uids());
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode()),
new TreeEqualityVisitor.Reason()));
assertTrue(TreeEqualityVisitor.isEqual(JexlNodeFactory.createScript(origQueryTree), JexlNodeFactory.createScript(merged.getNode())));
}

/**
Expand Down
Loading

0 comments on commit 7cd19f6

Please sign in to comment.