Permalink
Browse files

Changes to specify array ranges while projecting fields.

(0) Update PathSpec to include attributes (start and count).
(1) Update Java DataTemplate codegen to generate field PathSpec methods for specifying array ranges.
(2) Update MaskComposition to take care of merging array ranges.

RB=1148299
BUG=SI-4801
G=si-dev
R=kbalasub,xma,mnchen
A=kbalasub
  • Loading branch information...
saponniah committed Nov 6, 2017
1 parent 9a4197d commit 58bc55e4eb8e06d2f7d53fdffd20f8b6ca12bfb9
View
@@ -1,17 +1,17 @@
16.0.3
------
(RB=1148212)
Error out if zookeeper connection is not ready for markup/markdown
(RB=1139217)
Update URIMaskUtil is parse array ranges in URI fields projection and fix bugs in Filer and CopyFilter.
(RB=1148299)
Changes to specify array ranges while projecting fields.
(0) Update PathSpec to include attributes (start and count).
(1) Update Java DataTemplate codegen to generate field PathSpec methods for specifying array ranges.
(2) Update MaskComposition to take care of merging array ranges.
(RB=1139217)
Update URIMaskUtil is parse array ranges in URI fields projection and fix bugs in Filer and CopyFilter.
(RB=1148212)
Error out if zookeeper connection is not ready for markup/markdown
16.0.2
------
(RB=1048086)
@@ -0,0 +1,96 @@
package com.linkedin.data.transform.filter;
/**
* A helper POJO to hold the array range values (start and count).
*/
class ArrayRange
{
static final Integer DEFAULT_START = 0;
static final Integer DEFAULT_COUNT = Integer.MAX_VALUE;
private final Integer _start;
private final Integer _count;
/**
* Default constructor.
*/
ArrayRange(Integer start, Integer count)
{
_start = start;
_count = count;
}
/**
* Returns the start value. If the start value is not present, returns null.
*/
Integer getStart()
{
return _start;
}
/**
* Returns the start value. If the start value is not present, returns the default value.
*/
Integer getStartOrDefault()
{
return hasStart() ? _start : DEFAULT_START;
}
/**
* Returns true if the count value is present and false otherwise.
*/
boolean hasStart()
{
return _start != null;
}
/**
* Returns the count value. If the start value is not present, returns null.
*/
Integer getCount()
{
return _count;
}
/**
* Returns the count value. If the start value is not present, returns the default value.
*/
Integer getCountOrDefault()
{
return hasCount() ? _count : DEFAULT_COUNT;
}
/**
* Returns true if the count value is present and false otherwise.
*/
boolean hasCount()
{
return _count != null;
}
/**
* Returns the end index (excluded) for this array range. The returned value is labelled excluded as it is one more
* than the index of the last element included. If adding count to the start is more than the allowed maximum
* ({@link #DEFAULT_COUNT}), the allowed maximum value will be returned.
*/
Integer getEnd()
{
return getStartOrDefault() + Math.min(DEFAULT_COUNT - getStartOrDefault(), getCountOrDefault());
}
/**
* Returns true if either of start and count is present. If neither of them is present the method returns false.
*/
boolean hasAnyValue()
{
return hasStart() || hasCount();
}
@Override
public String toString()
{
StringBuilder sb = new StringBuilder();
sb.append("[start: ").append(_start).append("][").append("count: ").append(_count).append("]");
return sb.toString();
}
}
@@ -19,7 +19,6 @@
*/
package com.linkedin.data.transform.filter;
import static com.linkedin.data.transform.filter.FilterUtil.getIntegerWithDefaultValue;
import static com.linkedin.data.transform.filter.FilterUtil.isMarkedAsMergedWith1;
import java.util.ArrayList;
@@ -31,6 +30,7 @@
import com.linkedin.data.transform.Interpreter;
import com.linkedin.data.transform.InterpreterContext;
/**
* This interpreter performs masks composition. Both data and operation are treated as
* masks. After processing is finished, the data contains composition of masks and
@@ -70,26 +70,26 @@ public void interpret(InterpreterContext instrCtx)
}
else
{
//process array range
composeArrayRange(data, op, instrCtx);
//process array range
composeArrayRange(data, op, instrCtx);
// process all fields
for (Entry<String, Object> entry : op.entrySet())
{
String fieldName = entry.getKey();
Object opMask = entry.getValue();
Object dataMask = data.get(fieldName);
// process all fields
for (Entry<String, Object> entry : op.entrySet())
if (!fieldName.equals(FilterConstants.START) && !fieldName.equals(FilterConstants.COUNT))
{
String fieldName = entry.getKey();
Object opMask = entry.getValue();
Object dataMask = data.get(fieldName);
if (!fieldName.equals(FilterConstants.START) && !fieldName.equals(FilterConstants.COUNT))
{
composeField(fieldName,
opMask,
dataMask,
data,
dataWildcard,
instrCtx);
}
composeField(fieldName,
opMask,
dataMask,
data,
dataWildcard,
instrCtx);
}
}
}
}
}
@@ -105,41 +105,110 @@ public void interpret(InterpreterContext instrCtx)
* </ul>
* Values: 0 for start and {@link Integer#MAX_VALUE} for count are not stored explicitly.
*/
protected void composeArrayRange(final DataMap data, final DataMap op, InterpreterContext instrCtx)
private void composeArrayRange(DataMap first, DataMap second, InterpreterContext interpreterContext)
{
ArrayRange firstArrayRange = extractArrayRange(first, interpreterContext);
ArrayRange secondArrayRange = extractArrayRange(second, interpreterContext);
Integer mergedStart = mergeStart(firstArrayRange, secondArrayRange);
Integer mergedCount = mergeCount(firstArrayRange, secondArrayRange, mergedStart);
storeNonDefaultValue(first, FilterConstants.START, ArrayRange.DEFAULT_START, mergedStart);
storeNonDefaultValue(first, FilterConstants.COUNT, ArrayRange.DEFAULT_COUNT, mergedCount);
}
private ArrayRange extractArrayRange(DataMap data, InterpreterContext interpreterContext)
{
Integer start = extractRangeValue(data, FilterConstants.START, interpreterContext);
Integer count = extractRangeValue(data, FilterConstants.COUNT, interpreterContext);
return new ArrayRange(start, count);
}
/**
* Extract the value for the specified range key in the provided data. If the extracted value is a positive integer
* a non-null value is returned.
*/
private Integer extractRangeValue(DataMap data, String key, InterpreterContext instrCtx)
{
Integer value = null;
final Object o = data.get(key);
if (o != null)
{
if (o instanceof Integer)
{
Integer integerValue = (Integer) o;
if (integerValue >= 0)
{
value = integerValue;
}
else
{
addNegativeIntegerError(data, key, integerValue, instrCtx);
}
}
else
{
addValueTypeNotIntegerError(data, key, instrCtx);
}
}
return value;
}
/**
* Get the merged start value from the two provided instances of {@link ArrayRange}s. The merge algorithm works as
* described below,
* <ul>
* <li>If both the instances have either start or count specified, the minimum value of their start is returned.</li>
* <li>If one of them have either start or count specified, the corresponding start value is returned.</li>
* <li>If both the instances have neither start nor count specified, the default start value is returned.</li>
* </ul>
*/
private Integer mergeStart(ArrayRange firstArrayRange, ArrayRange secondArrayRange)
{
// otherwise ranges need to be merged
final Integer startData = getIntegerWithDefaultValue(data, FilterConstants.START, 0);
if (startData == null)
addValueTypeNotIntegerError(data, FilterConstants.START, instrCtx);
final Integer startOp = getIntegerWithDefaultValue(op, FilterConstants.START, 0);
if (startOp == null)
addValueTypeNotIntegerError(op, FilterConstants.START, instrCtx);
final Integer countData = getIntegerWithDefaultValue(data, FilterConstants.COUNT, Integer.MAX_VALUE);
if (countData == null)
addValueTypeNotIntegerError(data, FilterConstants.COUNT, instrCtx);
final Integer countOp = getIntegerWithDefaultValue(op, FilterConstants.COUNT, Integer.MAX_VALUE);
if (countOp == null)
addValueTypeNotIntegerError(op, FilterConstants.COUNT, instrCtx);
if (startData < 0)
addNegativeIntegerError(data, FilterConstants.START, startData, instrCtx);
if (startOp < 0)
addNegativeIntegerError(op, FilterConstants.START, startOp, instrCtx);
if (countData < 0)
addNegativeIntegerError(data, FilterConstants.COUNT, countData, instrCtx);
if (countOp < 0)
addNegativeIntegerError(op, FilterConstants.COUNT, countOp, instrCtx);
if (startData != null && startOp != null && countData != null && countOp != null &&
startData >= 0 && startOp >= 0 && countData >= 0 && countOp >= 0)
Integer mergedStart = ArrayRange.DEFAULT_START;
if (firstArrayRange.hasAnyValue() && secondArrayRange.hasAnyValue())
{
mergedStart = Math.min(firstArrayRange.getStartOrDefault(), secondArrayRange.getStartOrDefault());
}
else if (firstArrayRange.hasAnyValue())
{
mergedStart = firstArrayRange.getStartOrDefault();
}
else if (secondArrayRange.hasAnyValue())
{
final Integer start = Math.min(startData, startOp);
final Integer count = Math.max(startData + countData, startOp + countOp) - start;
storeNonDefaultValue(data, FilterConstants.START, 0, start);
storeNonDefaultValue(data, FilterConstants.COUNT, Integer.MAX_VALUE, count);
mergedStart = secondArrayRange.getStartOrDefault();
}
return mergedStart;
}
/**
* Get the merged count value from the two provided instances of {@link ArrayRange}s. The merge algorithm works as
* described below,
* <ul>
* <li>If both the instances have either start or count specified, the returned count will be such that it covers
* both the ranges relative to the merged start value.</li>
* <li>If one of them have either start or count specified, the returned count value will cover the range for the
* instance that has either one of the values specified.</li>
* <li>If both the instances have neither start nor count specified, the default count value is returned.</li>
* </ul>
*/
private Integer mergeCount(ArrayRange firstArrayRange, ArrayRange secondArrayRange, Integer mergedStart)
{
Integer mergedEnd = ArrayRange.DEFAULT_COUNT;
if (firstArrayRange.hasAnyValue() && secondArrayRange.hasAnyValue())
{
mergedEnd = Math.max(firstArrayRange.getEnd(), secondArrayRange.getEnd());
}
else if (firstArrayRange.hasAnyValue())
{
mergedEnd = firstArrayRange.getEnd();
}
else if (secondArrayRange.hasAnyValue())
{
mergedEnd = secondArrayRange.getEnd();
}
return (mergedEnd - mergedStart);
}
private void addNegativeIntegerError(DataMap data, String fieldName, Integer value, InterpreterContext instrCtx)
@@ -25,6 +25,7 @@
import com.linkedin.data.transform.DataComplexProcessor;
import com.linkedin.data.transform.DataProcessingException;
import com.linkedin.data.transform.Escaper;
import com.linkedin.data.transform.filter.FilterConstants;
import com.linkedin.data.transform.filter.MaskComposition;
import java.util.HashMap;
@@ -73,7 +74,7 @@ public MaskTree(DataMap rep)
public void addOperation(PathSpec path, MaskOperation op)
{
List<String> segments = path.getPathComponents();
Map<String, Object> attributes = path.getPathAttributes();
final DataMap fieldMask = new DataMap();
DataMap map = fieldMask; //map variable contains DataMap, into which current segment will be put
@@ -84,8 +85,30 @@ public void addOperation(PathSpec path, MaskOperation op)
map.put(segment, childMap);
map = childMap;
}
String lastSegment = Escaper.escapePathSegment(segments.get(segments.size()-1));
map.put(lastSegment, op.getRepresentation());
Object start = attributes.get(PathSpec.ATTR_ARRAY_START);
Object count = attributes.get(PathSpec.ATTR_ARRAY_COUNT);
if (start != null || count != null)
{
DataMap childMap = new DataMap();
map.put(lastSegment, childMap);
if (start != null)
{
childMap.put(FilterConstants.START, start);
}
if (count != null)
{
childMap.put(FilterConstants.COUNT, count);
}
}
else
{
map.put(lastSegment, op.getRepresentation());
}
//compose existing tree with mask for specific field
try
Oops, something went wrong.

0 comments on commit 58bc55e

Please sign in to comment.