Skip to content

Commit

Permalink
[ASTERIXDB-2507][RT] Objects Creation in ARecordSerializerDeserializer
Browse files Browse the repository at this point in the history 
- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Reuse UTF8 hash function and comparator from
  evaluators when extracting a field's offset.

Change-Id: Icdb47db9e909e49d65ebf1cd667e27e2467e4085
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3109
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
  • Loading branch information
@mhubail
mhubail committed Jan 8, 2019
1 parent 6b908d1 commit 5363b33
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 15 deletions.
17 changes: 5 additions & 12 deletions .../java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
View file
Expand Up @@ -305,8 +305,8 @@ public static int getFieldOffsetById(byte[] serRecord, int offset, int fieldId,
return offset + AInt32SerializerDeserializer.getInt(serRecord, pointer + nullBitmapSize + (4 * fieldId));
}

public static int getFieldOffsetByName(byte[] serRecord, int start, int len, byte[] fieldName, int nstart)
throws HyracksDataException {
public static int getFieldOffsetByName(byte[] serRecord, int start, int len, byte[] fieldName, int nstart,
IBinaryHashFunction nameHashFunction, IBinaryComparator nameComparator) throws HyracksDataException {
// 5 is the index of the byte that determines whether the record is expanded or not, i.e. it has an open part.
if (hasNoFields(serRecord, start, len) || serRecord[start + 5] != 1) {
return -1;
Expand All @@ -316,14 +316,7 @@ public static int getFieldOffsetByName(byte[] serRecord, int start, int len, byt
int numberOfOpenField = AInt32SerializerDeserializer.getInt(serRecord, openPartOffset);
int fieldUtflength = UTF8StringUtil.getUTFLength(fieldName, nstart + 1);
int fieldUtfMetaLen = UTF8StringUtil.getNumBytesToStoreLength(fieldUtflength);

IBinaryHashFunction utf8HashFunction =
BinaryHashFunctionFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryHashFunction();

IBinaryComparator utf8BinaryComparator =
BinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryComparator();

int fieldNameHashCode = utf8HashFunction.hash(fieldName, nstart + 1, fieldUtflength + fieldUtfMetaLen);
int fieldNameHashCode = nameHashFunction.hash(fieldName, nstart + 1, fieldUtflength + fieldUtfMetaLen);

int offset = openPartOffset + 4;
int fieldOffset = -1;
Expand All @@ -337,7 +330,7 @@ public static int getFieldOffsetByName(byte[] serRecord, int start, int len, byt
if (h == fieldNameHashCode) {
fieldOffset = start + AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * mid) + 4);
// the utf8 comparator do not require to put the precise length, we can just pass a estimated limit.
if (utf8BinaryComparator.compare(serRecord, fieldOffset, len, fieldName, nstart + 1,
if (nameComparator.compare(serRecord, fieldOffset, len, fieldName, nstart + 1,
fieldUtflength + fieldUtfMetaLen) == 0) {
// since they are equal, we can directly use the meta length and the utf length.
return fieldOffset + fieldUtfMetaLen + fieldUtflength;
Expand All @@ -346,7 +339,7 @@ public static int getFieldOffsetByName(byte[] serRecord, int start, int len, byt
h = AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * j));
if (h == fieldNameHashCode) {
fieldOffset = start + AInt32SerializerDeserializer.getInt(serRecord, offset + (8 * j) + 4);
if (utf8BinaryComparator.compare(serRecord, fieldOffset, len, fieldName, nstart + 1,
if (nameComparator.compare(serRecord, fieldOffset, len, fieldName, nstart + 1,
fieldUtflength) == 0) {
return fieldOffset + fieldUtfMetaLen + fieldUtflength;
}
Expand Down
10 changes: 9 additions & 1 deletion ...org/apache/asterix/runtime/evaluators/functions/records/FieldAccessByNameEvalFactory.java
View file
Expand Up @@ -22,13 +22,17 @@
import java.io.IOException;

import org.apache.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.BinaryHashFunctionFactoryProvider;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.runtime.exceptions.TypeMismatchException;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.data.std.api.IPointable;
Expand All @@ -55,6 +59,10 @@ public FieldAccessByNameEvalFactory(IScalarEvaluatorFactory recordEvalFactory,
public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
return new IScalarEvaluator() {

private final IBinaryHashFunction fieldNameHashFunction =
BinaryHashFunctionFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryHashFunction();
private final IBinaryComparator fieldNameComparator =
BinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryComparator();
private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
private DataOutput out = resultStorage.getDataOutput();

Expand Down Expand Up @@ -83,7 +91,7 @@ public void evaluate(IFrameTupleReference tuple, IPointable result) throws Hyrac
byte[] serFldName = inputArg1.getByteArray();
int serFldNameOffset = inputArg1.getStartOffset();
fieldValueOffset = ARecordSerializerDeserializer.getFieldOffsetByName(serRecord, serRecordOffset,
serRecordLen, serFldName, serFldNameOffset);
serRecordLen, serFldName, serFldNameOffset, fieldNameHashFunction, fieldNameComparator);
if (fieldValueOffset < 0) {
out.writeByte(ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
result.set(resultStorage);
Expand Down
11 changes: 10 additions & 1 deletion ...org/apache/asterix/runtime/evaluators/functions/records/FieldAccessNestedEvalFactory.java
View file
Expand Up @@ -23,6 +23,8 @@
import java.util.List;

import org.apache.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.BinaryHashFunctionFactoryProvider;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.om.base.AMissing;
import org.apache.asterix.om.base.ANull;
Expand All @@ -40,6 +42,8 @@
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.SourceLocation;
Expand Down Expand Up @@ -70,6 +74,10 @@ public FieldAccessNestedEvalFactory(IScalarEvaluatorFactory recordEvalFactory, A
public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
return new IScalarEvaluator() {

private final IBinaryHashFunction fieldNameHashFunction =
BinaryHashFunctionFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryHashFunction();
private final IBinaryComparator fieldNameComparator =
BinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryComparator();
private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
private final DataOutput out = resultStorage.getDataOutput();
private final ByteArrayAccessibleOutputStream subRecordTmpStream = new ByteArrayAccessibleOutputStream();
Expand Down Expand Up @@ -204,7 +212,8 @@ public void evaluate(IFrameTupleReference tuple, IPointable result) throws Hyrac
for (; pathIndex < fieldPointables.length; pathIndex++) {
openField = true;
subFieldOffset = ARecordSerializerDeserializer.getFieldOffsetByName(serRecord, start, len,
fieldPointables[pathIndex].getByteArray(), fieldPointables[pathIndex].getStartOffset());
fieldPointables[pathIndex].getByteArray(), fieldPointables[pathIndex].getStartOffset(),
fieldNameHashFunction, fieldNameComparator);
if (subFieldOffset < 0) {
out.writeByte(ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
result.set(resultStorage);
Expand Down
10 changes: 9 additions & 1 deletion ...g/apache/asterix/runtime/evaluators/functions/records/GetRecordFieldValueEvalFactory.java
View file
Expand Up @@ -22,6 +22,8 @@
import java.io.IOException;

import org.apache.asterix.dataflow.data.nontagged.serde.ARecordSerializerDeserializer;
import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.BinaryHashFunctionFactoryProvider;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
Expand All @@ -33,6 +35,8 @@
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.data.std.api.IPointable;
Expand Down Expand Up @@ -61,6 +65,10 @@ public GetRecordFieldValueEvalFactory(IScalarEvaluatorFactory recordEvalFactory,
public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
return new IScalarEvaluator() {

private final IBinaryHashFunction fieldNameHashFunction =
BinaryHashFunctionFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryHashFunction();
private final IBinaryComparator fieldNameComparator =
BinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryComparator();
private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
private final DataOutput out = resultStorage.getDataOutput();

Expand Down Expand Up @@ -120,7 +128,7 @@ public void evaluate(IFrameTupleReference tuple, IPointable result) throws Hyrac

// Look at open fields.
subFieldOffset = ARecordSerializerDeserializer.getFieldOffsetByName(serRecord, serRecordOffset,
serRecordLen, serFldName, serFldNameOffset);
serRecordLen, serFldName, serFldNameOffset, fieldNameHashFunction, fieldNameComparator);
if (subFieldOffset < 0) {
out.writeByte(ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
result.set(resultStorage);
Expand Down

0 comments on commit 5363b33

Please sign in to comment.