Skip to content

Commit

Permalink
ARROW-7993: [Java] Support decimal type in ComplexCopier
Browse files Browse the repository at this point in the history 
Closes #6523 from projjal/decimalcomplexcopier and squashes the following commits:

105e94e <Projjal Chanda> added missing methods
0f0bb26 <Projjal Chanda> added test
b523e95 <Projjal Chanda> added fix
45a2c07 <Projjal Chanda> Added more tests
574f13c <Projjal Chanda> added (some) review comments
dc849c1 <Projjal Chanda> Support DecimalType in ComplexCopier

Authored-by: Projjal Chanda <iam@pchanda.com>
Signed-off-by: Praveen <praveen@dremio.com>
  • Loading branch information
@projjal @praveenbingo
projjal authored and praveenbingo committed Mar 6, 2020
1 parent af17e44 commit 88e3267
Show file tree
Hide file tree
Showing 17 changed files with 402 additions and 84 deletions.
8 changes: 8 additions & 0 deletions java/vector/src/main/codegen/templates/AbstractFieldWriter.java
View file
Expand Up @@ -81,9 +81,17 @@ public void write(${name}Holder holder) {
fail("${name}");
}

public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
fail("${name}");
}

public void writeBigEndianBytesToDecimal(byte[] value) {
fail("${name}");
}

public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
fail("${name}");
}
</#if>

</#list></#list>
Expand Down
20 changes: 19 additions & 1 deletion java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
View file
Expand Up @@ -75,6 +75,7 @@ public void endList() {

<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#if minor.class != "Decimal">
@Override
public void write(${name}Holder holder) {
getWriter(MinorType.${name?upper_case}).write(holder);
Expand All @@ -84,7 +85,24 @@ public void write(${name}Holder holder) {
getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
}

<#if minor.class == "Decimal">
<#else>
@Override
public void write(DecimalHolder holder) {
getWriter(MinorType.DECIMAL).write(holder);
}

public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
getWriter(MinorType.DECIMAL).writeDecimal(start, buffer, arrowType);
}

public void writeDecimal(int start, ArrowBuf buffer) {
getWriter(MinorType.DECIMAL).writeDecimal(start, buffer);
}

public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) {
getWriter(MinorType.DECIMAL).writeBigEndianBytesToDecimal(value, arrowType);
}

public void writeBigEndianBytesToDecimal(byte[] value) {
getWriter(MinorType.DECIMAL).writeBigEndianBytesToDecimal(value);
}
Expand Down
17 changes: 13 additions & 4 deletions java/vector/src/main/codegen/templates/ComplexCopier.java
View file
Expand Up @@ -96,14 +96,14 @@ private static void writeValue(FieldReader reader, FieldWriter writer) {
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>

<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class?starts_with("Decimal") >

case ${name?upper_case}:
if (reader.isSet()) {
Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
reader.read(${uncappedName}Holder);
if (${uncappedName}Holder.isSet == 1) {
writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list>);
writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, new ArrowType.Decimal(decimalHolder.precision, decimalHolder.scale)</#if>);
}
}
break;
Expand All @@ -118,10 +118,19 @@ private static FieldWriter getStructWriterForReader(FieldReader reader, StructWr
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams??>
case ${name?upper_case}:
return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name);
</#if>
<#if minor.class == "Decimal">
case ${name?upper_case}:
if (reader.getField().getType() instanceof ArrowType.Decimal) {
ArrowType.Decimal type = (ArrowType.Decimal) reader.getField().getType();
return (FieldWriter) writer.${uncappedName}(name, type.getScale(), type.getPrecision());
} else {
return (FieldWriter) writer.${uncappedName}(name);
}
</#if>
</#list></#list>
case STRUCT:
return (FieldWriter) writer.struct(name);
Expand All @@ -139,7 +148,7 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
case ${name?upper_case}:
return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>();
</#if>
Expand Down
55 changes: 51 additions & 4 deletions java/vector/src/main/codegen/templates/ComplexWriters.java
View file
Expand Up @@ -99,6 +99,7 @@ public void setPosition(int idx) {

<#else>

<#if minor.class != "Decimal">
public void write(${minor.class}Holder h) {
vector.setSafe(idx(), h);
vector.setValueCount(idx()+1);
Expand All @@ -113,19 +114,59 @@ public void write(Nullable${minor.class}Holder h) {
vector.setSafe(idx(), 1<#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.setValueCount(idx()+1);
}
</#if>

<#if minor.class == "Decimal" ||
minor.class == "VarChar">
<#if minor.class == "VarChar">
public void write${minor.class}(${friendlyType} value) {
vector.setSafe(idx(), value);
vector.setValueCount(idx()+1);
}
</#if>

<#if minor.class == "Decimal">
public void writeBigEndianBytesToDecimal(byte[] value) {

public void write(DecimalHolder h){
DecimalUtility.checkPrecisionAndScale(h.precision, h.scale, vector.getPrecision(), vector.getScale());
vector.setSafe(idx(), h);
vector.setValueCount(idx() + 1);
}

public void write(NullableDecimalHolder h){
if (h.isSet == 1) {
DecimalUtility.checkPrecisionAndScale(h.precision, h.scale, vector.getPrecision(), vector.getScale());
}
vector.setSafe(idx(), h);
vector.setValueCount(idx() + 1);
}

public void writeDecimal(int start, ArrowBuf buffer){
vector.setSafe(idx(), 1, start, buffer);
vector.setValueCount(idx() + 1);
}

public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType){
DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(),
((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale());
vector.setSafe(idx(), 1, start, buffer);
vector.setValueCount(idx() + 1);
}

public void writeDecimal(BigDecimal value){
// vector.setSafe already does precision and scale checking
vector.setSafe(idx(), value);
vector.setValueCount(idx() + 1);
}

public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType){
DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(),
((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale());
vector.setBigEndianSafe(idx(), value);
vector.setValueCount(idx()+1);
vector.setValueCount(idx() + 1);
}

public void writeBigEndianBytesToDecimal(byte[] value){
vector.setBigEndianSafe(idx(), value);
vector.setValueCount(idx() + 1);
}
</#if>

Expand All @@ -149,11 +190,17 @@ public void writeNull() {
public interface ${eName}Writer extends BaseWriter {
public void write(${minor.class}Holder h);

<#if minor.class == "Decimal">@Deprecated</#if>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
<#if minor.class == "Decimal">

public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, ArrowType arrowType);

public void write${minor.class}(${friendlyType} value);

public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType);

@Deprecated
public void writeBigEndianBytesToDecimal(byte[] value);
</#if>
}
Expand Down
4 changes: 2 additions & 2 deletions java/vector/src/main/codegen/templates/DenseUnionReader.java
View file
Expand Up @@ -92,7 +92,7 @@ private FieldReader getReaderForIndex(int index) {
<#list type.minor as minor>
<#assign name = minor.class?cap_first />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">
case ${name?upper_case}:
reader = (FieldReader) get${name}(typeId);
break;
Expand Down Expand Up @@ -165,7 +165,7 @@ public int size() {
<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">

private ${name}ReaderImpl get${name}(byte typeId) {
${name}ReaderImpl reader = (${name}ReaderImpl) readers[typeId];
Expand Down
16 changes: 10 additions & 6 deletions java/vector/src/main/codegen/templates/DenseUnionVector.java
View file
Expand Up @@ -239,6 +239,10 @@ private <T extends FieldVector> T addOrGet(byte typeId, MinorType minorType, Cla
return internalStruct.addOrGet(fieldName(typeId, minorType), fieldType(minorType), c);
}

private <T extends FieldVector> T addOrGet(byte typeId, MinorType minorType, ArrowType arrowType, Class<T> c) {
return internalStruct.addOrGet(fieldName(typeId, minorType), FieldType.nullable(arrowType), c);
}

@Override
public long getOffsetBufferAddress() {
return offsetBuffer.memoryAddress();
Expand Down Expand Up @@ -285,13 +289,13 @@ public StructVector getStruct(byte typeId) {
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#assign lowerCaseName = name?lower_case/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">
public ${name}Vector get${name}Vector(byte typeId) {
public ${name}Vector get${name}Vector(byte typeId<#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
ValueVector vector = childVectors[typeId];
if (vector == null) {
int vectorCount = internalStruct.size();
vector = addOrGet(typeId, MinorType.${name?upper_case}, ${name}Vector.class);
vector = addOrGet(typeId, MinorType.${name?upper_case}<#if minor.class == "Decimal">, arrowType</#if>, ${name}Vector.class);
childVectors[typeId] = vector;
if (internalStruct.size() > vectorCount) {
vector.allocateNew();
Expand Down Expand Up @@ -820,7 +824,7 @@ public void setSafe(int index, DenseUnionHolder holder) {
<#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">
case ${name?upper_case}:
Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
reader.read(${uncappedName}Holder);
Expand All @@ -844,7 +848,7 @@ public void setSafe(int index, DenseUnionHolder holder) {
<#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">
public void setSafe(int index, Nullable${name}Holder holder) {
while (index >= getOffsetBufferValueCapacity()) {
reallocOffsetBuffer();
Expand All @@ -854,7 +858,7 @@ public void setSafe(int index, Nullable${name}Holder holder) {
}
BitVectorHelper.setBit(validityBuffer, index);
byte typeId = getTypeId(index);
${name}Vector vector = get${name}Vector(typeId);
${name}Vector vector = get${name}Vector(typeId<#if minor.class == "Decimal">, new ArrowType.Decimal(holder.precision, holder.scale)</#if>);
int offset = vector.getValueCount();
vector.setValueCount(offset + 1);
vector.setSafe(offset, holder);
Expand Down
18 changes: 13 additions & 5 deletions java/vector/src/main/codegen/templates/DenseUnionWriter.java
View file
Expand Up @@ -123,7 +123,7 @@ BaseWriter getWriter(byte typeId) {
<#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams??>
<#if !minor.typeParams?? || minor.class == "Decimal">
case ${name?upper_case}:
return get${name}Writer(typeId);
</#if>
Expand All @@ -138,7 +138,7 @@ BaseWriter getWriter(byte typeId) {
<#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">

private ${name}Writer get${name}Writer(byte typeId) {
${name}Writer writer = (${name}Writer) writers[typeId];
Expand All @@ -159,10 +159,10 @@ public void write(${name}Holder holder) {
throw new UnsupportedOperationException();
}

public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, byte typeId) {
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, byte typeId<#if minor.class == "Decimal">, ArrowType arrowType</#if>) {
data.setTypeId(idx(), typeId);
get${name}Writer(typeId).setPosition(data.getOffset(idx()));
get${name}Writer(typeId).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
get${name}Writer(typeId).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>);
}
</#if>
</#list>
Expand Down Expand Up @@ -208,7 +208,7 @@ public StructWriter struct(String name) {
<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
<#assign upperName = minor.class?upper_case />
<#assign capName = minor.class?cap_first />
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal" >
@Override
public ${capName}Writer ${lowerName}(String name) {
byte typeId = data.getTypeId(idx());
Expand All @@ -225,6 +225,14 @@ public StructWriter struct(String name) {
return getListWriter(typeId).${lowerName}();
}
</#if>
<#if minor.class == "Decimal">
public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) {
byte typeId = data.getTypeId(idx());
data.setTypeId(idx(), typeId);
getStructWriter(typeId).setPosition(data.getOffset(idx()));
return getStructWriter(typeId).${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}</#list>);
}
</#if>
</#list></#list>

@Override
Expand Down
2 changes: 1 addition & 1 deletion java/vector/src/main/codegen/templates/StructWriters.java
View file
Expand Up @@ -265,7 +265,7 @@ public void end() {
} else {
if (writer instanceof PromotableWriter) {
// ensure writers are initialized
((PromotableWriter)writer).getWriter(MinorType.${upperName});
((PromotableWriter)writer).getWriter(MinorType.${upperName}<#if minor.class == "Decimal">, new ${minor.arrowType}(precision, scale)</#if>);
}
}
return writer;
Expand Down
4 changes: 2 additions & 2 deletions java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
View file
Expand Up @@ -181,11 +181,11 @@ public void write(DecimalHolder holder) {
writer.setPosition(writer.idx() + 1);
}

public void writeDecimal(int start, ArrowBuf buffer) {
public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
if (writer.idx() >= (idx() + 1) * listSize) {
throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
}
writer.writeDecimal(start, buffer);
writer.writeDecimal(start, buffer, arrowType);
writer.setPosition(writer.idx() + 1);
}

Expand Down
5 changes: 5 additions & 0 deletions java/vector/src/main/codegen/templates/UnionListWriter.java
View file
Expand Up @@ -178,6 +178,11 @@ public void write(DecimalHolder holder) {
writer.setPosition(writer.idx()+1);
}

public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
writer.writeDecimal(start, buffer, arrowType);
writer.setPosition(writer.idx()+1);
}

public void writeDecimal(int start, ArrowBuf buffer) {
writer.writeDecimal(start, buffer);
writer.setPosition(writer.idx()+1);
Expand Down
4 changes: 2 additions & 2 deletions java/vector/src/main/codegen/templates/UnionReader.java
View file
Expand Up @@ -88,7 +88,7 @@ private FieldReader getReaderForIndex(int index) {
<#list type.minor as minor>
<#assign name = minor.class?cap_first />
<#assign uncappedName = name?uncap_first/>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal">
case ${name?upper_case}:
return (FieldReader) get${name}();
</#if>
Expand Down Expand Up @@ -157,7 +157,7 @@ public int size() {
<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
<#if !minor.typeParams?? >
<#if !minor.typeParams?? || minor.class == "Decimal" >

private ${name}ReaderImpl ${uncappedName}Reader;

Expand Down

0 comments on commit 88e3267

Please sign in to comment.