Skip to content

HIVE-22828 Support Decimal64 in NVL & CASE statements #910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

HIVE-22828 Support Decimal64 in NVL & CASE statements #910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions data/files/decimal64table1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
null,10,101,10
1000,10,101,10
1000,10,101,10
1000,10,101,10
1 change: 1 addition & 0 deletions itests/src/test/resources/testconfiguration.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -845,6 +845,7 @@ minillaplocal.query.files=\
vector_create_struct_table.q,\
vector_decimal_2.q,\
vector_decimal_udf.q,\
vector_decimal64_case_when_nvl.q,\
vector_full_outer_join.q,\
vector_fullouter_mapjoin_1_fast.q,\
vector_fullouter_mapjoin_1_optimized.q,\
Expand Down
57 changes: 44 additions & 13 deletions ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2473,14 +2473,39 @@ private VectorExpression getCoalesceExpression(List<ExprNodeDesc> childExpr,
final int size = vectorChildren.length;
TypeInfo[] inputTypeInfos = new TypeInfo[size];
DataTypePhysicalVariation[] inputDataTypePhysicalVariations = new DataTypePhysicalVariation[size];
int i = 0;
for (VectorExpression ve : vectorChildren) {
DataTypePhysicalVariation outputDataTypePhysicalVariation = DataTypePhysicalVariation.DECIMAL_64;
boolean fixConstants = false;
for (int i = 0; i < vectorChildren.length; ++i) {
VectorExpression ve = vectorChildren[i];
inputColumns[i] = ve.getOutputColumnNum();
inputTypeInfos[i] = ve.getOutputTypeInfo();
inputDataTypePhysicalVariations[i++] = ve.getOutputDataTypePhysicalVariation();
inputDataTypePhysicalVariations[i] = ve.getOutputDataTypePhysicalVariation();
if (inputDataTypePhysicalVariations[i] == DataTypePhysicalVariation.NONE) {
if (childExpr.get(i) instanceof ExprNodeConstantDesc && inputTypeInfos[i] instanceof DecimalTypeInfo &&
((DecimalTypeInfo)inputTypeInfos[i]).precision() <= 18) {
fixConstants = true;
} else {
outputDataTypePhysicalVariation = DataTypePhysicalVariation.NONE;
}
}
}

final int outputColumnNum = ocm.allocateOutputColumn(returnType);
if (outputDataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64 && fixConstants) {
for (int i = 0; i < vectorChildren.length; ++i) {
if (inputDataTypePhysicalVariations[i] == DataTypePhysicalVariation.NONE &&
vectorChildren[i] instanceof ConstantVectorExpression) {
ConstantVectorExpression cve = ((ConstantVectorExpression)vectorChildren[i]);
HiveDecimal hd = cve.getDecimalValue();
Long longValue = new HiveDecimalWritable(hd).serialize64(((DecimalTypeInfo)cve.getOutputTypeInfo()).getScale());
((ConstantVectorExpression)vectorChildren[i]).setLongValue(longValue);
vectorChildren[i].setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.DECIMAL_64);
int scratchColIndex = vectorChildren[i].getOutputColumnNum() - ocm.initialOutputCol;
ocm.scratchDataTypePhysicalVariations[scratchColIndex] = DataTypePhysicalVariation.DECIMAL_64;
}
}
}

final int outputColumnNum = ocm.allocateOutputColumn(returnType, outputDataTypePhysicalVariation);
VectorCoalesce vectorCoalesce = new VectorCoalesce(inputColumns, outputColumnNum);

vectorCoalesce.setChildExpressions(vectorChildren);
Expand All @@ -2489,7 +2514,7 @@ private VectorExpression getCoalesceExpression(List<ExprNodeDesc> childExpr,
vectorCoalesce.setInputDataTypePhysicalVariations(inputDataTypePhysicalVariations);

vectorCoalesce.setOutputTypeInfo(returnType);
vectorCoalesce.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE);
vectorCoalesce.setOutputDataTypePhysicalVariation(outputDataTypePhysicalVariation);

freeNonColumns(vectorChildren);

Expand Down Expand Up @@ -3877,8 +3902,9 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List<ExprN
getVectorExpression(ifDesc, VectorExpressionDescriptor.Mode.PROJECTION);
final VectorExpression elseExpr =
getVectorExpression(elseDesc, VectorExpressionDescriptor.Mode.PROJECTION);
DataTypePhysicalVariation outputDataTypePhysicalVariation = elseExpr.getOutputDataTypePhysicalVariation();

final int outputColumnNum = ocm.allocateOutputColumn(returnType);
final int outputColumnNum = ocm.allocateOutputColumn(returnType, outputDataTypePhysicalVariation);

final VectorExpression resultExpr;
if (!isElseCondExpr || isOnlyGood) {
Expand All @@ -3903,11 +3929,11 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List<ExprN
elseExpr.getOutputTypeInfo());
resultExpr.setInputDataTypePhysicalVariations(
whenExpr.getOutputDataTypePhysicalVariation(),
DataTypePhysicalVariation.NONE,
outputDataTypePhysicalVariation,
elseExpr.getOutputDataTypePhysicalVariation());

resultExpr.setOutputTypeInfo(returnType);
resultExpr.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE);
resultExpr.setOutputDataTypePhysicalVariation(outputDataTypePhysicalVariation);

return resultExpr;
}
Expand All @@ -3917,8 +3943,9 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List<ExprN
getVectorExpression(ifDesc, VectorExpressionDescriptor.Mode.PROJECTION);
final VectorExpression thenExpr =
getVectorExpression(thenDesc, VectorExpressionDescriptor.Mode.PROJECTION);
DataTypePhysicalVariation outputDataTypePhysicalVariation = thenExpr.getOutputDataTypePhysicalVariation();

final int outputColumnNum = ocm.allocateOutputColumn(returnType);
final int outputColumnNum = ocm.allocateOutputColumn(returnType, outputDataTypePhysicalVariation);

final VectorExpression resultExpr;
if (!isThenCondExpr || isOnlyGood) {
Expand All @@ -3944,10 +3971,10 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List<ExprN
resultExpr.setInputDataTypePhysicalVariations(
whenExpr.getOutputDataTypePhysicalVariation(),
thenExpr.getOutputDataTypePhysicalVariation(),
DataTypePhysicalVariation.NONE);
outputDataTypePhysicalVariation);

resultExpr.setOutputTypeInfo(returnType);
resultExpr.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE);
resultExpr.setOutputDataTypePhysicalVariation(outputDataTypePhysicalVariation);

return resultExpr;
}
Expand All @@ -3962,8 +3989,12 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List<ExprN

// Only proceed if the THEN/ELSE types were aligned.
if (thenExpr.getOutputColumnVectorType() == elseExpr.getOutputColumnVectorType()) {
DataTypePhysicalVariation outputDataTypePhysicalVariation =
(thenExpr.getOutputDataTypePhysicalVariation() == elseExpr.getOutputDataTypePhysicalVariation())
? thenExpr.getOutputDataTypePhysicalVariation()
: DataTypePhysicalVariation.NONE;

final int outputColumnNum = ocm.allocateOutputColumn(returnType);
final int outputColumnNum = ocm.allocateOutputColumn(returnType, outputDataTypePhysicalVariation);

final VectorExpression resultExpr;
if (isThenCondExpr && isElseCondExpr) {
Expand Down Expand Up @@ -4001,7 +4032,7 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List<ExprN
elseExpr.getOutputDataTypePhysicalVariation());

resultExpr.setOutputTypeInfo(returnType);
resultExpr.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE);
resultExpr.setOutputDataTypePhysicalVariation(outputDataTypePhysicalVariation);

return resultExpr;
}
Expand Down
18 changes: 17 additions & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,18 @@ private void evaluateDecimal(ColumnVector colVector) {
}
}

private void evaluateDecimal64(ColumnVector colVector) {
Decimal64ColumnVector dcv = (Decimal64ColumnVector) colVector;
dcv.isRepeating = true;
if (!isNullValue) {
dcv.isNull[0] = false;
dcv.vector[0] = longValue;
} else {
dcv.isNull[0] = true;
dcv.noNulls = false;
}
}

private void evaluateTimestamp(ColumnVector colVector) {
TimestampColumnVector tcv = (TimestampColumnVector) colVector;
tcv.isRepeating = true;
Expand Down Expand Up @@ -369,7 +381,11 @@ private void evaluateColumn(ColumnVector colVector) {
evaluateBytes(colVector);
break;
case DECIMAL:
evaluateDecimal(colVector);
if (outputDataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
evaluateDecimal64(colVector);
} else {
evaluateDecimal(colVector);
}
break;
case TIMESTAMP:
evaluateTimestamp(colVector);
Expand Down
12 changes: 12 additions & 0 deletions ql/src/test/queries/clientpositive/vector_decimal64_case_when_nvl.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
set hive.cbo.enable=false;
set hive.explain.user=false;
create external table vector_decimal64_case_when(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price double) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH '../../data/files/decimal64table1.csv' OVERWRITE INTO TABLE vector_decimal64_case_when;
create table vector_decimal64_case_when_tmp(ss_ext_list_price decimal(7,2), ss_ext_wholesale_cost decimal(19,1), ss_ext_discount_amt int, ss_ext_sales_price decimal(7,2)) stored as ORC;
insert into table vector_decimal64_case_when_tmp select * from vector_decimal64_case_when;
explain vectorization detail select sum(case when (ss_ext_discount_amt=101) then ss_ext_list_price else null end) from vector_decimal64_case_when_tmp;
select sum(case when (ss_ext_discount_amt=101) then ss_ext_list_price else null end) from vector_decimal64_case_when_tmp;
explain vectorization detail select sum(NVL(ss_ext_list_price, 1.1)) from vector_decimal64_case_when_tmp;
select sum(NVL(ss_ext_list_price, 1.1)) from vector_decimal64_case_when_tmp;
explain vectorization detail select sum(NVL(ss_ext_list_price, 1.1BD)) from vector_decimal64_case_when_tmp;
select sum(NVL(ss_ext_list_price, 1.1BD)) from vector_decimal64_case_when_tmp;
Loading