Skip to content

Commit

Permalink
Revert "HIVE-13084: Vectorization add support for PROJECTION Multi-AN…
Browse files Browse the repository at this point in the history 
…D/OR (Matt McCline, reviewed by Sergey Shelukhin)"

This reverts commit 0a24c88.

Conflicts:
	ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
  • Loading branch information
Matt McCline committed Jul 7, 2016
1 parent 2233508 commit 5c58dce
Show file tree
Hide file tree
Showing 13 changed files with 497 additions and 4,667 deletions.
12 changes: 6 additions & 6 deletions .../hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizedLogicBench.java
View file
Expand Up @@ -55,7 +55,7 @@ public static class ColAndColBench extends AbstractExpression {
public void setup() {
rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
getBooleanLongColumnVector());
expression = new ColAndCol(new int[] {0, 1}, 2);
expression = new ColAndCol(0, 1, 2);
}
}

Expand All @@ -64,7 +64,7 @@ public static class ColAndRepeatingColBench extends AbstractExpression {
public void setup() {
rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
getBooleanRepeatingLongColumnVector());
expression = new ColAndCol(new int[] {0, 1}, 2);
expression = new ColAndCol(0, 1, 2);
}
}

Expand All @@ -73,7 +73,7 @@ public static class RepeatingColAndColBench extends AbstractExpression {
public void setup() {
rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanRepeatingLongColumnVector(),
getBooleanLongColumnVector());
expression = new ColAndCol(new int[] {0, 1}, 2);
expression = new ColAndCol(0, 1, 2);
}
}

Expand All @@ -82,7 +82,7 @@ public static class ColOrColBench extends AbstractExpression {
public void setup() {
rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
getBooleanLongColumnVector());
expression = new ColOrCol(new int[] {0, 1}, 2);
expression = new ColOrCol(0, 1, 2);
}
}

Expand All @@ -91,7 +91,7 @@ public static class ColOrRepeatingColBench extends AbstractExpression {
public void setup() {
rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
getBooleanRepeatingLongColumnVector());
expression = new ColOrCol(new int[] {0, 1}, 2);
expression = new ColOrCol(0, 1, 2);
}
}

Expand All @@ -100,7 +100,7 @@ public static class RepeatingColOrColBench extends AbstractExpression {
public void setup() {
rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanRepeatingLongColumnVector(),
getBooleanLongColumnVector());
expression = new ColOrCol(new int[] {0, 1}, 2);
expression = new ColOrCol(0, 1, 2);
}
}

Expand Down
2 changes: 0 additions & 2 deletions itests/src/test/resources/testconfiguration.properties
View file
Expand Up @@ -315,9 +315,7 @@ minitez.query.files.shared=acid_globallimit.q,\
vector_leftsemi_mapjoin.q,\
vector_mapjoin_reduce.q,\
vector_mr_diff_schema_alias.q,\
vector_multi_and_projection.q,\
vector_multi_insert.q,\
vector_multi_or_projection.q,\
vector_non_string_partition.q,\
vector_nullsafe_join.q,\
vector_null_projection.q,\
Expand Down
125 changes: 43 additions & 82 deletions ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
View file
Expand Up @@ -123,7 +123,6 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
Expand Down Expand Up @@ -979,33 +978,36 @@ private VectorExpression getVectorExpressionForUdf(GenericUDF genericeUdf,

int numChildren = (childExpr == null) ? 0 : childExpr.size();

if (genericeUdf != null &&
if (numChildren > 2 && genericeUdf != null && mode == VectorExpressionDescriptor.Mode.FILTER &&
((genericeUdf instanceof GenericUDFOPOr) || (genericeUdf instanceof GenericUDFOPAnd))) {

// Special case handling for Multi-OR and Multi-AND FILTER and PROJECTION.
// Special case handling for Multi-OR and Multi-AND.

for (int i = 0; i < numChildren; i++) {
ExprNodeDesc child = childExpr.get(i);
String childTypeString = child.getTypeString();
if (childTypeString == null) {
throw new HiveException("Null child type name string");
}
TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(childTypeString);
Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
if (columnVectorType != ColumnVector.Type.LONG){
return null;
}
if (!(child instanceof ExprNodeGenericFuncDesc) && !(child instanceof ExprNodeColumnDesc)) {
return null;
}
}
Class<?> vclass;
if (genericeUdf instanceof GenericUDFOPOr) {
if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
vclass = ColOrCol.class;
} else {
vclass = FilterExprOrExpr.class;
}
vclass = FilterExprOrExpr.class;
} else if (genericeUdf instanceof GenericUDFOPAnd) {
if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
vclass = ColAndCol.class;
} else {
vclass = FilterExprAndExpr.class;
}
vclass = FilterExprAndExpr.class;
} else {
throw new RuntimeException("Unexpected multi-child UDF");
}
VectorExpressionDescriptor.Mode childrenMode = getChildrenMode(mode, udfClass);
if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
return createVectorMultiAndOrProjectionExpr(vclass, childExpr, childrenMode, returnType);
} else {
return createVectorExpression(vclass, childExpr, childrenMode, returnType);
}
return createVectorExpression(vclass, childExpr, childrenMode, returnType);
}
if (numChildren > VectorExpressionDescriptor.MAX_NUM_ARGUMENTS) {
return null;
Expand Down Expand Up @@ -1044,79 +1046,38 @@ private VectorExpression getVectorExpressionForUdf(GenericUDF genericeUdf,
return createVectorExpression(vclass, childExpr, childrenMode, returnType);
}

private void determineChildrenVectorExprAndArguments(Class<?> vectorClass,
List<ExprNodeDesc> childExpr, int numChildren, VectorExpressionDescriptor.Mode childrenMode,
VectorExpression.Type [] inputTypes, List<VectorExpression> children, Object[] arguments)
throws HiveException {
for (int i = 0; i < numChildren; i++) {
ExprNodeDesc child = childExpr.get(i);
String undecoratedName = getUndecoratedName(child.getTypeInfo().getTypeName());
inputTypes[i] = VectorExpression.Type.getValue(undecoratedName);
if (inputTypes[i] == VectorExpression.Type.OTHER){
throw new HiveException("No vector type for " + vectorClass.getSimpleName() + " argument #" + i + " type name " + undecoratedName);
}
if (child instanceof ExprNodeGenericFuncDesc) {
VectorExpression vChild = getVectorExpression(child, childrenMode);
children.add(vChild);
arguments[i] = vChild.getOutputColumn();
} else if (child instanceof ExprNodeColumnDesc) {
int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child);
if (childrenMode == VectorExpressionDescriptor.Mode.FILTER) {
// In filter mode, the column must be a boolean
children.add(new SelectColumnIsTrue(colIndex));
}
arguments[i] = colIndex;
} else if (child instanceof ExprNodeConstantDesc) {
Object scalarValue = getVectorTypeScalarValue((ExprNodeConstantDesc) child);
arguments[i] = (null == scalarValue) ? getConstantVectorExpression(null, child.getTypeInfo(), childrenMode) : scalarValue;
} else {
throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName());
}
}
}

private VectorExpression createVectorExpression(Class<?> vectorClass,
List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode childrenMode, TypeInfo returnType) throws HiveException {
int numChildren = childExpr == null ? 0: childExpr.size();
VectorExpression.Type [] inputTypes = new VectorExpression.Type[numChildren];
List<VectorExpression> children = new ArrayList<VectorExpression>();
Object[] arguments = new Object[numChildren];
try {
determineChildrenVectorExprAndArguments(vectorClass, childExpr, numChildren, childrenMode,
inputTypes, children, arguments);
VectorExpression vectorExpression = instantiateExpression(vectorClass, returnType, arguments);
vectorExpression.setInputTypes(inputTypes);
if ((vectorExpression != null) && !children.isEmpty()) {
vectorExpression.setChildExpressions(children.toArray(new VectorExpression[0]));
}
return vectorExpression;
} catch (Exception ex) {
throw new HiveException(ex);
} finally {
for (VectorExpression ve : children) {
ocm.freeOutputColumn(ve.getOutputColumn());
}
}
}

private VectorExpression createVectorMultiAndOrProjectionExpr(Class<?> vectorClass,
List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode childrenMode, TypeInfo returnType) throws HiveException {
int numChildren = childExpr == null ? 0: childExpr.size();
VectorExpression.Type [] inputTypes = new VectorExpression.Type[numChildren];
List<VectorExpression> children = new ArrayList<VectorExpression>();
Object[] arguments = new Object[numChildren];
try {
determineChildrenVectorExprAndArguments(vectorClass, childExpr, numChildren, childrenMode,
inputTypes, children, arguments);

// For Multi-AND/OR, transform the arguments -- column indices into an array of int.
int[] colNums = new int[numChildren];
for (int i = 0; i < numChildren; i++) {
colNums[i] = (Integer) arguments[i];
ExprNodeDesc child = childExpr.get(i);
String undecoratedName = getUndecoratedName(child.getTypeInfo().getTypeName());
inputTypes[i] = VectorExpression.Type.getValue(undecoratedName);
if (inputTypes[i] == VectorExpression.Type.OTHER){
throw new HiveException("No vector type for " + vectorClass.getSimpleName() + " argument #" + i + " type name " + undecoratedName);
}
if (child instanceof ExprNodeGenericFuncDesc) {
VectorExpression vChild = getVectorExpression(child, childrenMode);
children.add(vChild);
arguments[i] = vChild.getOutputColumn();
} else if (child instanceof ExprNodeColumnDesc) {
int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child);
if (childrenMode == VectorExpressionDescriptor.Mode.FILTER) {
// In filter mode, the column must be a boolean
children.add(new SelectColumnIsTrue(colIndex));
}
arguments[i] = colIndex;
} else if (child instanceof ExprNodeConstantDesc) {
Object scalarValue = getVectorTypeScalarValue((ExprNodeConstantDesc) child);
arguments[i] = (null == scalarValue) ? getConstantVectorExpression(null, child.getTypeInfo(), childrenMode) : scalarValue;
} else {
throw new HiveException("Cannot handle expression type: " + child.getClass().getSimpleName());
}
}
arguments = new Object[1];
arguments[0] = colNums;

VectorExpression vectorExpression = instantiateExpression(vectorClass, returnType, arguments);
vectorExpression.setInputTypes(inputTypes);
if ((vectorExpression != null) && !children.isEmpty()) {
Expand Down

0 comments on commit 5c58dce

Please sign in to comment.