Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HIVE-24886: Support simple equality operations between MAP/LIST/STRUCT types #2107

Merged
merged 2 commits into from Mar 30, 2021
Merged

HIVE-24886: Support simple equality operations between MAP/LIST/STRUCT types #2107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d51e618
HIVE-24886: Support simple equality operations between MAP/LIST/STRUC…
zabetak Mar 11, 2021
4854025
Update error message for udf_in.q
zabetak Mar 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 11 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
View file
Open in desktop
Expand Up @@ -1151,9 +1151,17 @@ private TypeInfo getTypeInfo(ASTNode funcNameNode) throws SemanticException {
private T interpretNode(T columnDesc, T valueDesc)
throws SemanticException {
if (exprFactory.isColumnRefExpr(columnDesc)) {
final PrimitiveTypeInfo typeInfo = TypeInfoFactory.getPrimitiveTypeInfo(
exprFactory.getTypeInfo(columnDesc).getTypeName().toLowerCase());
return interpretNodeAsConstant(typeInfo, valueDesc);
final TypeInfo info = exprFactory.getTypeInfo(columnDesc);
switch (info.getCategory()) {
case MAP:
case LIST:
case UNION:
case STRUCT:
return valueDesc;
case PRIMITIVE:
PrimitiveTypeInfo primitiveInfo = TypeInfoFactory.getPrimitiveTypeInfo(info.getTypeName().toLowerCase());
return interpretNodeAsConstant(primitiveInfo, valueDesc);
}
}
boolean columnStruct = exprFactory.isSTRUCTFuncCallExpr(columnDesc);
if (columnStruct) {
Expand Down
131 changes: 73 additions & 58 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java
View file
Open in desktop
Expand Up @@ -22,7 +22,6 @@
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ReturnObjectInspectorResolver;
Expand Down Expand Up @@ -82,84 +81,100 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen

argumentOIs = arguments;

for (int i = 0; i < arguments.length; i++) {
Category category = arguments[i].getCategory();
if (category != Category.PRIMITIVE) {
throw new UDFArgumentTypeException(i, "The "
+ GenericUDFUtils.getOrdinal(i + 1)
+ " argument of " + opName + " is expected to a "
+ Category.PRIMITIVE.toString().toLowerCase() + " type, but "
+ category.toString().toLowerCase() + " is found");
}
Category c1 = arguments[0].getCategory();
Category c2 = arguments[1].getCategory();
if (c1 != c2) {
throw new UDFArgumentException("Type mismatch in " + opName + "(" + c1 + "," + c2 + ")");
}
if (!supportsCategory(c1)) {
throw new UDFArgumentException(opName + " does not support " + c1 + " types");
}

switch (c1) {
case PRIMITIVE:
initForPrimitives(arguments[0], arguments[1]);
break;
case MAP:
case STRUCT:
case LIST:
initForNonPrimitives(arguments[0], arguments[1]);
break;
default:
throw new AssertionError("Missing init method for " + c1 + " types");
}
return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;

}

if (TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals(
TypeInfoFactory.stringTypeInfo) &&
TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals(
TypeInfoFactory.stringTypeInfo)) {
soi0 = (StringObjectInspector) arguments[0];
soi1 = (StringObjectInspector) arguments[1];
/**
* Returns whether the operator can handle operands with the specified category.
*/
protected boolean supportsCategory(Category c){
return c == Category.PRIMITIVE;
}

private void initForPrimitives(ObjectInspector arg0, ObjectInspector arg1) throws UDFArgumentException {
assert arg0.getCategory() == Category.PRIMITIVE;
assert arg1.getCategory() == Category.PRIMITIVE;
final TypeInfo type0 = TypeInfoUtils.getTypeInfoFromObjectInspector(arg0);
final TypeInfo type1 = TypeInfoUtils.getTypeInfoFromObjectInspector(arg1);
if (type0.equals(TypeInfoFactory.stringTypeInfo) && type1.equals(TypeInfoFactory.stringTypeInfo)) {
soi0 = (StringObjectInspector) arg0;
soi1 = (StringObjectInspector) arg1;
if (soi0.preferWritable() || soi1.preferWritable()) {
compareType = CompareType.COMPARE_TEXT;
} else {
compareType = CompareType.COMPARE_STRING;
}
} else if (TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals(
TypeInfoFactory.intTypeInfo) &&
TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals(
TypeInfoFactory.intTypeInfo)) {
} else if (type0.equals(TypeInfoFactory.intTypeInfo) && type1.equals(TypeInfoFactory.intTypeInfo)) {
compareType = CompareType.COMPARE_INT;
ioi0 = (IntObjectInspector) arguments[0];
ioi1 = (IntObjectInspector) arguments[1];
} else if (TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals(
TypeInfoFactory.longTypeInfo) &&
TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals(
TypeInfoFactory.longTypeInfo)) {
compareType = CompareType.COMPARE_LONG;
loi0 = (LongObjectInspector) arguments[0];
loi1 = (LongObjectInspector) arguments[1];
} else if (TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals(
TypeInfoFactory.byteTypeInfo) &&
TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals(
TypeInfoFactory.byteTypeInfo)) {
compareType = CompareType.COMPARE_BYTE;
byoi0 = (ByteObjectInspector) arguments[0];
byoi1 = (ByteObjectInspector) arguments[1];
} else if (TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals(
TypeInfoFactory.booleanTypeInfo) &&
TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals(
TypeInfoFactory.booleanTypeInfo)) {
ioi0 = (IntObjectInspector) arg0;
ioi1 = (IntObjectInspector) arg1;
} else if (type0.equals(TypeInfoFactory.longTypeInfo) && type1.equals(TypeInfoFactory.longTypeInfo)) {
compareType = CompareType.COMPARE_LONG;
loi0 = (LongObjectInspector) arg0;
loi1 = (LongObjectInspector) arg1;
} else if (type0.equals(TypeInfoFactory.byteTypeInfo) && type1.equals(TypeInfoFactory.byteTypeInfo)) {
compareType = CompareType.COMPARE_BYTE;
byoi0 = (ByteObjectInspector) arg0;
byoi1 = (ByteObjectInspector) arg1;
} else if (type0.equals(TypeInfoFactory.booleanTypeInfo) && type1.equals(TypeInfoFactory.booleanTypeInfo)) {
compareType = CompareType.COMPARE_BOOL;
boi0 = (BooleanObjectInspector) arguments[0];
boi1 = (BooleanObjectInspector) arguments[1];
} else {
TypeInfo oiTypeInfo0 = TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]);
TypeInfo oiTypeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]);

if (oiTypeInfo0 == oiTypeInfo1
|| TypeInfoUtils.doPrimitiveCategoriesMatch(oiTypeInfo0, oiTypeInfo1)) {
boi0 = (BooleanObjectInspector) arg0;
boi1 = (BooleanObjectInspector) arg1;
} else {
if (type0 == type1 || TypeInfoUtils.doPrimitiveCategoriesMatch(type0, type1)) {
compareType = CompareType.SAME_TYPE;
} else {
compareType = CompareType.NEED_CONVERT;
TypeInfo compareType = FunctionRegistry.getCommonClassForComparison(
oiTypeInfo0, oiTypeInfo1);
TypeInfo compareType = FunctionRegistry.getCommonClassForComparison(type0, type1);

// For now, we always convert to double if we can't find a common type
compareOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(
(compareType == null) ?
TypeInfoFactory.doubleTypeInfo : compareType);
(compareType == null) ? TypeInfoFactory.doubleTypeInfo : compareType);

converter0 = ObjectInspectorConverters.getConverter(arguments[0], compareOI);
converter1 = ObjectInspectorConverters.getConverter(arguments[1], compareOI);
converter0 = ObjectInspectorConverters.getConverter(arg0, compareOI);
converter1 = ObjectInspectorConverters.getConverter(arg1, compareOI);

checkConversionAllowed(arguments[0], compareOI);
checkConversionAllowed(arguments[1], compareOI);
checkConversionAllowed(arg0, compareOI);
checkConversionAllowed(arg1, compareOI);
}
}
return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;

}

private void initForNonPrimitives(ObjectInspector arg0, ObjectInspector arg1) throws UDFArgumentException {
assert arg0.getCategory() != Category.PRIMITIVE;
assert arg1.getCategory() != Category.PRIMITIVE;
assert arg0.getCategory() == arg1.getCategory();
final TypeInfo type0 = TypeInfoUtils.getTypeInfoFromObjectInspector(arg0);
final TypeInfo type1 = TypeInfoUtils.getTypeInfoFromObjectInspector(arg1);
if (type0.equals(type1)) {
compareType = CompareType.SAME_TYPE;
} else {
throw new UDFArgumentException("Type mismatch in " + opName + "(" + type0 + "," + type1 + ")");
}
}

protected void checkConversionAllowed(ObjectInspector argOI, ObjectInspector compareOI)
throws UDFArgumentException {
if (primitiveGroupOf(argOI) != PrimitiveGrouping.DATE_GROUP) {
Expand Down
2 changes: 1 addition & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java
View file
Open in desktop
Expand Up @@ -88,7 +88,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments)
for (ObjectInspector oi : arguments) {
if(!conversionHelper.updateForComparison(oi)) {
StringBuilder sb = new StringBuilder();
sb.append("The arguments for IN should be the same type! Types are: {");
sb.append("Type mismatch: {");
sb.append(arguments[0].getTypeName());
sb.append(" IN (");
for(int i=1; i<arguments.length; i++) {
Expand Down
9 changes: 9 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
View file
Open in desktop
Expand Up @@ -23,6 +23,7 @@
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;

/**
Expand Down Expand Up @@ -157,6 +158,14 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException {
return result;
}

@Override
protected boolean supportsCategory(ObjectInspector.Category c) {
return super.supportsCategory(c) ||
c == ObjectInspector.Category.MAP ||
c == ObjectInspector.Category.STRUCT ||
c == ObjectInspector.Category.LIST;
}

@Override
public GenericUDF negative() {
return new GenericUDFOPNotEqual();
Expand Down
9 changes: 9 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
View file
Open in desktop
Expand Up @@ -23,6 +23,7 @@
import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressionsSupportDecimal64;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;

/**
Expand Down Expand Up @@ -157,6 +158,14 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException {
return result;
}

@Override
protected boolean supportsCategory(ObjectInspector.Category c) {
return super.supportsCategory(c) ||
c == ObjectInspector.Category.MAP ||
c == ObjectInspector.Category.STRUCT ||
c == ObjectInspector.Category.LIST;
}

@Override
public GenericUDF negative() {
return new GenericUDFOPEqual();
Expand Down