Skip to content
Permalink
Browse files
[NO ISSUE][FUN] Add cast-null functions for constructors
- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Add cast-null type constructors that produce null for missing input.
- Make views use these types of constructors.
- Make secondary indexes with CAST (DEFAULT NULL) modifier use
  these types of constructors.

Change-Id: I120d18c4c7bbcf098fceaa6784cce265a2811e3b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14027
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
  • Loading branch information
AliSolaiman committed Nov 12, 2021
1 parent bd7208d commit 9f6bf4071ffcef1117435b4b150d7cf757d6d7fb
Show file tree
Hide file tree
Showing 53 changed files with 2,575 additions and 848 deletions.
@@ -890,21 +890,14 @@ private ScalarFunctionCallExpression castFunction(FunctionIdentifier castFun, IA

private ScalarFunctionCallExpression constructorFunction(IAType requiredType,
AbstractFunctionCallExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
FunctionIdentifier typeConstructorFun = TypeUtil.getTypeConstructor(requiredType);
FunctionIdentifier typeConstructorFun = TypeUtil.getTypeConstructorDefaultNull(requiredType);
if (typeConstructorFun == null) {
throw new CompilationException(ErrorCode.COMPILATION_TYPE_UNSUPPORTED, sourceLoc, "index",
requiredType.getTypeName());
}
// make CONSTRUCTOR(IF_MISSING(input, NULL))
BuiltinFunctionInfo ifMissingInfo = BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.IF_MISSING);
ScalarFunctionCallExpression ifMissingExpr = new ScalarFunctionCallExpression(ifMissingInfo);
ifMissingExpr.getArguments().add(new MutableObject<>(inputExpr));
ifMissingExpr.getArguments().add(new MutableObject<>(ConstantExpression.NULL));
ifMissingExpr.setSourceLocation(sourceLoc);

BuiltinFunctionInfo typeConstructorInfo = BuiltinFunctions.getBuiltinFunctionInfo(typeConstructorFun);
ScalarFunctionCallExpression constructorExpr = new ScalarFunctionCallExpression(typeConstructorInfo);
constructorExpr.getArguments().add(new MutableObject<>(ifMissingExpr));
constructorExpr.getArguments().add(new MutableObject<>(inputExpr));
constructorExpr.setSourceLocation(sourceLoc);
return constructorExpr;
}
@@ -77,7 +77,9 @@ public static Collection<Object[]> tests() {

// We test all functions except record and cast functions, which requires type settings (we test them
// in runtime tests).
if (!className.contains("record") && !className.contains("Cast")) {
// TODO(ali): ASTERIXDB-2982 do it in a proper way so that it does not exclude classes inadvertently
if (!className.contains("record") && !className.contains("Cast")
&& !className.contains("DefaultNull")) {
tests.add(new Object[] { getTestName(functionDescriptor.getClass()), functionDescriptor });
} else {
LOGGER.log(Level.INFO, "Excluding " + className);
@@ -89,8 +89,9 @@ public static Collection<Object[]> tests() {
// We test all functions except record, cast and full-text contains functions,
// which requires type settings or argument settings.
// Instead, we test them in runtime tests.
// TODO(ali): ASTERIXDB-2982 do it in a proper way so that it does not exclude classes inadvertently
if (!className.contains("record") && !className.contains("Cast")
&& !className.contains("FullTextContains")) {
&& !className.contains("FullTextContains") && !className.contains("DefaultNull")) {
tests.add(new Object[] { getTestName(functionDescriptor.getClass()), functionDescriptor });
} else {
LOGGER.log(Level.INFO, "Excluding " + className);
@@ -13324,19 +13324,19 @@
<expected-warn>ASX0006: Invalid format for duration in k (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0006: Invalid format for yearmonthduration in m (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0006: Invalid format for daytimeduration in n (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: boolean() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int8() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int16() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int32() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int64() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: float() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: double() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: datetime() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: date() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: time() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: duration() cannot process input type date (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: year-month-duration() cannot process input type date (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: day-time-duration() cannot process input type date (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: boolean-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int8-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int16-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int32-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: int64-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: float-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: double-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: datetime-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: date-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: time-default-null() cannot process input type duration (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: duration-default-null() cannot process input type date (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: year-month-duration-default-null() cannot process input type date (in line 30, at column 6)</expected-warn>
<expected-warn>ASX0004: Unsupported type: day-time-duration-default-null() cannot process input type date (in line 30, at column 6)</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="view" check-warnings="true">
@@ -144,7 +144,7 @@ public static void validateViewItemType(ARecordType recordType, SourceLocation s
} else {
primeType = fieldType;
}
if (TypeUtil.getTypeConstructor(primeType) == null) {
if (TypeUtil.getTypeConstructorDefaultNull(primeType) == null) {
throw new CompilationException(ErrorCode.COMPILATION_TYPE_UNSUPPORTED, sourceLoc, "view",
primeType.getTypeName());
}
@@ -176,8 +176,8 @@ public static Expression createTypeConvertExpression(Expression inExpr, IAType t
SourceLocation sourceLoc) throws CompilationException {
String format = temporalDataFormat != null ? getTemporalFormat(targetType, temporalDataFormat) : null;
boolean withFormat = format != null;
FunctionIdentifier constrFid = withFormat ? TypeUtil.getTypeConstructorWithFormat(targetType)
: TypeUtil.getTypeConstructor(targetType);
FunctionIdentifier constrFid = withFormat ? TypeUtil.getTypeConstructorWithFormatDefaultNull(targetType)
: TypeUtil.getTypeConstructorDefaultNull(targetType);
if (constrFid == null) {
throw new CompilationException(ErrorCode.COMPILATION_TYPE_UNSUPPORTED, sourceLoc, viewName.toString(),
targetType.getTypeName());
@@ -173,7 +173,6 @@ static Expression castViewBodyAsType(LangRewritingContext context, Expression bo
primeType = fieldType;
}
Expression expr = ViewUtil.createFieldAccessExpression(fromVar, fieldName, sourceLoc);
expr = ViewUtil.createMissingToNullExpression(expr, sourceLoc); // Default Null handling
Expression projectExpr =
ViewUtil.createTypeConvertExpression(expr, primeType, temporalDataFormat, viewName, sourceLoc);
VarIdentifier projectVar = context.newVariable();
@@ -31,7 +31,6 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.functions.IFunctionManager;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
@@ -41,7 +40,6 @@
import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
@@ -315,7 +313,7 @@ private IScalarEvaluatorFactory createFieldAccessors(int field, boolean isOverri

IScalarEvaluatorFactory castFieldEvalFactory;
if (IndexUtil.castDefaultNull(index)) {
castFieldEvalFactory = createConstructorFunction(funManger, dataFormat, fieldEvalFactory, fieldType);
castFieldEvalFactory = createConstructorFunction(funManger, fieldEvalFactory, fieldType);
} else if (index.isEnforced()) {
IScalarEvaluatorFactory[] castArg = new IScalarEvaluatorFactory[] { fieldEvalFactory };
castFieldEvalFactory =
@@ -328,22 +326,17 @@ private IScalarEvaluatorFactory createFieldAccessors(int field, boolean isOverri
return castFieldEvalFactory;
}

private IScalarEvaluatorFactory createConstructorFunction(IFunctionManager funManager, IDataFormat dataFormat,
private IScalarEvaluatorFactory createConstructorFunction(IFunctionManager funManager,
IScalarEvaluatorFactory fieldEvalFactory, IAType fieldType) throws AlgebricksException {
// make CONSTRUCTOR(IF_MISSING(field_access, NULL))
IFunctionDescriptor ifMissing = funManager.lookupFunction(BuiltinFunctions.IF_MISSING, sourceLoc);
IScalarEvaluatorFactory nullEvalFactory = dataFormat.getConstantEvalFactory(ConstantExpression.NULL.getValue());
IScalarEvaluatorFactory[] ifMissingArgs = new IScalarEvaluatorFactory[] { fieldEvalFactory, nullEvalFactory };
ifMissing.setSourceLocation(sourceLoc);
IScalarEvaluatorFactory ifMissingEvalFactory = ifMissing.createEvaluatorFactory(ifMissingArgs);
FunctionIdentifier typeConstructorFun = TypeUtil.getTypeConstructor(TypeComputeUtils.getActualType(fieldType));
FunctionIdentifier typeConstructorFun =
TypeUtil.getTypeConstructorDefaultNull(TypeComputeUtils.getActualType(fieldType));
if (typeConstructorFun == null) {
throw new CompilationException(ErrorCode.COMPILATION_TYPE_UNSUPPORTED, sourceLoc, "index",
fieldType.getTypeName());
}
IFunctionDescriptor typeConstructor = funManager.lookupFunction(typeConstructorFun, sourceLoc);
typeConstructor.setSourceLocation(sourceLoc);
return typeConstructor.createEvaluatorFactory(new IScalarEvaluatorFactory[] { ifMissingEvalFactory });
return typeConstructor.createEvaluatorFactory(new IScalarEvaluatorFactory[] { fieldEvalFactory });
}

private int[] createFieldPermutationForBulkLoadOp(int numSecondaryKeyFields) {
@@ -62,53 +62,63 @@ public class TypeUtil {
private TypeUtil() {
}

public static FunctionIdentifier getTypeConstructor(IAType type) {
/**
* @param type type
*
* @return a type constructor that produces NULL for MISSING input
*/
public static FunctionIdentifier getTypeConstructorDefaultNull(IAType type) {
switch (type.getTypeTag()) {
case TINYINT:
return BuiltinFunctions.INT8_CONSTRUCTOR;
return BuiltinFunctions.INT8_DEFAULT_NULL_CONSTRUCTOR;
case SMALLINT:
return BuiltinFunctions.INT16_CONSTRUCTOR;
return BuiltinFunctions.INT16_DEFAULT_NULL_CONSTRUCTOR;
case INTEGER:
return BuiltinFunctions.INT32_CONSTRUCTOR;
return BuiltinFunctions.INT32_DEFAULT_NULL_CONSTRUCTOR;
case BIGINT:
return BuiltinFunctions.INT64_CONSTRUCTOR;
return BuiltinFunctions.INT64_DEFAULT_NULL_CONSTRUCTOR;
case FLOAT:
return BuiltinFunctions.FLOAT_CONSTRUCTOR;
return BuiltinFunctions.FLOAT_DEFAULT_NULL_CONSTRUCTOR;
case DOUBLE:
return BuiltinFunctions.DOUBLE_CONSTRUCTOR;
return BuiltinFunctions.DOUBLE_DEFAULT_NULL_CONSTRUCTOR;
case BOOLEAN:
return BuiltinFunctions.BOOLEAN_CONSTRUCTOR;
return BuiltinFunctions.BOOLEAN_DEFAULT_NULL_CONSTRUCTOR;
case STRING:
return BuiltinFunctions.STRING_CONSTRUCTOR;
return BuiltinFunctions.STRING_DEFAULT_NULL_CONSTRUCTOR;
case DATE:
return BuiltinFunctions.DATE_CONSTRUCTOR;
return BuiltinFunctions.DATE_DEFAULT_NULL_CONSTRUCTOR;
case TIME:
return BuiltinFunctions.TIME_CONSTRUCTOR;
return BuiltinFunctions.TIME_DEFAULT_NULL_CONSTRUCTOR;
case DATETIME:
return BuiltinFunctions.DATETIME_CONSTRUCTOR;
return BuiltinFunctions.DATETIME_DEFAULT_NULL_CONSTRUCTOR;
case YEARMONTHDURATION:
return BuiltinFunctions.YEAR_MONTH_DURATION_CONSTRUCTOR;
return BuiltinFunctions.YEAR_MONTH_DURATION_DEFAULT_NULL_CONSTRUCTOR;
case DAYTIMEDURATION:
return BuiltinFunctions.DAY_TIME_DURATION_CONSTRUCTOR;
return BuiltinFunctions.DAY_TIME_DURATION_DEFAULT_NULL_CONSTRUCTOR;
case DURATION:
return BuiltinFunctions.DURATION_CONSTRUCTOR;
return BuiltinFunctions.DURATION_DEFAULT_NULL_CONSTRUCTOR;
case UUID:
return BuiltinFunctions.UUID_CONSTRUCTOR;
return BuiltinFunctions.UUID_DEFAULT_NULL_CONSTRUCTOR;
case BINARY:
return BuiltinFunctions.BINARY_BASE64_CONSTRUCTOR;
return BuiltinFunctions.BINARY_BASE64_DEFAULT_NULL_CONSTRUCTOR;
default:
return null;
}
}

public static FunctionIdentifier getTypeConstructorWithFormat(IAType type) {
/**
* @param type type
*
* @return a type constructor that produces NULL for MISSING input
*/
public static FunctionIdentifier getTypeConstructorWithFormatDefaultNull(IAType type) {
switch (type.getTypeTag()) {
case DATE:
return BuiltinFunctions.DATE_CONSTRUCTOR_WITH_FORMAT;
return BuiltinFunctions.DATE_DEFAULT_NULL_CONSTRUCTOR_WITH_FORMAT;
case TIME:
return BuiltinFunctions.TIME_CONSTRUCTOR_WITH_FORMAT;
return BuiltinFunctions.TIME_DEFAULT_NULL_CONSTRUCTOR_WITH_FORMAT;
case DATETIME:
return BuiltinFunctions.DATETIME_CONSTRUCTOR_WITH_FORMAT;
return BuiltinFunctions.DATETIME_DEFAULT_NULL_CONSTRUCTOR_WITH_FORMAT;
default:
return null;
}

0 comments on commit 9f6bf40

Please sign in to comment.