Skip to content

Commit

Permalink
Enable SPLIT for BINARY and VARBINARY arguments
Browse files Browse the repository at this point in the history 
Implement SqlFunctions.split from scratch because argument is a string
not a regex.
  • Loading branch information
@julianhyde
julianhyde committed Mar 21, 2023
1 parent a47a9ef commit 32aa40f
Show file tree
Hide file tree
Showing 9 changed files with 206 additions and 16 deletions.
23 changes: 23 additions & 0 deletions babel/src/test/resources/sql/big-query.iq
View file
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,29 @@ SELECT SPLIT("") as result;

!ok

WITH letters AS
(SELECT x'' as letter_group
UNION ALL
SELECT x'41' as letter_group
UNION ALL
SELECT x'42ff43ff44' as letter_group)
SELECT SPLIT(letter_group, x'ff') as example
FROM letters;
+-----------+
| example |
+-----------+
| [] |
| [A] |
| [B, C, D] |
+-----------+
(3 rows)

!ok

SELECT SPLIT(x'abc2') as result;
Call to function 'SPLIT' with operand of type 'BINARY(2)' requires delimiter argument
!error

#####################################################################
# STRING
#
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/apache/calcite/runtime/CalciteResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,10 @@ ExInst<SqlValidatorException> intervalFractionalSecondPrecisionOutOfRange(
ExInst<SqlValidatorException> argumentMustBeValidPrecision(String a0, int a1,
int a2);

@BaseMessage("Call to function ''{0}'' with argument of type ''{1}'' requires extra delimiter argument")
ExInst<SqlValidatorException> delimiterIsRequired(String functionName,
String argumentTypeName);

@BaseMessage("Wrong arguments for table function ''{0}'' call. Expected ''{1}'', actual ''{2}''")
ExInst<CalciteException> illegalArgumentForTableFunctionCall(String a0,
String a1, String a2);
Expand Down
40 changes: 38 additions & 2 deletions core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@

import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;

import org.checkerframework.checker.nullness.qual.Nullable;
import org.checkerframework.checker.nullness.qual.PolyNull;
Expand Down Expand Up @@ -464,15 +465,50 @@ public static boolean startsWith(ByteString s0, ByteString s1) {
}

/** SQL {@code SPLIT(string, string)} function. */
public static List<String> split(String s0, String s1) {
return Arrays.asList(s0.split(s1));
public static List<String> split(String s, String delimiter) {
if (s.isEmpty()) {
return ImmutableList.of();
}
if (delimiter.isEmpty()) {
return ImmutableList.of(s); // prevent mischief
}
final List<String> list = new ArrayList<>();
for (int i = 0;;) {
int j = s.indexOf(delimiter, i);
if (j < 0) {
list.add(s.substring(i));
return list;
}
list.add(s.substring(i, j));
i = j + delimiter.length();
}
}

/** SQL {@code SPLIT(string)} function. */
public static List<String> split(String s) {
return split(s, ",");
}

/** SQL {@code SPLIT(binary, binary)} function. */
public static List<ByteString> split(ByteString s, ByteString delimiter) {
if (s.length() == 0) {
return ImmutableList.of();
}
if (delimiter.length() == 0) {
return ImmutableList.of(s); // prevent mischief
}
final List<ByteString> list = new ArrayList<>();
for (int i = 0;;) {
int j = s.indexOf(delimiter, i);
if (j < 0) {
list.add(s.substring(i));
return list;
}
list.add(s.substring(i, j));
i = j + delimiter.length();
}
}

/** SQL SUBSTRING(string FROM ...) function. */
public static String substring(String c, int s) {
final int s0 = s - 1;
Expand Down
46 changes: 35 additions & 11 deletions core/src/main/java/org/apache/calcite/sql/SqlBasicFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.validate.SqlMonotonicity;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorScope;

import org.checkerframework.checker.nullness.qual.Nullable;

Expand All @@ -45,6 +46,7 @@ public class SqlBasicFunction extends SqlFunction {
private final SqlSyntax syntax;
private final boolean deterministic;
private final SqlOperandHandler operandHandler;
private final int callValidator;
private final Function<SqlOperatorBinding, SqlMonotonicity> monotonicityInference;

//~ Constructors -----------------------------------------------------------
Expand All @@ -60,6 +62,7 @@ public class SqlBasicFunction extends SqlFunction {
* @param operandTypeInference Strategy to use for parameter type inference
* @param operandHandler Strategy to use for handling operands
* @param operandTypeChecker Strategy to use for parameter type checking
* @param callValidator Strategy to validate calls
* @param category Categorization for function
* @param monotonicityInference Strategy to infer monotonicity of a call
*/
Expand All @@ -68,6 +71,7 @@ private SqlBasicFunction(String name, SqlKind kind, SqlSyntax syntax,
@Nullable SqlOperandTypeInference operandTypeInference,
SqlOperandHandler operandHandler,
SqlOperandTypeChecker operandTypeChecker,
Integer callValidator,
SqlFunctionCategory category,
Function<SqlOperatorBinding, SqlMonotonicity> monotonicityInference) {
super(name, kind,
Expand All @@ -77,6 +81,7 @@ private SqlBasicFunction(String name, SqlKind kind, SqlSyntax syntax,
this.syntax = requireNonNull(syntax, "syntax");
this.deterministic = deterministic;
this.operandHandler = requireNonNull(operandHandler, "operandHandler");
this.callValidator = requireNonNull(callValidator, "callValidator");
this.monotonicityInference =
requireNonNull(monotonicityInference, "monotonicityInference");
}
Expand All @@ -88,7 +93,7 @@ public static SqlBasicFunction create(SqlKind kind,
SqlOperandTypeChecker operandTypeChecker) {
return new SqlBasicFunction(kind.name(), kind,
SqlSyntax.FUNCTION, true, returnTypeInference, null,
OperandHandlers.DEFAULT, operandTypeChecker,
OperandHandlers.DEFAULT, operandTypeChecker, 0,
SqlFunctionCategory.SYSTEM, call -> SqlMonotonicity.NOT_MONOTONIC);
}

Expand All @@ -100,7 +105,7 @@ public static SqlBasicFunction create(String name,
SqlOperandTypeChecker operandTypeChecker) {
return new SqlBasicFunction(name, SqlKind.OTHER_FUNCTION,
SqlSyntax.FUNCTION, true, returnTypeInference, null,
OperandHandlers.DEFAULT, operandTypeChecker,
OperandHandlers.DEFAULT, operandTypeChecker, 0,
SqlFunctionCategory.NUMERIC, call -> SqlMonotonicity.NOT_MONOTONIC);
}

Expand All @@ -111,7 +116,7 @@ public static SqlBasicFunction create(String name,
SqlOperandTypeChecker operandTypeChecker, SqlFunctionCategory category) {
return new SqlBasicFunction(name, SqlKind.OTHER_FUNCTION,
SqlSyntax.FUNCTION, true, returnTypeInference, null,
OperandHandlers.DEFAULT, operandTypeChecker,
OperandHandlers.DEFAULT, operandTypeChecker, 0,
category, call -> SqlMonotonicity.NOT_MONOTONIC);
}

Expand Down Expand Up @@ -141,32 +146,40 @@ public static SqlBasicFunction create(String name,
return operandHandler.rewriteCall(validator, call);
}

@Override public void validateCall(SqlCall call, SqlValidator validator,
SqlValidatorScope scope, SqlValidatorScope operandScope) {
super.validateCall(call, validator, scope, operandScope);
}

/** Returns a copy of this function with a given name. */
public SqlBasicFunction withName(String name) {
return new SqlBasicFunction(name, kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}

/** Returns a copy of this function with a given kind. */
public SqlBasicFunction withKind(SqlKind kind) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}

/** Returns a copy of this function with a given category. */
public SqlBasicFunction withFunctionType(SqlFunctionCategory category) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), category, monotonicityInference);
getOperandTypeChecker(), callValidator, category, monotonicityInference);
}

/** Returns a copy of this function with a given syntax. */
public SqlBasicFunction withSyntax(SqlSyntax syntax) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}

/** Returns a copy of this function with a given strategy for inferring
Expand All @@ -175,21 +188,24 @@ public SqlBasicFunction withOperandTypeInference(
SqlOperandTypeInference operandTypeInference) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), operandTypeInference, operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}

/** Returns a copy of this function with a given strategy for handling
* operands. */
public SqlBasicFunction withOperandHandler(SqlOperandHandler operandHandler) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}
/** Returns a copy of this function with a given determinism. */
public SqlBasicFunction withDeterministic(boolean deterministic) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}

/** Returns a copy of this function with a given strategy for inferring
Expand All @@ -198,6 +214,14 @@ public SqlBasicFunction withMonotonicityInference(
Function<SqlOperatorBinding, SqlMonotonicity> monotonicityInference) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), getFunctionType(), monotonicityInference);
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}

public SqlFunction withValidation(int callValidator) {
return new SqlBasicFunction(getName(), kind, syntax, deterministic,
getReturnTypeInference(), getOperandTypeInference(), operandHandler,
getOperandTypeChecker(), callValidator,
getFunctionType(), monotonicityInference);
}
}
24 changes: 21 additions & 3 deletions core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,10 @@
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeTransforms;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.Litmus;
import org.apache.calcite.util.Optionality;
import org.apache.calcite.util.Static;

import org.checkerframework.checker.nullness.qual.Nullable;

Expand Down Expand Up @@ -259,9 +261,25 @@ private SqlLibraryOperators() {
@LibraryOperator(libraries = {BIG_QUERY})
public static final SqlFunction SPLIT =
SqlBasicFunction.create("SPLIT",
ReturnTypes.ARG0.andThen(SqlTypeTransforms.TO_ARRAY),
OperandTypes.STRING_OPTIONAL_STRING,
SqlFunctionCategory.STRING);
ReturnTypes.ARG0
.andThen(SqlLibraryOperators::deriveTypeSplit)
.andThen(SqlTypeTransforms.TO_ARRAY),
OperandTypes.or(OperandTypes.CHARACTER_CHARACTER,
OperandTypes.CHARACTER,
OperandTypes.BINARY_BINARY,
OperandTypes.BINARY),
SqlFunctionCategory.STRING)
.withValidation(3);

static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding,
RelDataType type) {
if (SqlTypeUtil.isBinary(type) && operatorBinding.getOperandCount() == 1) {
throw operatorBinding.newError(
Static.RESOURCE.delimiterIsRequired(
operatorBinding.getOperator().getName(), type.toString()));
}
return type;
}

/** Generic "SUBSTR(string, position [, substringLength ])" function. */
private static final SqlBasicFunction SUBSTR =
Expand Down
3 changes: 3 additions & 0 deletions core/src/main/java/org/apache/calcite/sql/type/OperandTypes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,9 @@ public static SqlOperandTypeChecker variadic(
public static final SqlSingleOperandTypeChecker BINARY =
family(SqlTypeFamily.BINARY);

public static final SqlSingleOperandTypeChecker BINARY_BINARY =
family(SqlTypeFamily.BINARY, SqlTypeFamily.BINARY);

public static final SqlSingleOperandTypeChecker STRING =
family(SqlTypeFamily.STRING);

Expand Down
1 change: 1 addition & 0 deletions core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ NullIllegal=Illegal use of ''NULL''
DynamicParamIllegal=Illegal use of dynamic parameter
InvalidBoolean=''{0}'' is not a valid boolean value
ArgumentMustBeValidPrecision=Argument to function ''{0}'' must be a valid precision between ''{1,number,#}'' and ''{2,number,#}''
DelimiterIsRequired=Call to function ''{0}'' with argument of type ''{1}'' requires extra delimiter argument
IllegalArgumentForTableFunctionCall=Wrong arguments for table function ''{0}'' call. Expected ''{1}'', actual ''{2}''
CannotCallTableFunctionHere=Cannot call table function here: ''{0}''
InvalidTimeFrame=''{0}'' is not a valid time frame
Expand Down

0 comments on commit 32aa40f

Please sign in to comment.