diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java index 49db8ed7e2445c..590a6582594e35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java @@ -78,18 +78,14 @@ public static String toSql(ScalarFunction fn, boolean ifNotExists) { .append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\""); boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; sb.append(",\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\""); - if (!fn.isUDTFunction()) { - sb.append(",\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\""); - } + sb.append(",\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\""); } else if (fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) { appendFileIfPresent(sb, fn, true); boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; sb.append(",\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\""); sb.append(",\n \"RUNTIME_VERSION\"=").append("\"" + Strings.nullToEmpty(fn.getRuntimeVersion()) + "\""); appendExpirationTimeIfNeeded(sb, fn); - if (!fn.isUDTFunction()) { - sb.append(",\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\""); - } + sb.append(",\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\""); } else { sb.append(",\n \"OBJECT_FILE\"=") .append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\""); @@ -167,6 +163,7 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) { .append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\","); boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE; sb.append("\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\","); + sb.append("\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\","); } else if (fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) { appendFileIfPresent(sb, fn, false); if (fn.getLocation() != null) { @@ -179,6 +176,7 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) { if (fn.getExpirationTime() != DEFAULT_EXPIRATION_TIME) { sb.append("\n \"EXPIRATION_TIME\"=").append("\"" + fn.getExpirationTime() + "\","); } + sb.append("\n \"VOLATILITY\"=").append("\"" + fn.getVolatility().toSql() + "\","); } else { sb.append("\n \"OBJECT_FILE\"=") .append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\","); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java index 7b4229058c1f66..08b78f05cb69e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Function; import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.FunctionVolatility; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; @@ -45,8 +46,27 @@ default boolean nullable() { NullableMode getNullableMode(); + FunctionVolatility getVolatility(); + List children(); + default boolean isImmutable() { + return getVolatility() == FunctionVolatility.IMMUTABLE; + } + + default boolean isStable() { + return getVolatility() == FunctionVolatility.STABLE; + } + + default boolean isVolatile() { + return getVolatility() == FunctionVolatility.VOLATILE; + } + + @Override + default boolean isDeterministic() { + return isImmutable(); + } + @Override default boolean foldable() { // Udf should not be folded in FE. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java index c3eebfc283fd0c..0c2618702b5268 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.FunctionVolatility; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.URI; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -50,6 +51,7 @@ public class JavaUdaf extends AggregateFunction implements ExplicitlyCastableSig private final FunctionSignature signature; private final DataType intermediateType; private final NullableMode nullableMode; + private final FunctionVolatility volatility; private final String objectFile; private final String symbol; private final String initFn; @@ -69,7 +71,7 @@ public class JavaUdaf extends AggregateFunction implements ExplicitlyCastableSig public JavaUdaf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, DataType intermediateType, NullableMode nullableMode, - String objectFile, String symbol, + FunctionVolatility volatility, String objectFile, String symbol, String initFn, String updateFn, String mergeFn, String serializeFn, String finalizeFn, String getValueFn, String removeFn, boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) { @@ -80,6 +82,7 @@ public JavaUdaf(String name, long functionId, String dbName, Function.BinaryType this.signature = signature; this.intermediateType = intermediateType == null ? signature.returnType : intermediateType; this.nullableMode = nullableMode; + this.volatility = volatility; this.objectFile = objectFile; this.symbol = symbol; this.initFn = initFn; @@ -121,8 +124,8 @@ public NullableMode getNullableMode() { public JavaUdaf withDistinctAndChildren(boolean isDistinct, List children) { Preconditions.checkArgument(children.size() == this.children.size()); return new JavaUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, - objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, - isDistinct, checkSum, isStaticLoad, expirationTime, children.toArray(new Expression[0])); + volatility, objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, + removeFn, isDistinct, checkSum, isStaticLoad, expirationTime, children.toArray(new Expression[0])); } /** @@ -152,6 +155,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca JavaUdaf udaf = new JavaUdaf(fnName, aggregate.getId(), dbName, aggregate.getBinaryType(), sig, intermediateType, aggregate.getNullableMode(), + aggregate.getVolatility(), aggregate.getLocation() == null ? null : aggregate.getLocation().getLocation(), aggregate.getSymbolName(), aggregate.getInitFnSymbol(), @@ -201,9 +205,15 @@ public Function getCatalogFunction() { expr.setId(functionId); expr.setStaticLoad(isStaticLoad); expr.setExpirationTime(expirationTime); + expr.setVolatility(volatility); return expr; } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); } } + + @Override + public FunctionVolatility getVolatility() { + return volatility; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java index 3e53200fa9cdce..7137f6149b1fd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java @@ -144,11 +144,6 @@ public JavaUdf withFreshVolatileIdentity() { children.toArray(new Expression[0])); } - @Override - public boolean isDeterministic() { - return volatility == FunctionVolatility.IMMUTABLE; - } - @Override public boolean equals(Object o) { if (!(o instanceof JavaUdf)) { @@ -226,6 +221,11 @@ public Function getCatalogFunction() { } } + @Override + public FunctionVolatility getVolatility() { + return volatility; + } + private static VolatileIdentity createVolatileIdentity(FunctionVolatility volatility) { return volatility == FunctionVolatility.VOLATILE ? VolatileIdentity.newVolatileIdentity() : VolatileIdentity.NON_VOLATILE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java index 2e04dec1d68163..2eb299f04056ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.FunctionVolatility; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.URI; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -49,6 +50,7 @@ public class JavaUdtf extends TableGeneratingFunction implements ExplicitlyCasta private final Function.BinaryType binaryType; private final FunctionSignature signature; private final NullableMode nullableMode; + private final FunctionVolatility volatility; private final String objectFile; private final String symbol; private final String prepareFn; @@ -62,7 +64,8 @@ public class JavaUdtf extends TableGeneratingFunction implements ExplicitlyCasta */ public JavaUdtf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, - NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, + NullableMode nullableMode, FunctionVolatility volatility, String objectFile, String symbol, + String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) { super(name, args); this.dbName = dbName; @@ -70,6 +73,7 @@ public JavaUdtf(String name, long functionId, String dbName, Function.BinaryType this.binaryType = binaryType; this.signature = signature; this.nullableMode = nullableMode; + this.volatility = volatility; this.objectFile = objectFile; this.symbol = symbol; this.prepareFn = prepareFn; @@ -86,7 +90,7 @@ public JavaUdtf(String name, long functionId, String dbName, Function.BinaryType public JavaUdtf withChildren(List children) { Preconditions.checkArgument(children.size() == this.children.size()); return new JavaUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, - objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, + volatility, objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, children.toArray(new Expression[0])); } @@ -125,6 +129,7 @@ public Function getCatalogFunction() { expr.setStaticLoad(isStaticLoad); expr.setExpirationTime(expirationTime); expr.setUDTFunction(true); + expr.setVolatility(volatility); return expr; } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); @@ -152,6 +157,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca JavaUdtf udf = new JavaUdtf(fnName, scalar.getId(), dbName, scalar.getBinaryType(), sig, scalar.getNullableMode(), + scalar.getVolatility(), scalar.getLocation() == null ? null : scalar.getLocation().getLocation(), scalar.getSymbolName(), scalar.getPrepareFnSymbol(), @@ -170,6 +176,11 @@ public NullableMode getNullableMode() { return nullableMode; } + @Override + public FunctionVolatility getVolatility() { + return volatility; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitJavaUdtf(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java index 456e0f1a6eac42..cd0d8de4978350 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.FunctionVolatility; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.URI; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -50,6 +51,7 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableS private final FunctionSignature signature; private final DataType intermediateType; private final NullableMode nullableMode; + private final FunctionVolatility volatility; private final String objectFile; private final String symbol; private final String initFn; @@ -71,7 +73,7 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableS public PythonUdaf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, DataType intermediateType, NullableMode nullableMode, - String objectFile, String symbol, + FunctionVolatility volatility, String objectFile, String symbol, String initFn, String updateFn, String mergeFn, String serializeFn, String finalizeFn, String getValueFn, String removeFn, boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime, @@ -83,6 +85,7 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy this.signature = signature; this.intermediateType = intermediateType == null ? signature.returnType : intermediateType; this.nullableMode = nullableMode; + this.volatility = volatility; this.objectFile = objectFile; this.symbol = symbol; this.initFn = initFn; @@ -126,7 +129,8 @@ public NullableMode getNullableMode() { public PythonUdaf withDistinctAndChildren(boolean isDistinct, List children) { Preconditions.checkArgument(children.size() == this.children.size()); return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode, - objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn, + volatility, objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, + removeFn, isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, children.toArray(new Expression[0])); } @@ -158,6 +162,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca PythonUdaf udaf = new PythonUdaf(fnName, aggregate.getId(), dbName, aggregate.getBinaryType(), sig, intermediateType, aggregate.getNullableMode(), + aggregate.getVolatility(), aggregate.getLocation() == null ? null : aggregate.getLocation().getLocation(), aggregate.getSymbolName(), aggregate.getInitFnSymbol(), @@ -211,9 +216,15 @@ public Function getCatalogFunction() { expr.setExpirationTime(expirationTime); expr.setRuntimeVersion(runtimeVersion); expr.setFunctionCode(functionCode); + expr.setVolatility(volatility); return expr; } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); } } + + @Override + public FunctionVolatility getVolatility() { + return volatility; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java index 65c8f2f14d7ddb..da3e866af27634 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java @@ -149,11 +149,6 @@ public PythonUdf withFreshVolatileIdentity() { runtimeVersion, functionCode, children.toArray(new Expression[0])); } - @Override - public boolean isDeterministic() { - return volatility == FunctionVolatility.IMMUTABLE; - } - @Override public boolean equals(Object o) { if (!(o instanceof PythonUdf)) { @@ -235,6 +230,11 @@ public Function getCatalogFunction() { } } + @Override + public FunctionVolatility getVolatility() { + return volatility; + } + private static VolatileIdentity createVolatileIdentity(FunctionVolatility volatility) { return volatility == FunctionVolatility.VOLATILE ? VolatileIdentity.newVolatileIdentity() : VolatileIdentity.NON_VOLATILE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java index 74e662aee7297e..c288e2b0a710ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdtf.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.FunctionName; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.catalog.FunctionVolatility; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.URI; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -49,6 +50,7 @@ public class PythonUdtf extends TableGeneratingFunction implements ExplicitlyCas private final Function.BinaryType binaryType; private final FunctionSignature signature; private final NullableMode nullableMode; + private final FunctionVolatility volatility; private final String objectFile; private final String symbol; private final String prepareFn; @@ -64,7 +66,8 @@ public class PythonUdtf extends TableGeneratingFunction implements ExplicitlyCas */ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryType binaryType, FunctionSignature signature, - NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn, + NullableMode nullableMode, FunctionVolatility volatility, String objectFile, String symbol, + String prepareFn, String closeFn, String checkSum, boolean isStaticLoad, long expirationTime, String runtimeVersion, String functionCode, Expression... args) { super(name, args); @@ -73,6 +76,7 @@ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryTy this.binaryType = binaryType; this.signature = signature; this.nullableMode = nullableMode; + this.volatility = volatility; this.objectFile = objectFile; this.symbol = symbol; this.prepareFn = prepareFn; @@ -91,7 +95,7 @@ public PythonUdtf(String name, long functionId, String dbName, Function.BinaryTy public PythonUdtf withChildren(List children) { Preconditions.checkArgument(children.size() == this.children.size()); return new PythonUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode, - objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, + volatility, objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, children.toArray(new Expression[0])); } @@ -132,6 +136,7 @@ public Function getCatalogFunction() { expr.setUDTFunction(true); expr.setRuntimeVersion(runtimeVersion); expr.setFunctionCode(functionCode); + expr.setVolatility(volatility); return expr; } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); @@ -159,6 +164,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca PythonUdtf udtf = new PythonUdtf(fnName, scalar.getId(), dbName, scalar.getBinaryType(), sig, scalar.getNullableMode(), + scalar.getVolatility(), scalar.getLocation() == null ? null : scalar.getLocation().getLocation(), scalar.getSymbolName(), scalar.getPrepareFnSymbol(), @@ -179,6 +185,11 @@ public NullableMode getNullableMode() { return nullableMode; } + @Override + public FunctionVolatility getVolatility() { + return volatility; + } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitPythonUdtf(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index bc5edcbb59ba52..aaa21e53af0194 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -184,6 +184,7 @@ public class CreateFunctionCommand extends Command implements ForwardWithSync { // if not, will core dump when input is not null column, but need return null // like https://github.com/apache/doris/pull/14002/files private NullableMode returnNullMode = NullableMode.ALWAYS_NULLABLE; + // Keep IMMUTABLE as the UDAF/UDTF default for compatibility with previous behavior. private FunctionVolatility volatility = FunctionVolatility.IMMUTABLE; private String runtimeVersion; private String functionCode; @@ -323,10 +324,6 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { throw new AnalysisException("do not support 'NATIVE' udf type after doris version 1.2.0," + "please use JAVA_UDF or RPC instead"); } - if (properties.containsKey(VOLATILITY) && (isAggregate || isTableFunction)) { - throw new AnalysisException("volatility property only supports scalar JAVA_UDF and PYTHON_UDF"); - } - userFile = properties.getOrDefault(FILE_KEY, properties.get(OBJECT_FILE_KEY)); originalUserFile = userFile; // Keep original jar name for BE // Convert userFile to realUrl only for FE checksum calculation @@ -346,9 +343,7 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { } if (binaryType == Function.BinaryType.JAVA_UDF) { FunctionUtil.checkEnableJavaUdf(); - if (!isAggregate && !isTableFunction) { - volatility = analyzeVolatility(); - } + volatility = analyzeVolatility(defaultVolatility()); // always_nullable the default value is true, equal null means true Boolean isReturnNull = parseBooleanFromProperties(IS_RETURN_NULL); @@ -363,9 +358,7 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { extractExpirationTime(); } else if (binaryType == Function.BinaryType.PYTHON_UDF) { FunctionUtil.checkEnablePythonUdf(); - if (!isAggregate && !isTableFunction) { - volatility = analyzeVolatility(); - } + volatility = analyzeVolatility(defaultVolatility()); // always_nullable the default value is true, equal null means true Boolean isReturnNull = parseBooleanFromProperties(IS_RETURN_NULL); @@ -387,9 +380,13 @@ private void analyzeCommon(ConnectContext ctx) throws AnalysisException { } } - private FunctionVolatility analyzeVolatility() throws AnalysisException { + private FunctionVolatility defaultVolatility() { + return isAggregate || isTableFunction ? FunctionVolatility.IMMUTABLE : FunctionVolatility.VOLATILE; + } + + private FunctionVolatility analyzeVolatility(FunctionVolatility defaultVolatility) throws AnalysisException { if (!properties.containsKey(VOLATILITY)) { - return FunctionVolatility.VOLATILE; + return defaultVolatility; } try { return FunctionVolatility.fromString(properties.get(VOLATILITY)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommand.java index 081d482308a1c1..58a849ce3082db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommand.java @@ -295,9 +295,8 @@ private String buildProperties(Function function) { } if (function instanceof ScalarFunction) { ScalarFunction scalarFunction = (ScalarFunction) function; - if (!scalarFunction.isUDTFunction() - && (function.getBinaryType() == Function.BinaryType.JAVA_UDF - || function.getBinaryType() == Function.BinaryType.PYTHON_UDF)) { + if (function.getBinaryType() == Function.BinaryType.JAVA_UDF + || function.getBinaryType() == Function.BinaryType.PYTHON_UDF) { properties.put("VOLATILITY", function.getVolatility().toSql()); } properties.put("SYMBOL", Strings.nullToEmpty(scalarFunction.getSymbolName())); @@ -311,6 +310,10 @@ private String buildProperties(Function function) { if (function instanceof AggregateFunction) { AggregateFunction aggregateFunction = (AggregateFunction) function; + if (function.getBinaryType() == Function.BinaryType.JAVA_UDF + || function.getBinaryType() == Function.BinaryType.PYTHON_UDF) { + properties.put("VOLATILITY", function.getVolatility().toSql()); + } properties.put("INIT_FN", Strings.nullToEmpty(aggregateFunction.getInitFnSymbol())); properties.put("UPDATE_FN", Strings.nullToEmpty(aggregateFunction.getUpdateFnSymbol())); properties.put("MERGE_FN", Strings.nullToEmpty(aggregateFunction.getMergeFnSymbol())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index 426a45074b85c3..cafef1cfb3b833 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -128,6 +128,17 @@ public void test() throws Exception { + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2');"; createFunction(defaultVolatileSql, ctx); Assert.assertEquals(FunctionVolatility.VOLATILE, findFunction(db, "py_default").getVolatility()); + + String defaultImmutableUdafSql = "create aggregate function db1.py_agg_default(int) returns int " + + "properties('type'='PYTHON_UDF', 'symbol'='Agg', 'runtime_version'='3.10.2');"; + createFunction(defaultImmutableUdafSql, ctx); + Assert.assertEquals(FunctionVolatility.IMMUTABLE, findFunction(db, "py_agg_default").getVolatility()); + + String stableUdtfSql = "create tables function db1.py_table_stable(int) returns array " + + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2', " + + "'volatility'='stable');"; + createFunction(stableUdtfSql, ctx); + Assert.assertEquals(FunctionVolatility.STABLE, findFunction(db, "py_table_stable").getVolatility()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToSqlConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToSqlConverterTest.java index d60a4502d2d2fd..9013c7671db51d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToSqlConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FunctionToSqlConverterTest.java @@ -154,7 +154,7 @@ void testScalarFunction_javaUdtfReplaySql() { ScalarFunction fn = ScalarFunction.createUdf(BinaryType.JAVA_UDF, name, argTypes, Type.INT, false, null, "com.example.TableFn", null, null); fn.setUDTFunction(true); - fn.setVolatility(FunctionVolatility.IMMUTABLE); + fn.setVolatility(FunctionVolatility.STABLE); String sql = FunctionToSqlConverter.toSql(fn, true); @@ -162,7 +162,7 @@ void testScalarFunction_javaUdtfReplaySql() { Assertions.assertTrue(sql.contains("java_table_fn(int)")); Assertions.assertTrue(sql.contains("RETURNS array")); Assertions.assertTrue(sql.contains("\"TYPE\"=\"JAVA_UDF\"")); - Assertions.assertFalse(sql.contains("VOLATILITY")); + Assertions.assertTrue(sql.contains("\"VOLATILITY\"=\"stable\"")); } @Test @@ -174,7 +174,7 @@ void testScalarFunction_pythonUdtfReplaySql() { fn.setUDTFunction(true); fn.setRuntimeVersion("3.10.2"); fn.setFunctionCode("def evaluate(x):\n yield x"); - fn.setVolatility(FunctionVolatility.IMMUTABLE); + fn.setVolatility(FunctionVolatility.VOLATILE); String sql = FunctionToSqlConverter.toSql(fn, false); @@ -182,10 +182,10 @@ void testScalarFunction_pythonUdtfReplaySql() { Assertions.assertTrue(sql.contains("py_table_fn(int)")); Assertions.assertTrue(sql.contains("RETURNS array")); Assertions.assertTrue(sql.contains("\"RUNTIME_VERSION\"=\"3.10.2\"")); + Assertions.assertTrue(sql.contains("\"VOLATILITY\"=\"volatile\"")); Assertions.assertTrue(sql.contains("\"TYPE\"=\"PYTHON_UDF\"")); Assertions.assertTrue(sql.contains("AS $$\ndef evaluate(x):\n yield x\n$$;")); Assertions.assertFalse(sql.contains("\"FILE\"=")); - Assertions.assertFalse(sql.contains("VOLATILITY")); } // ======================== ScalarFunction — IF NOT EXISTS ======================== @@ -263,6 +263,7 @@ void testAggregateFunction_javaUdf_basicSql() throws AnalysisException { .symbolName("com.example.MySum") .location(URI.create("file:///tmp/java-udaf.jar")) .build(); + fn.setVolatility(FunctionVolatility.STABLE); String sql = FunctionToSqlConverter.toSql(fn, false); @@ -272,6 +273,7 @@ void testAggregateFunction_javaUdf_basicSql() throws AnalysisException { Assertions.assertTrue(sql.contains("\"SYMBOL\"=\"com.example.MySum\"")); Assertions.assertTrue(sql.contains("\"FILE\"=\"file:///tmp/java-udaf.jar\"")); Assertions.assertTrue(sql.contains("\"TYPE\"=\"JAVA_UDF\"")); + Assertions.assertTrue(sql.contains("\"VOLATILITY\"=\"stable\"")); Assertions.assertTrue(sql.contains("\"ALWAYS_NULLABLE\"=")); Assertions.assertFalse(sql.contains("INIT_FN")); Assertions.assertFalse(sql.contains("UPDATE_FN")); @@ -313,11 +315,13 @@ void testAggregateFunction_pythonUdf_inlineReplaySql() { fn.setRuntimeVersion("3.10.2"); fn.setExpirationTime(45); fn.setFunctionCode("class SumState:\n pass"); + fn.setVolatility(FunctionVolatility.IMMUTABLE); String sql = FunctionToSqlConverter.toSql(fn, false); Assertions.assertTrue(sql.contains("\"RUNTIME_VERSION\"=\"3.10.2\"")); Assertions.assertTrue(sql.contains("\"EXPIRATION_TIME\"=\"45\"")); + Assertions.assertTrue(sql.contains("\"VOLATILITY\"=\"immutable\"")); Assertions.assertTrue(sql.contains("\"TYPE\"=\"PYTHON_UDF\"")); Assertions.assertTrue(sql.contains("AS $$\nclass SumState:\n pass\n$$;")); Assertions.assertFalse(sql.contains("\"FILE\"=")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java index 89fd24821bad40..08414067fbf331 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfVolatilityTest.java @@ -37,6 +37,9 @@ void testImmutablePythonUdfIsNotVolatileExpression() { PythonUdf udf = pythonUdf(FunctionVolatility.IMMUTABLE, VolatileIdentity.NON_VOLATILE); Assertions.assertTrue(udf.isDeterministic()); + Assertions.assertTrue(udf.isImmutable()); + Assertions.assertFalse(udf.isStable()); + Assertions.assertFalse(udf.isVolatile()); Assertions.assertFalse(udf.containsVolatileExpression()); Assertions.assertEquals(PythonUdf.class, new PythonUdfBuilder(udf).functionClass()); } @@ -47,6 +50,9 @@ void testVolatilePythonUdfUsesUniqueIdentity() { PythonUdf second = pythonUdf(FunctionVolatility.VOLATILE, VolatileIdentity.newVolatileIdentity()); Assertions.assertFalse(first.isDeterministic()); + Assertions.assertFalse(first.isImmutable()); + Assertions.assertFalse(first.isStable()); + Assertions.assertTrue(first.isVolatile()); Assertions.assertTrue(first.containsVolatileExpression()); Assertions.assertNotEquals(first, second); @@ -69,9 +75,32 @@ void testJavaUdfVolatility() { JavaUdf udf = javaUdf(FunctionVolatility.STABLE, VolatileIdentity.NON_VOLATILE); Assertions.assertFalse(udf.isDeterministic()); + Assertions.assertFalse(udf.isImmutable()); + Assertions.assertTrue(udf.isStable()); + Assertions.assertFalse(udf.isVolatile()); Assertions.assertFalse(udf.containsVolatileExpression()); } + @Test + void testPythonUdafVolatility() { + PythonUdaf immutable = pythonUdaf(FunctionVolatility.IMMUTABLE); + PythonUdaf stable = pythonUdaf(FunctionVolatility.STABLE); + + Assertions.assertTrue(immutable.isDeterministic()); + Assertions.assertFalse(stable.isDeterministic()); + Assertions.assertEquals(FunctionVolatility.STABLE, stable.getCatalogFunction().getVolatility()); + } + + @Test + void testPythonUdtfVolatility() { + PythonUdtf immutable = pythonUdtf(FunctionVolatility.IMMUTABLE); + PythonUdtf volatileUdtf = pythonUdtf(FunctionVolatility.VOLATILE); + + Assertions.assertTrue(immutable.isDeterministic()); + Assertions.assertFalse(volatileUdtf.isDeterministic()); + Assertions.assertEquals(FunctionVolatility.VOLATILE, volatileUdtf.getCatalogFunction().getVolatility()); + } + private PythonUdf pythonUdf(FunctionVolatility volatility, VolatileIdentity volatileIdentity) { return new PythonUdf("py_fn", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), NullableMode.ALWAYS_NULLABLE, volatility, volatileIdentity, @@ -85,6 +114,19 @@ private JavaUdf javaUdf(FunctionVolatility volatility, VolatileIdentity volatile null, "evaluate", null, null, "", false, 360, new IntegerLiteral(1)); } + private PythonUdaf pythonUdaf(FunctionVolatility volatility) { + return new PythonUdaf("py_agg", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), + IntegerType.INSTANCE, NullableMode.ALWAYS_NULLABLE, volatility, + null, "Agg", null, null, null, null, null, null, null, false, "", false, 360, + "3.10.2", "", new IntegerLiteral(1)); + } + + private PythonUdtf pythonUdtf(FunctionVolatility volatility) { + return new PythonUdtf("py_table", 1, "db1", Function.BinaryType.PYTHON_UDF, signature(), + NullableMode.ALWAYS_NULLABLE, volatility, + null, "evaluate", null, null, "", false, 360, "3.10.2", "", new IntegerLiteral(1)); + } + private FunctionSignature signature() { return FunctionSignature.ret(IntegerType.INSTANCE).args(IntegerType.INSTANCE); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommandTest.java index 716b9fa24da4db..3c1f3ba6b88898 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowFunctionsCommandTest.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AccessPrivilege; import org.apache.doris.catalog.AccessPrivilegeWithCols; +import org.apache.doris.catalog.AggregateFunction; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionName; @@ -130,35 +131,55 @@ void testBuildProperties_scalarUdfEmitsVolatility() { } @Test - void testBuildProperties_javaUdtfDoesNotEmitVolatility() { + void testBuildProperties_javaUdtfEmitsVolatility() { ShowFunctionsCommand sf = new ShowFunctionsCommand("test", true, null); ScalarFunction fn = ScalarFunction.createUdf(Function.BinaryType.JAVA_UDF, new FunctionName("test", "java_table_fn"), new Type[] {Type.INT}, Type.INT, false, null, "com.example.TableFn", null, null); fn.setUDTFunction(true); - fn.setVolatility(FunctionVolatility.IMMUTABLE); + fn.setVolatility(FunctionVolatility.STABLE); String properties = sf.buildPropertiesForTest(fn); Assertions.assertTrue(properties.contains("SYMBOL=com.example.TableFn")); - Assertions.assertFalse(properties.contains("VOLATILITY")); + Assertions.assertTrue(properties.contains("VOLATILITY=stable")); } @Test - void testBuildProperties_pythonUdtfDoesNotEmitVolatility() { + void testBuildProperties_pythonUdtfEmitsVolatility() { ShowFunctionsCommand sf = new ShowFunctionsCommand("test", true, null); ScalarFunction fn = ScalarFunction.createUdf(Function.BinaryType.PYTHON_UDF, new FunctionName("test", "py_table_fn"), new Type[] {Type.INT}, Type.INT, false, null, "evaluate", null, null); fn.setUDTFunction(true); fn.setRuntimeVersion("3.10.2"); - fn.setVolatility(FunctionVolatility.IMMUTABLE); + fn.setVolatility(FunctionVolatility.VOLATILE); String properties = sf.buildPropertiesForTest(fn); Assertions.assertTrue(properties.contains("RUNTIME_VERSION=3.10.2")); Assertions.assertTrue(properties.contains("SYMBOL=evaluate")); - Assertions.assertFalse(properties.contains("VOLATILITY")); + Assertions.assertTrue(properties.contains("VOLATILITY=volatile")); + } + + @Test + void testBuildProperties_udafEmitsVolatility() { + ShowFunctionsCommand sf = new ShowFunctionsCommand("test", true, null); + AggregateFunction fn = AggregateFunction.AggregateFunctionBuilder.createUdfBuilder() + .binaryType(Function.BinaryType.PYTHON_UDF) + .name(new FunctionName("test", "py_agg_fn")) + .argsType(new Type[] {Type.INT}) + .retType(Type.INT) + .intermediateType(Type.INT) + .hasVarArgs(false) + .symbolName("Agg") + .build(); + fn.setVolatility(FunctionVolatility.STABLE); + + String properties = sf.buildPropertiesForTest(fn); + + Assertions.assertTrue(properties.contains("SYMBOL=Agg")); + Assertions.assertTrue(properties.contains("VOLATILITY=stable")); } @Test