Skip to content

Commit

Permalink
Porting Presto UDFs for Presto 0.193
Browse files Browse the repository at this point in the history
  • Loading branch information
@amoghmargoor @shubhamtagra
amoghmargoor authored and shubhamtagra committed Apr 20, 2018
1 parent 4034260 commit 890b6ab
Show file tree
Hide file tree
Showing 6 changed files with 128 additions and 43 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Expand Up @@ -24,7 +24,8 @@ The details about how to plug in presto UDFs can be found [here](https://www.qub

| Presto Version| Last Compatible Release|
| ------------- |:-------------:|
| _ver 0.180_ | current |
| _ver 0.193_ | current |
| _ver 0.180_ | udfs-2.0.2 |
| _ver 0.157_ | udfs-2.0.1 |
| _ver 0.142_ | udfs-1.0.0 |
| _ver 0.119_ | udfs-0.1.3 |
Expand Down
8 changes: 4 additions & 4 deletions dependency-reduced-pom.xml
View file
Expand Up @@ -4,7 +4,7 @@
<groupId>com.qubole.presto</groupId>
<artifactId>udfs</artifactId>
<name>PrestoUDFs</name>
<version>2.0.1-SNAPSHOT</version>
<version>2.0.3-SNAPSHOT</version>
<description>Common Functions for the Facebook Presto SQL Engine</description>
<url>https://github.com/qubole/presto-udfs</url>
<developers>
Expand Down Expand Up @@ -151,13 +151,13 @@
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-spi</artifactId>
<version>0.180</version>
<version>0.193</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-main</artifactId>
<version>0.180</version>
<version>0.193</version>
<scope>compile</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -189,7 +189,7 @@
<javax.inject.version>1</javax.inject.version>
<guava.version>18.0</guava.version>
<maven.compiler.source>1.8</maven.compiler.source>
<presto.version>0.180</presto.version>
<presto.version>0.193</presto.version>
<maven.compiler.target>1.8</maven.compiler.target>
<io.airlift.log.version>0.125</io.airlift.log.version>
</properties>
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Expand Up @@ -21,7 +21,7 @@
<!--
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
-->
<presto.version>0.180</presto.version>
<presto.version>0.193</presto.version>
<slice.version>0.15</slice.version>
<guava.version>18.0</guava.version>
<javax.inject.version>1</javax.inject.version>
Expand Down
80 changes: 77 additions & 3 deletions src/main/java/com/qubole/presto/udfs/aggregation/ArrayAggregation.java
View file
Expand Up @@ -15,7 +15,6 @@
*/
package com.qubole.presto.udfs.aggregation;

import com.facebook.presto.bytecode.DynamicClassLoader;
import com.facebook.presto.metadata.BoundVariables;
import com.facebook.presto.metadata.FunctionRegistry;
import com.facebook.presto.metadata.Signature;
Expand All @@ -28,30 +27,38 @@
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.BlockBuilder;
import com.facebook.presto.spi.block.BlockBuilderStatus;
import com.facebook.presto.spi.type.Decimals;
import com.facebook.presto.spi.type.SqlDecimal;
import com.facebook.presto.spi.type.StandardTypes;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.spi.type.TypeSignature;
import com.facebook.presto.spi.type.ArrayType;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.qubole.presto.udfs.aggregation.state.ArrayAggregationState;
import com.qubole.presto.udfs.aggregation.state.ArrayAggregationStateFactory;
import com.qubole.presto.udfs.aggregation.state.ArrayAggregationStateSerializer;
import io.airlift.bytecode.DynamicClassLoader;
import io.airlift.slice.DynamicSliceOutput;
import io.airlift.slice.Slice;
import io.airlift.slice.SliceInput;
import io.airlift.slice.SliceOutput;
import io.airlift.slice.Slices;

import java.lang.invoke.MethodHandle;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INDEX;
import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INPUT_CHANNEL;
import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.STATE;
import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName;
import static com.facebook.presto.type.TypeJsonUtils.appendToBlockBuilder;
import static com.facebook.presto.spi.type.RealType.REAL;
import static com.facebook.presto.util.Reflection.methodHandle;
import static java.lang.Float.floatToRawIntBits;

public class ArrayAggregation
extends SqlAggregationFunction
Expand Down Expand Up @@ -101,7 +108,7 @@ private static InternalAggregationFunction generateAggregation(Type valueType)
outputType);

GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader);
return new InternalAggregationFunction(NAME, inputTypes, intermediateType, outputType, true, factory);
return new InternalAggregationFunction(NAME, inputTypes, intermediateType, outputType, true, true, factory);
}

private static List<ParameterMetadata> createInputParameterMetadata(Type value)
Expand Down Expand Up @@ -217,4 +224,71 @@ public static Block arrayBlockOf(List<Object> values, Type elementType)
}
return blockBuilder.build();
}

@VisibleForTesting
public static void appendToBlockBuilder(Type type, Object element, BlockBuilder blockBuilder)
{
Class<?> javaType = type.getJavaType();
if (element == null) {
blockBuilder.appendNull();
}
else if (type.getTypeSignature().getBase().equals(StandardTypes.ARRAY) && element instanceof Iterable<?>) {
BlockBuilder subBlockBuilder = blockBuilder.beginBlockEntry();
for (Object subElement : (Iterable<?>) element) {
appendToBlockBuilder(type.getTypeParameters().get(0), subElement, subBlockBuilder);
}
blockBuilder.closeEntry();
}
else if (type.getTypeSignature().getBase().equals(StandardTypes.ROW) && element instanceof Iterable<?>) {
BlockBuilder subBlockBuilder = blockBuilder.beginBlockEntry();
int field = 0;
for (Object subElement : (Iterable<?>) element) {
appendToBlockBuilder(type.getTypeParameters().get(field), subElement, subBlockBuilder);
field++;
}
blockBuilder.closeEntry();
}
else if (type.getTypeSignature().getBase().equals(StandardTypes.MAP) && element instanceof Map<?, ?>) {
BlockBuilder subBlockBuilder = blockBuilder.beginBlockEntry();
for (Map.Entry<?, ?> entry : ((Map<?, ?>) element).entrySet()) {
appendToBlockBuilder(type.getTypeParameters().get(0), entry.getKey(), subBlockBuilder);
appendToBlockBuilder(type.getTypeParameters().get(1), entry.getValue(), subBlockBuilder);
}
blockBuilder.closeEntry();
}
else if (javaType == boolean.class) {
type.writeBoolean(blockBuilder, (Boolean) element);
}
else if (javaType == long.class) {
if (element instanceof SqlDecimal) {
type.writeLong(blockBuilder, ((SqlDecimal) element).getUnscaledValue().longValue());
}
else if (REAL.equals(type)) {
type.writeLong(blockBuilder, floatToRawIntBits(((Number) element).floatValue()));
}
else {
type.writeLong(blockBuilder, ((Number) element).longValue());
}
}
else if (javaType == double.class) {
type.writeDouble(blockBuilder, ((Number) element).doubleValue());
}
else if (javaType == Slice.class) {
if (element instanceof String) {
type.writeSlice(blockBuilder, Slices.utf8Slice(element.toString()));
}
else if (element instanceof byte[]) {
type.writeSlice(blockBuilder, Slices.wrappedBuffer((byte[]) element));
}
else if (element instanceof SqlDecimal) {
type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(((SqlDecimal) element).getUnscaledValue()));
}
else {
type.writeSlice(blockBuilder, (Slice) element);
}
}
else {
type.writeObject(blockBuilder, element);
}
}
}
42 changes: 25 additions & 17 deletions src/main/java/com/qubole/presto/udfs/sqlFunction/hiveUdfs/Hash.java
View file
Expand Up @@ -15,14 +15,7 @@
*/
package com.qubole.presto.udfs.sqlFunction.hiveUdfs;

import com.facebook.presto.bytecode.BytecodeBlock;
import com.facebook.presto.bytecode.ClassDefinition;
import com.facebook.presto.bytecode.CompilerUtils;
import com.facebook.presto.bytecode.DynamicClassLoader;
import com.facebook.presto.bytecode.MethodDefinition;
import com.facebook.presto.bytecode.Parameter;
import com.facebook.presto.bytecode.Scope;
import com.facebook.presto.bytecode.Variable;
import com.facebook.presto.util.CompilerUtils;
import com.facebook.presto.metadata.BoundVariables;
import com.facebook.presto.metadata.FunctionKind;
import com.facebook.presto.metadata.FunctionRegistry;
Expand All @@ -39,27 +32,37 @@
import com.facebook.presto.type.BigintOperators;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import io.airlift.bytecode.BytecodeBlock;
import io.airlift.bytecode.ClassDefinition;
import io.airlift.bytecode.DynamicClassLoader;
import io.airlift.bytecode.MethodDefinition;
import io.airlift.bytecode.Parameter;
import io.airlift.bytecode.Scope;
import io.airlift.bytecode.Variable;
import io.airlift.slice.Slice;

import java.lang.invoke.MethodHandle;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import static com.facebook.presto.bytecode.Access.FINAL;
import static com.facebook.presto.bytecode.Access.PRIVATE;
import static com.facebook.presto.bytecode.Access.PUBLIC;
import static com.facebook.presto.bytecode.Access.STATIC;
import static com.facebook.presto.bytecode.Access.a;
import static com.facebook.presto.bytecode.CompilerUtils.defineClass;
import static com.facebook.presto.bytecode.Parameter.arg;
import static com.facebook.presto.bytecode.ParameterizedType.type;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE;
import static com.facebook.presto.util.CompilerUtils.defineClass;
import static com.facebook.presto.metadata.Signature.typeVariable;
import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
import static com.facebook.presto.sql.gen.SqlTypeBytecodeExpression.constantType;
import static com.facebook.presto.util.Reflection.methodHandle;
import static io.airlift.bytecode.Access.FINAL;
import static io.airlift.bytecode.Access.PRIVATE;
import static io.airlift.bytecode.Access.PUBLIC;
import static io.airlift.bytecode.Access.STATIC;
import static io.airlift.bytecode.Access.a;
import static io.airlift.bytecode.Parameter.arg;
import static io.airlift.bytecode.ParameterizedType.type;
import static java.lang.String.format;

public final class Hash
Expand Down Expand Up @@ -120,7 +123,12 @@ public ScalarFunctionImplementation specialize(BoundVariables types, int arity,
MethodHandle methodHandle = methodHandle(clazz, "hash", stackTypes.toArray(new Class<?>[stackTypes.size()]));
List<Boolean> nullableParameters = ImmutableList.copyOf(Collections.nCopies(stackTypes.size(), false));

return new ScalarFunctionImplementation(false, nullableParameters, methodHandle, isDeterministic());
List<ScalarFunctionImplementation.ArgumentProperty> argumentProperties = nullableParameters.stream()
.map(nullable -> nullable
? valueTypeArgumentProperty(USE_BOXED_TYPE)
: valueTypeArgumentProperty(RETURN_NULL_ON_NULL))
.collect(Collectors.toList());
return new ScalarFunctionImplementation(false, argumentProperties, methodHandle, isDeterministic());
}

public static Class<?> generateHash(List<Class<?>> nativeContainerTypes, Type type)
Expand Down
36 changes: 19 additions & 17 deletions src/main/java/com/qubole/presto/udfs/sqlFunction/hiveUdfs/Nvl.java
View file
Expand Up @@ -16,14 +16,6 @@
package com.qubole.presto.udfs.sqlFunction.hiveUdfs;

import com.facebook.presto.annotation.UsedByGeneratedCode;
import com.facebook.presto.bytecode.BytecodeBlock;
import com.facebook.presto.bytecode.ClassDefinition;
import com.facebook.presto.bytecode.CompilerUtils;
import com.facebook.presto.bytecode.DynamicClassLoader;
import com.facebook.presto.bytecode.MethodDefinition;
import com.facebook.presto.bytecode.Parameter;
import com.facebook.presto.bytecode.Scope;
import com.facebook.presto.bytecode.control.IfStatement;
import com.facebook.presto.metadata.BoundVariables;
import com.facebook.presto.metadata.FunctionKind;
import com.facebook.presto.metadata.FunctionRegistry;
Expand All @@ -34,22 +26,32 @@
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.TypeManager;
import com.facebook.presto.sql.gen.CallSiteBinder;
import com.facebook.presto.util.CompilerUtils;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import io.airlift.bytecode.BytecodeBlock;
import io.airlift.bytecode.ClassDefinition;
import io.airlift.bytecode.DynamicClassLoader;
import io.airlift.bytecode.MethodDefinition;
import io.airlift.bytecode.Parameter;
import io.airlift.bytecode.Scope;
import io.airlift.bytecode.control.IfStatement;
import io.airlift.slice.Slice;

import java.lang.invoke.MethodHandle;
import java.util.List;
import java.util.stream.Collectors;

import static com.facebook.presto.bytecode.Access.FINAL;
import static com.facebook.presto.bytecode.Access.PRIVATE;
import static com.facebook.presto.bytecode.Access.PUBLIC;
import static com.facebook.presto.bytecode.Access.STATIC;
import static com.facebook.presto.bytecode.Access.a;
import static com.facebook.presto.bytecode.CompilerUtils.defineClass;
import static com.facebook.presto.bytecode.Parameter.arg;
import static com.facebook.presto.bytecode.ParameterizedType.type;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty;
import static com.facebook.presto.operator.scalar.ScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE;
import static com.facebook.presto.util.CompilerUtils.defineClass;
import static io.airlift.bytecode.Access.FINAL;
import static io.airlift.bytecode.Access.PRIVATE;
import static io.airlift.bytecode.Access.PUBLIC;
import static io.airlift.bytecode.Access.STATIC;
import static io.airlift.bytecode.Access.a;
import static io.airlift.bytecode.Parameter.arg;
import static io.airlift.bytecode.ParameterizedType.type;
import static com.facebook.presto.metadata.Signature.typeVariable;
import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
Expand Down Expand Up @@ -130,7 +132,7 @@ else if (type.getJavaType() == boolean.class) {
Class<?> clazz = ifNull(stackTypes);
MethodHandle nvlMethodHandle = methodHandle(clazz, "nvl", stackTypes.toArray(new Class<?>[stackTypes.size()]));

return new ScalarFunctionImplementation(true, ImmutableList.of(true, true), nvlMethodHandle, isDeterministic());
return new ScalarFunctionImplementation(true, ImmutableList.of(valueTypeArgumentProperty(USE_BOXED_TYPE), valueTypeArgumentProperty(USE_BOXED_TYPE)), nvlMethodHandle, isDeterministic());
}

private Class<?> ifNull(List<Class<?>> nativeContainerTypes)
Expand Down

0 comments on commit 890b6ab

Please sign in to comment.