Implement VARCHAR(length)

1. Implement VARCHAR(length)

2. Add mini-language to calculate return type parameter
co-authored with Dain Sundstrom

3. Insert cast function even for type-only coercions

This patch changes the way optimization for not inserting cast calls for
type-type only coercions work.

Original code caused errors during bytecode preparation for plan execution.
Problematic case:

ARRAY('dog', 'kitten').
During semantic analysis this was treated as:
ARRAY((CAST 'dog' as VARCHAR(6)), 'kitten') and therefore
implementation of ArrayConstructor for parameter type VARCHAR(6) was
generated.

But cast expression was not added to result query expression, as
case VARCHAR(3) -> VARCHAR(6) is type-only cast.

Then byte generating code was failing as it was looking for
instantiation of ARRAY_CONSTRUCTOR(VARCHAR(3), VARCHAR(6)) despite the fact
only ARRAY_CONSTRUCTOR(VARCHAR(6), VARCHAR(6)) was generated.

Now we add Cast operator to AST. But it has typeOnly marker set to true.
And then when execution plan is generated no cast operation call is
generated.
co-authored with Lukas Osipiuk

4. Add explicit literalParameters

presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/MetadataUtil.java

@@ -53,7 +53,7 @@ public static final class TestingTypeDeserializer
     {
         private final Map<String, Type> types = ImmutableMap.<String, Type>of(
                 StandardTypes.BIGINT, BIGINT,
-                StandardTypes.VARCHAR, VARCHAR);
+                VARCHAR.getTypeSignature().toString(), VARCHAR); // for varchar(MAX_INT)
 
         public TestingTypeDeserializer()
         {

presto-example-http/src/test/java/com/facebook/presto/example/MetadataUtil.java

@@ -59,7 +59,8 @@ public static final class TestingTypeDeserializer
                 StandardTypes.BOOLEAN, BOOLEAN,
                 StandardTypes.BIGINT, BIGINT,
                 StandardTypes.DOUBLE, DOUBLE,
-                StandardTypes.VARCHAR, VARCHAR);
+                StandardTypes.VARCHAR, VARCHAR,
+                VARCHAR.getTypeSignature().toString(), VARCHAR); // varchar(MAX_INT)
 
         public TestingTypeDeserializer()
         {

presto-jdbc/src/main/java/com/facebook/presto/jdbc/ColumnInfo.java

@@ -174,7 +174,7 @@ private static int getType(TypeSignature type)
         if (type.getBase().equals("array")) {
             return Types.ARRAY;
         }
-        switch (type.toString()) {
+        switch (type.getBase()) {
             case "boolean":
                 return Types.BOOLEAN;
             case "bigint":

presto-jmx/src/test/java/com/facebook/presto/connector/jmx/MetadataUtil.java

@@ -51,7 +51,7 @@ public static final class TestingTypeDeserializer
     {
         private final Map<String, Type> types = ImmutableMap.<String, Type>builder()
                 .put(StandardTypes.BIGINT, BIGINT)
-                .put(StandardTypes.VARCHAR, VARCHAR)
+                .put(VARCHAR.toString(), VARCHAR) // with max value length in signature
                 .build();
 
         public TestingTypeDeserializer()

presto-main/pom.xml

@@ -247,6 +247,11 @@
             <artifactId>jgrapht-core</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.antlr</groupId>
+            <artifactId>antlr4-runtime</artifactId>
+        </dependency>
+
         <!-- for testing -->
         <dependency>
             <groupId>org.testng</groupId>
@@ -307,7 +312,10 @@
                     <failOnViolations>false</failOnViolations>
                 </configuration>
             </plugin>
-
+            <plugin>
+                <groupId>org.antlr</groupId>
+                <artifactId>antlr4-maven-plugin</artifactId>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>

presto-main/src/main/antlr4/com/facebook/presto/type/TypeCalculation.g4

@@ -0,0 +1,56 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//TODO: consider using the SQL grammar for this
+grammar TypeCalculation;
+
+typeCalculation
+    : expression
+    ;
+
+expression
+    : NULL                                                          #nullLiteral
+    | INTEGER_VALUE                                                 #numericLiteral
+    | IDENTIFIER                                                    #identifier
+    | '(' expression ')'                                            #parenthesizedExpression
+    | operator=(MINUS | PLUS) expression                            #arithmeticUnary
+    | left=expression operator=(ASTERISK | SLASH) right=expression  #arithmeticBinary
+    | left=expression operator=(PLUS | MINUS) right=expression      #arithmeticBinary
+    ;
+
+PLUS: '+';
+MINUS: '-';
+ASTERISK: '*';
+SLASH: '/';
+NULL: 'NULL';
+
+IDENTIFIER
+    : (LETTER | '_') (LETTER | DIGIT | '_' )*
+    ;
+
+INTEGER_VALUE
+    : DIGIT+
+    ;
+
+fragment DIGIT
+    : ('0'..'9')
+    ;
+
+fragment LETTER
+    : [A-Za-z]
+    ;
+
+WS
+    : [ \r\n\t]+ -> channel(HIDDEN)
+    ;

presto-main/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java

@@ -29,8 +29,8 @@
 import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.spi.type.TypeManager;
 import com.facebook.presto.spi.type.TypeSignature;
+import com.facebook.presto.type.LiteralParameters;
 import com.facebook.presto.type.SqlType;
-import com.facebook.presto.util.ImmutableCollectors;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -57,7 +57,6 @@
 import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR;
 import static com.facebook.presto.spi.type.BigintType.BIGINT;
 import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
-import static com.facebook.presto.type.TypeUtils.resolveTypes;
 import static com.google.common.base.CaseFormat.LOWER_CAMEL;
 import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE;
 import static com.google.common.base.Preconditions.checkArgument;
@@ -68,7 +67,7 @@
 
 public class FunctionListBuilder
 {
-    private static final Set<Class<?>> NON_NULLABLE_ARGUMENT_TYPES = ImmutableSet.<Class<?>>of(
+    private static final Set<Class<?>> NON_NULLABLE_ARGUMENT_TYPES = ImmutableSet.of(
             long.class,
             double.class,
             boolean.class,
@@ -95,7 +94,14 @@ public FunctionListBuilder window(String name, Type returnType, List<? extends T
 
     public FunctionListBuilder window(String name, Class<? extends ValueWindowFunction> clazz, String typeVariable, String... argumentTypes)
     {
-        Signature signature = new Signature(name, WINDOW, ImmutableList.of(typeParameter(typeVariable)), typeVariable, ImmutableList.copyOf(argumentTypes), false);
+        Signature signature = new Signature(
+                name,
+                WINDOW,
+                ImmutableList.of(typeParameter(typeVariable)),
+                typeVariable,
+                ImmutableList.copyOf(argumentTypes),
+                false,
+                ImmutableSet.of());
         functions.add(new SqlWindowFunction(new ReflectionWindowFunctionSupplier<>(signature, clazz)));
         return this;
     }
@@ -124,20 +130,50 @@ public FunctionListBuilder aggregate(Class<?> aggregationDefinition)
         return this;
     }
 
-    public FunctionListBuilder scalar(Signature signature, MethodHandle function, Optional<MethodHandle> instanceFactory, boolean deterministic, String description, boolean hidden, boolean nullable, List<Boolean> nullableArguments)
+    public FunctionListBuilder scalar(
+            Signature signature,
+            MethodHandle function,
+            Optional<MethodHandle> instanceFactory,
+            boolean deterministic,
+            String description,
+            boolean hidden,
+            boolean nullable,
+            List<Boolean> nullableArguments,
+            Set<String> literalParameters)
     {
-        functions.add(SqlScalarFunction.create(signature, description, hidden, function, instanceFactory, deterministic, nullable, nullableArguments));
+        functions.add(SqlScalarFunction.create(
+                signature,
+                description,
+                hidden,
+                function,
+                instanceFactory,
+                deterministic,
+                nullable,
+                nullableArguments,
+                literalParameters));
         return this;
     }
 
-    private FunctionListBuilder operator(OperatorType operatorType, Type returnType, List<Type> parameterTypes, MethodHandle function, Optional<MethodHandle> instanceFactory, boolean nullable, List<Boolean> nullableArguments)
+    private FunctionListBuilder operator(
+            OperatorType operatorType,
+            TypeSignature returnType,
+            List<TypeSignature> argumentTypes,
+            MethodHandle function,
+            Optional<MethodHandle> instanceFactory,
+            boolean nullable,
+            List<Boolean> nullableArguments,
+            Set<String> literalParameters)
     {
-        TypeSignature returnTypeSignature = returnType.getTypeSignature();
-        List<TypeSignature> argumentTypes = parameterTypes.stream()
-                .map(Type::getTypeSignature)
-                .collect(ImmutableCollectors.toImmutableList());
-        operatorType.validateSignature(returnTypeSignature, argumentTypes);
-        functions.add(SqlOperator.create(operatorType, argumentTypes, returnTypeSignature, function, instanceFactory, nullable, nullableArguments));
+        operatorType.validateSignature(returnType, argumentTypes);
+        functions.add(SqlOperator.create(
+                operatorType,
+                argumentTypes,
+                returnType,
+                function,
+                instanceFactory,
+                nullable,
+                nullableArguments,
+                literalParameters));
         return this;
     }
 
@@ -188,16 +224,42 @@ private boolean processScalarFunction(Method method)
         }
         SqlType returnTypeAnnotation = method.getAnnotation(SqlType.class);
         checkArgument(returnTypeAnnotation != null, "Method %s return type does not have a @SqlType annotation", method);
-        Type returnType = type(typeManager, returnTypeAnnotation);
-        Signature signature = new Signature(name.toLowerCase(ENGLISH), SCALAR, returnType.getTypeSignature(), Lists.transform(parameterTypes(typeManager, method), Type::getTypeSignature));
+        LiteralParameters literalParametersAnnotation = method.getAnnotation(LiteralParameters.class);
+        Set<String> literalParameters = ImmutableSet.of();
+        if (literalParametersAnnotation != null) {
+            literalParameters = ImmutableSet.copyOf(literalParametersAnnotation.value());
+        }
+
+        Signature signature = new Signature(
+                name.toLowerCase(ENGLISH),
+                SCALAR,
+                parseTypeSignature(returnTypeAnnotation.value(), literalParameters),
+                parameterTypeSignatures(method, literalParameters));
 
         verifyMethodSignature(method, signature.getReturnType(), signature.getArgumentTypes(), typeManager);
 
         List<Boolean> nullableArguments = getNullableArguments(method);
 
-        scalar(signature, methodHandle, instanceFactory, scalarFunction.deterministic(), getDescription(method), scalarFunction.hidden(), method.isAnnotationPresent(Nullable.class), nullableArguments);
+        scalar(
+                signature,
+                methodHandle,
+                instanceFactory,
+                scalarFunction.deterministic(),
+                getDescription(method),
+                scalarFunction.hidden(),
+                method.isAnnotationPresent(Nullable.class),
+                nullableArguments,
+                literalParameters);
         for (String alias : scalarFunction.alias()) {
-            scalar(signature.withAlias(alias.toLowerCase(ENGLISH)), methodHandle, instanceFactory, scalarFunction.deterministic(), getDescription(method), scalarFunction.hidden(), method.isAnnotationPresent(Nullable.class), nullableArguments);
+            scalar(signature.withAlias(alias.toLowerCase(ENGLISH)),
+                    methodHandle,
+                    instanceFactory,
+                    scalarFunction.deterministic(),
+                    getDescription(method),
+                    scalarFunction.hidden(),
+                    method.isAnnotationPresent(Nullable.class),
+                    nullableArguments,
+                    literalParameters);
         }
         return true;
     }
@@ -224,11 +286,11 @@ private static Type type(TypeManager typeManager, SqlType explicitType)
         return type;
     }
 
-    private static List<Type> parameterTypes(TypeManager typeManager, Method method)
+    private static List<TypeSignature> parameterTypeSignatures(Method method, Set<String> literalParameters)
     {
         Annotation[][] parameterAnnotations = method.getParameterAnnotations();
 
-        ImmutableList.Builder<Type> types = ImmutableList.builder();
+        ImmutableList.Builder<TypeSignature> types = ImmutableList.builder();
         for (int i = 0; i < method.getParameterTypes().length; i++) {
             Class<?> clazz = method.getParameterTypes()[i];
             // skip session parameters
@@ -245,21 +307,23 @@ private static List<Type> parameterTypes(TypeManager typeManager, Method method)
                 }
             }
             checkArgument(explicitType != null, "Method %s argument %s does not have a @SqlType annotation", method, i);
-            types.add(type(typeManager, explicitType));
+            types.add(parseTypeSignature(explicitType.value(), literalParameters));
         }
         return types.build();
     }
 
     private static void verifyMethodSignature(Method method, TypeSignature returnTypeName, List<TypeSignature> argumentTypeNames, TypeManager typeManager)
     {
-        Type returnType = typeManager.getType(returnTypeName);
-        requireNonNull(returnType, "returnType is null");
-        List<Type> argumentTypes = resolveTypes(argumentTypeNames, typeManager);
-        checkArgument(Primitives.unwrap(method.getReturnType()) == returnType.getJavaType(),
-                "Expected method %s return type to be %s (%s)",
-                method,
-                returnType.getJavaType().getName(),
-                returnType);
+        // todo figure out how to validate java type for calculated SQL type
+        if (!returnTypeName.isCalculated()) {
+            Type returnType = typeManager.getType(returnTypeName);
+            requireNonNull(returnType, "returnType is null");
+            checkArgument(Primitives.unwrap(method.getReturnType()) == returnType.getJavaType(),
+                    "Expected method %s return type to be %s (%s)",
+                    method,
+                    returnType.getJavaType().getName(),
+                    returnType);
+        }
 
         // skip Session argument
         Class<?>[] parameterTypes = method.getParameterTypes();
@@ -269,9 +333,13 @@ private static void verifyMethodSignature(Method method, TypeSignature returnTyp
             annotations = Arrays.copyOfRange(annotations, 1, annotations.length);
         }
 
-        for (int i = 0; i < parameterTypes.length; i++) {
+        for (int i = 0; i < argumentTypeNames.size(); i++) {
+            TypeSignature expectedTypeName = argumentTypeNames.get(i);
+            if (expectedTypeName.isCalculated()) {
+                continue;
+            }
+            Type expectedType = typeManager.getType(expectedTypeName);
             Class<?> actualType = parameterTypes[i];
-            Type expectedType = argumentTypes.get(i);
             boolean nullable = Arrays.asList(annotations[i]).stream().anyMatch(Nullable.class::isInstance);
             // Only allow boxing for functions that need to see nulls
             if (Primitives.isWrapperType(actualType)) {
@@ -323,24 +391,37 @@ private boolean processScalarOperator(Method method)
         MethodHandle methodHandle = lookup().unreflect(method);
         OperatorType operatorType = scalarOperator.value();
 
-        List<Type> parameterTypes = parameterTypes(typeManager, method);
+        LiteralParameters literalParametersAnnotation = method.getAnnotation(LiteralParameters.class);
+        Set<String> literalParameters = ImmutableSet.of();
+        if (literalParametersAnnotation != null) {
+            literalParameters = ImmutableSet.copyOf(literalParametersAnnotation.value());
+        }
+
+        List<TypeSignature> argumentTypes = parameterTypeSignatures(method, literalParameters);
+        TypeSignature returnTypeSignature;
 
-        Type returnType;
         if (operatorType == OperatorType.HASH_CODE) {
             // todo hack for hashCode... should be int
-            returnType = BIGINT;
+            returnTypeSignature = BIGINT.getTypeSignature();
         }
         else {
             SqlType explicitType = method.getAnnotation(SqlType.class);
             checkArgument(explicitType != null, "Method %s return type does not have a @SqlType annotation", method);
-            returnType = type(typeManager, explicitType);
-
-            verifyMethodSignature(method, returnType.getTypeSignature(), Lists.transform(parameterTypes, Type::getTypeSignature), typeManager);
+            returnTypeSignature = parseTypeSignature(explicitType.value(), literalParameters);
+            verifyMethodSignature(method, returnTypeSignature, argumentTypes, typeManager);
         }
 
         List<Boolean> nullableArguments = getNullableArguments(method);
 
-        operator(operatorType, returnType, parameterTypes, methodHandle, instanceFactory, method.isAnnotationPresent(Nullable.class), nullableArguments);
+        operator(
+                operatorType,
+                returnTypeSignature,
+                argumentTypes,
+                methodHandle,
+                instanceFactory,
+                method.isAnnotationPresent(Nullable.class),
+                nullableArguments,
+                literalParameters);
         return true;
     }
 
@@ -367,15 +448,6 @@ private static void checkValidMethod(Method method)
         }
     }
 
-    private static List<Class<?>> getParameterTypes(Class<?>... types)
-    {
-        ImmutableList<Class<?>> parameterTypes = ImmutableList.copyOf(types);
-        if (!parameterTypes.isEmpty() && parameterTypes.get(0) == ConnectorSession.class) {
-            parameterTypes = parameterTypes.subList(1, parameterTypes.size());
-        }
-        return parameterTypes;
-    }
-
     public List<SqlFunction> getFunctions()
     {
         return ImmutableList.copyOf(functions);