From ac8590d2b78b4b5f97199af48e425e8f4d205068 Mon Sep 17 00:00:00 2001 From: Jacques Nadeau Date: Sat, 10 Aug 2013 13:02:59 -0700 Subject: [PATCH] DRILL-166 Update CodeGenerator code to support more generic signatures. --- .../drill/common/expression/IfExpression.java | 29 +- .../common/expression/LogicalExpression.java | 2 +- .../drill/common/expression/SchemaPath.java | 7 + .../common/expression/ValueExpressions.java | 21 + .../expression/visitors/AggregateChecker.java | 2 +- .../expression/visitors/ConstantChecker.java | 3 +- sandbox/prototype/exec/java-exec/pom.xml | 10 + .../exec/compile/TemplateClassDefinition.java | 36 +- .../compile/sig/CodeGeneratorArgument.java | 23 + .../exec/compile/sig/CodeGeneratorMethod.java | 60 +++ .../compile/sig/CodeGeneratorSignature.java | 5 + .../sig/ConstantExpressionIdentifier.java | 114 +++++ .../sig/DefaultGeneratorSignature.java | 22 + .../exec/compile/sig/GeneratorMapping.java | 40 ++ .../drill/exec/compile/sig/MappingSet.java | 67 +++ .../drill/exec/compile/sig/Signature.java | 14 + .../exec/compile/sig/SignatureHolder.java | 57 +++ .../drill/exec/compile/sig/VVReadBatch.java | 18 + .../drill/exec/compile/sig/VVWriteBatch.java | 18 + .../apache/drill/exec/expr/CodeGenerator.java | 172 ++++---- .../drill/exec/expr/EvaluationVisitor.java | 412 ++++++++++++------ .../exec/expr/ExpressionTreeMaterializer.java | 2 +- .../exec/expr/HoldingContainerExpression.java | 46 ++ .../exec/expr/ValueVectorReadExpression.java | 9 + .../exec/expr/ValueVectorWriteExpression.java | 9 + .../exec/expr/annotations/MethodMap.java | 21 + .../exec/expr/fn/DrillAggFuncHolder.java | 36 ++ .../drill/exec/expr/fn/DrillFuncHolder.java | 102 +++++ .../drill/exec/expr/fn/FunctionBody.java | 12 + .../drill/exec/expr/fn/FunctionConverter.java | 14 +- .../drill/exec/expr/fn/FunctionHolder.java | 110 +---- .../drill/exec/ops/FragmentContext.java | 8 - .../physical/impl/filter/FilterSignature.java | 14 + .../exec/physical/impl/filter/Filterer.java | 3 +- .../impl/filter/ReturnValueExpression.java | 10 +- .../PartitionSenderRootExec.java | 11 +- .../impl/partitionsender/Partitioner.java | 2 +- .../PartitionerInnerSignature.java | 17 + .../impl/project/ProjectEvaluator.java | 2 +- .../exec/physical/impl/project/Projector.java | 3 +- .../physical/impl/sort/ReadIndexRewriter.java | 2 +- .../exec/physical/impl/sort/SortBatch.java | 19 +- .../physical/impl/sort/SortSignature.java | 20 + .../exec/physical/impl/sort/SortTemplate.java | 8 +- .../drill/exec/physical/impl/sort/Sorter.java | 2 +- .../exec/physical/impl/svremover/Copier.java | 6 +- .../impl/svremover/RemovingRecordBatch.java | 4 +- .../drill/exec/record/NullExpression.java | 13 +- .../exec/compile/TestClassTransformation.java | 2 +- .../impl/TestHashToRandomExchange.java | 13 +- .../ExpressionTreeMaterializerTest.java | 58 +-- 51 files changed, 1302 insertions(+), 408 deletions(-) create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorArgument.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorSignature.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/Signature.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVReadBatch.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVWriteBatch.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HoldingContainerExpression.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/MethodMap.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionBody.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java create mode 100644 sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/IfExpression.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/IfExpression.java index 6f2e4a887ec..426aa6a910a 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/IfExpression.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/IfExpression.java @@ -18,6 +18,7 @@ package org.apache.drill.common.expression; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.apache.drill.common.expression.IfExpression.IfCondition; @@ -28,9 +29,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; import com.google.common.collect.UnmodifiableIterator; -public class IfExpression extends LogicalExpressionBase implements Iterable{ +public class IfExpression extends LogicalExpressionBase{ static final Logger logger = LoggerFactory.getLogger(IfExpression.class); public final ImmutableList conditions; @@ -106,9 +110,22 @@ public static Builder newBuilder(){ } - @Override - public UnmodifiableIterator iterator() { - return conditions.iterator(); - } - + public Iterable conditionIterable(){ + + return ImmutableList.copyOf(conditions); + } + + @Override + public Iterator iterator() { + List children = Lists.newLinkedList(); + + for(IfCondition ic : conditions){ + children.add(ic.condition); + children.add(ic.expression); + } + children.add(this.elseExpression); + return children.iterator(); + } + + } diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java index 165f4096914..ac19e01e0a7 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java @@ -43,7 +43,7 @@ //@JsonDeserialize(using = LogicalExpression.De.class) // Excluded as we need to register this with the DrillConfig. @JsonSerialize(using = LogicalExpression.Se.class) -public interface LogicalExpression { +public interface LogicalExpression extends Iterable{ static final Logger logger = LoggerFactory.getLogger(LogicalExpression.class); public abstract MajorType getMajorType(); diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java index 6f1a733ec4f..5910c30787d 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java @@ -17,6 +17,7 @@ ******************************************************************************/ package org.apache.drill.common.expression; +import java.util.Iterator; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -26,6 +27,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.Types; +import com.google.common.collect.Iterators; import com.google.protobuf.DescriptorProtos.UninterpretedOption.NamePart; public class SchemaPath extends LogicalExpressionBase { @@ -131,6 +133,11 @@ public boolean equals(Object obj) { return true; } + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + @Override public String toString() { return "SchemaPath [rootSegment=" + rootSegment + "]"; diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java index d613ebe36a5..33c4612c0f4 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java @@ -17,12 +17,16 @@ ******************************************************************************/ package org.apache.drill.common.expression; +import java.util.Iterator; + import org.apache.drill.common.expression.visitors.ExprVisitor; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; +import com.google.common.collect.Iterators; + public class ValueExpressions { public static LogicalExpression getNumericExpression(String s, ExpressionPosition ep) { @@ -54,6 +58,12 @@ protected ValueExpression(String value, ExpressionPosition pos) { } protected abstract V parseValue(String s); + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + } @@ -109,6 +119,11 @@ public T accept(ExprVisitor visitor, V valu return visitor.visitDoubleConstant(this, value); } + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + } public static class LongExpression extends LogicalExpressionBase { @@ -135,6 +150,12 @@ public MajorType getMajorType() { public T accept(ExprVisitor visitor, V value) throws E { return visitor.visitLongConstant(this, value); } + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + } public static class QuotedString extends ValueExpression { diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java index 4d99b151c1b..aa442818260 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java @@ -54,7 +54,7 @@ public Boolean visitFunctionCall(FunctionCall call) { @Override public Boolean visitIfExpression(IfExpression ifExpr) { - for(IfCondition c : ifExpr){ + for(IfCondition c : ifExpr.conditions){ if(c.condition.accept(this, null) || c.expression.accept(this, null)) return true; } return ifExpr.elseExpression.accept(this, null); diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java index 038fb85907d..f8984e9c6b5 100644 --- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java +++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java @@ -47,9 +47,10 @@ public Boolean visitFunctionCall(FunctionCall call) { @Override public Boolean visitIfExpression(IfExpression ifExpr) { - for(IfCondition c : ifExpr){ + for(IfCondition c : ifExpr.conditions){ if(!c.condition.accept(this, null) || !c.expression.accept(this, null)) return false; } + if(!ifExpr.elseExpression.accept(this, null)) return false; return true; } diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml index a1680e831e8..e7d3f16fb56 100644 --- a/sandbox/prototype/exec/java-exec/pom.xml +++ b/sandbox/prototype/exec/java-exec/pom.xml @@ -22,6 +22,11 @@ asm-commons 4.1 + + com.thoughtworks.paranamer + paranamer + 2.5.6 + com.sun.codemodel codemodel @@ -58,6 +63,11 @@ metrics-core 3.0.0-BETA1 + + javax.inject + javax.inject + 1 + org.apache.drill common diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java index 20ef3610625..bd238443378 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/TemplateClassDefinition.java @@ -17,21 +17,36 @@ ******************************************************************************/ package org.apache.drill.exec.compile; - +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; +import org.apache.drill.exec.compile.sig.SignatureHolder; public class TemplateClassDefinition{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TemplateClassDefinition.class); + private final Class externalInterface; private final String templateClassName; private final Class internalInterface; - private final Class evalReturnType; - - public TemplateClassDefinition(Class externalInterface, String templateClassName, Class internalInterface, Class evalReturnType) { + private final SignatureHolder signature; + + public TemplateClassDefinition(Class externalInterface, String templateClassName, Class internalInterface) { + this(externalInterface, templateClassName, internalInterface, DefaultGeneratorSignature.class); + } + + public TemplateClassDefinition(Class externalInterface, String templateClassName, Class internalInterface, Class signature) { super(); this.externalInterface = externalInterface; this.templateClassName = templateClassName; this.internalInterface = internalInterface; - this.evalReturnType = evalReturnType; + SignatureHolder holder = null; + try{ + holder = new SignatureHolder(signature); + }catch(Exception ex){ + logger.error("Failure while trying to build signature holder for signature. {}", signature.getSimpleName(), ex); + } + this.signature = holder; + } public Class getExternalInterface() { @@ -46,8 +61,15 @@ public String getTemplateClassName() { return templateClassName; } - public Class getEvalReturnType() { - return evalReturnType; + public SignatureHolder getSignature(){ + return signature; } + + @Override + public String toString() { + return "TemplateClassDefinition [externalInterface=" + externalInterface + ", templateClassName=" + + templateClassName + ", internalInterface=" + internalInterface + ", signature=" + signature + "]"; + } + } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorArgument.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorArgument.java new file mode 100644 index 00000000000..f1715b20cac --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorArgument.java @@ -0,0 +1,23 @@ +package org.apache.drill.exec.compile.sig; + + +public class CodeGeneratorArgument { + + private final String name; + private final Class type; + + public CodeGeneratorArgument(String name, Class type) { + super(); + this.name = name; + this.type = type; + } + + public String getName() { + return name; + } + + public Class getType() { + return type; + } + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java new file mode 100644 index 00000000000..11dcbe90daa --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorMethod.java @@ -0,0 +1,60 @@ +package org.apache.drill.exec.compile.sig; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Iterator; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterators; +import com.thoughtworks.paranamer.AnnotationParanamer; +import com.thoughtworks.paranamer.Paranamer; + +public class CodeGeneratorMethod implements Iterable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGeneratorMethod.class); + + private final String methodName; + private final Class returnType; + private final CodeGeneratorArgument[] arguments; + private final Class[] exs; + private final Method underlyingMethod; + + public CodeGeneratorMethod(Method m){ + this.underlyingMethod = m; + this.methodName = m.getName(); + this.returnType = m.getReturnType(); +// Paranamer para = new BytecodeReadingParanamer(); + Paranamer para = new AnnotationParanamer(); + String[] parameterNames = para.lookupParameterNames(m, true); + if(parameterNames == null) throw new RuntimeException(String.format("Unable to read the parameter names for method %s. This is likely due to the class files not including the appropriate debugging information. Look up java -g for more information.", m)); + Class[] types = m.getParameterTypes(); + if(parameterNames.length != types.length) throw new RuntimeException(String.format("Unexpected number of parameter names %s. Expected %s on method %s.", Arrays.toString(parameterNames), Arrays.toString(types), m.toGenericString())); + arguments = new CodeGeneratorArgument[parameterNames.length]; + for(int i =0 ; i < parameterNames.length; i++){ + arguments[i] = new CodeGeneratorArgument(parameterNames[i], types[i]); + } + exs = m.getExceptionTypes(); + } + + public String getMethodName() { + return methodName; + } + public Class getReturnType() { + return returnType; + } + + public Iterable> getThrowsIterable(){ + return ImmutableList.copyOf(exs); + } + + @Override + public Iterator iterator() { + return Iterators.forArray(arguments); + } + + @Override + public String toString() { + return "CodeGeneratorMethod [" + underlyingMethod.toGenericString() + "]"; + } + + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorSignature.java new file mode 100644 index 00000000000..f294fe7e9f2 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/CodeGeneratorSignature.java @@ -0,0 +1,5 @@ +package org.apache.drill.exec.compile.sig; + + +public interface CodeGeneratorSignature { +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java new file mode 100644 index 00000000000..62e6837a85d --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java @@ -0,0 +1,114 @@ +package org.apache.drill.exec.compile.sig; + +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Set; + +import org.apache.drill.common.expression.FunctionCall; +import org.apache.drill.common.expression.IfExpression; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; +import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; +import org.apache.drill.common.expression.ValueExpressions.LongExpression; +import org.apache.drill.common.expression.ValueExpressions.QuotedString; +import org.apache.drill.common.expression.visitors.ExprVisitor; + +import com.beust.jcommander.internal.Lists; + +public class ConstantExpressionIdentifier implements ExprVisitor, RuntimeException>{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConstantExpressionIdentifier.class); + + private ConstantExpressionIdentifier(){} + + /** + * Get a list of expressions that mark boundaries into a constant space. + * @param e + * @return + */ + public static Set getConstantExpressionSet(LogicalExpression e){ + IdentityHashMap map = new IdentityHashMap<>(); + ConstantExpressionIdentifier visitor = new ConstantExpressionIdentifier(); + + + if(e.accept(visitor, map) && map.isEmpty()){ + // if we receive a constant value here but the map is empty, this means the entire tree is a constant. + // note, we can't use a singleton collection here because we need an identity set. + map.put(e, true); + return map.keySet(); + }else if(map.isEmpty()){ + // so we don't continue to carry around a map, we let it go here and simply return an empty set. + return Collections.emptySet(); + }else{ + return map.keySet(); + } + } + + private boolean checkChildren(LogicalExpression e, IdentityHashMap value, boolean transmitsConstant){ + List constants = Lists.newLinkedList(); + boolean constant = true; + + for(LogicalExpression child : e){ + if(child.accept(this, value)){ + constants.add(child); + }else{ + constant = false; + } + } + + // if one or more clauses isn't constant, this isn't constant. this also isn't a constant if it operates on a set. + if(!constant || !transmitsConstant){ + for(LogicalExpression c: constants){ + value.put(c, true); + } + } + return constant && transmitsConstant; + } + + @Override + public Boolean visitFunctionCall(FunctionCall call, IdentityHashMap value){ + return checkChildren(call, value, !call.getDefinition().isAggregating()); + } + + + @Override + public Boolean visitIfExpression(IfExpression ifExpr, IdentityHashMap value){ + return checkChildren(ifExpr, value, true); + } + + @Override + public Boolean visitSchemaPath(SchemaPath path, IdentityHashMap value){ + return false; + } + + @Override + public Boolean visitLongConstant(LongExpression intExpr, IdentityHashMap value){ + value.put(intExpr, true); + return true; + } + + @Override + public Boolean visitDoubleConstant(DoubleExpression dExpr, IdentityHashMap value){ + value.put(dExpr, true); + return true; + } + + @Override + public Boolean visitBooleanConstant(BooleanExpression e, IdentityHashMap value){ + value.put(e, true); + return true; + } + + @Override + public Boolean visitQuotedStringConstant(QuotedString e, IdentityHashMap value){ + value.put(e, true); + return true; + } + + @Override + public Boolean visitUnknown(LogicalExpression e, IdentityHashMap value){ + return checkChildren(e, value, false); + } + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java new file mode 100644 index 00000000000..dc56206d6db --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/DefaultGeneratorSignature.java @@ -0,0 +1,22 @@ +package org.apache.drill.exec.compile.sig; + +import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM; + +import javax.inject.Named; + +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface DefaultGeneratorSignature extends CodeGeneratorSignature{ + + public static final GeneratorMapping DEFAULT_SCALAR_MAP = GM("doSetup", "doEval", null, null); + public static final GeneratorMapping DEFAULT_CONSTANT_MAP = GM("doSetup", "doSetup", null, null); + + public static final MappingSet DEFAULT_MAPPING = new MappingSet("inIndex", "outIndex", DEFAULT_SCALAR_MAP, DEFAULT_SCALAR_MAP); + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing); + public void doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex); + + + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java new file mode 100644 index 00000000000..06334469de2 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java @@ -0,0 +1,40 @@ +package org.apache.drill.exec.compile.sig; + +public class GeneratorMapping { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GeneratorMapping.class); + + private String setup; + private String eval; + private String reset; + private String cleanup; + + public GeneratorMapping(String setup, String eval, String reset, String cleanup) { + super(); + this.setup = setup; + this.eval = eval; + this.reset = reset; + this.cleanup = cleanup; + } + + public static GeneratorMapping GM(String setup, String eval, String reset, String cleanup){ + return new GeneratorMapping(setup, eval, reset, cleanup); + } + + public String getSetup() { + return setup; + } + + public String getEval() { + return eval; + } + + public String getReset() { + return reset; + } + + public String getCleanup() { + return cleanup; + } + + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java new file mode 100644 index 00000000000..e6dcf4e031c --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java @@ -0,0 +1,67 @@ +package org.apache.drill.exec.compile.sig; + +import java.util.Arrays; + +import com.google.common.base.Preconditions; +import com.sun.codemodel.JExpr; +import com.sun.codemodel.JExpression; + + +public class MappingSet { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MappingSet.class); + + private GeneratorMapping constant; + private GeneratorMapping[] mappings; + private int mappingIndex; + private GeneratorMapping current; + private JExpression readIndex; + private JExpression writeIndex; + + + public MappingSet(String readIndex, String writeIndex, GeneratorMapping... mappings) { + this.readIndex = JExpr.direct(readIndex); + this.writeIndex = JExpr.direct(writeIndex); + Preconditions.checkArgument(mappings.length >= 2); + this.constant = mappings[0]; + this.mappings = Arrays.copyOfRange(mappings, 1, mappings.length); + this.current = this.mappings[0]; + } + + public void enterConstant(){ +// assert constant != current; +// current = constant; + } + + public void exitConstant(){ +// assert constant == current; +// current = mappings[mappingIndex]; + } + + + public void enterChild(){ + assert current == mappings[mappingIndex]; + mappingIndex++; + if(mappingIndex >= mappings.length) throw new IllegalStateException("This generator does not support mappings beyond"); + current = mappings[mappingIndex]; + } + + public void exitChild(){ + assert current == mappings[mappingIndex]; + mappingIndex--; + if(mappingIndex < 0) throw new IllegalStateException("You tried to traverse higher than the provided mapping provides."); + } + + public GeneratorMapping getCurrentMapping(){ + return current; + } + + public JExpression getValueWriteIndex(){ + return writeIndex; + } + + public JExpression getValueReadIndex(){ + return readIndex; + } + + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/Signature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/Signature.java new file mode 100644 index 00000000000..ea49d967b3e --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/Signature.java @@ -0,0 +1,14 @@ +package org.apache.drill.exec.compile.sig; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Describes the field will provide output from the given function. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface Signature { +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java new file mode 100644 index 00000000000..68b24980f06 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/SignatureHolder.java @@ -0,0 +1,57 @@ +package org.apache.drill.exec.compile.sig; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; + +import com.beust.jcommander.internal.Maps; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; + +public class SignatureHolder implements Iterable{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SignatureHolder.class); + + private final CodeGeneratorMethod[] methods; + private final Map methodMap; + + public SignatureHolder(Class signature){ + Method[] reflectMethods = signature.getDeclaredMethods(); + methods = new CodeGeneratorMethod[reflectMethods.length]; + Map newMap = Maps.newHashMap(); + + for(int i =0; i < methods.length; i++){ + methods[i] = new CodeGeneratorMethod(reflectMethods[i]); + newMap.put(methods[i].getMethodName(), i); + } + + methodMap = ImmutableMap.copyOf(newMap); + + } + + @Override + public Iterator iterator() { + return Iterators.forArray(methods); + } + + public int size(){ + return methods.length; + } + + public int get(String method){ + Integer meth = methodMap.get(method); + if(meth == null){ + throw new IllegalStateException(String.format("Unknown method requested of name %s.", method)); + } + return meth; + } + + @Override + public String toString() { + final int maxLen = 10; + return "SignatureHolder [methods=" + + (methods != null ? Arrays.asList(methods).subList(0, Math.min(methods.length, maxLen)) : null) + "]"; + } + + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVReadBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVReadBatch.java new file mode 100644 index 00000000000..c20c6d82262 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVReadBatch.java @@ -0,0 +1,18 @@ +package org.apache.drill.exec.compile.sig; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; + +/** + * Marker annotation to determine which fields should be included as parameters for the function. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface VVReadBatch { + int value(); +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVWriteBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVWriteBatch.java new file mode 100644 index 00000000000..be07c6dc3a2 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/VVWriteBatch.java @@ -0,0 +1,18 @@ +package org.apache.drill.exec.compile.sig; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; + +/** + * Marker annotation to determine which fields should be included as parameters for the function. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface VVWriteBatch { + int value(); +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java index c37e75dab84..d2e4cf3e76e 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java @@ -8,13 +8,15 @@ import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.compile.TemplateClassDefinition; +import org.apache.drill.exec.compile.sig.CodeGeneratorArgument; +import org.apache.drill.exec.compile.sig.CodeGeneratorMethod; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; +import org.apache.drill.exec.compile.sig.GeneratorMapping; +import org.apache.drill.exec.compile.sig.MappingSet; +import org.apache.drill.exec.compile.sig.SignatureHolder; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.physical.impl.partitionsender.OutgoingRecordBatch; -import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.TypedFieldId; -import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.vector.TypeHelper; import com.google.common.base.Preconditions; @@ -24,22 +26,19 @@ import com.sun.codemodel.JCodeModel; import com.sun.codemodel.JDefinedClass; import com.sun.codemodel.JExpr; -import com.sun.codemodel.JExpression; import com.sun.codemodel.JFieldRef; import com.sun.codemodel.JMethod; import com.sun.codemodel.JMod; import com.sun.codemodel.JType; import com.sun.codemodel.JVar; -public class CodeGenerator { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGenerator.class); +public class CodeGenerator{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CodeGenerator.class); + public JDefinedClass clazz; - private JBlock parentEvalBlock; - private JBlock parentSetupBlock; - private JBlock currentEvalBlock; - private JBlock currentSetupBlock; - private final EvaluationVisitor evaluationVisitor; + private JBlock[] parentBlocks; + private JBlock[] childBlocks; private final TemplateClassDefinition definition; private JCodeModel model; private int index = 0; @@ -47,25 +46,69 @@ public class CodeGenerator { private String className; private String fqcn; private String packageName = "org.apache.drill.exec.test.generated"; + private final SignatureHolder sig; + private MappingSet mappings; + private final EvaluationVisitor evaluationVisitor; + public CodeGenerator(TemplateClassDefinition definition, FunctionImplementationRegistry funcRegistry) { - super(); - className = "Gen" + classCreator.incrementAndGet(); - fqcn = packageName + "." + className; + this(DefaultGeneratorSignature.DEFAULT_MAPPING, definition, funcRegistry); + } + + public CodeGenerator(MappingSet mappingSet, TemplateClassDefinition definition, FunctionImplementationRegistry funcRegistry) { + Preconditions.checkNotNull(definition.getSignature(), "The signature for defintion %s was incorrectly initialized.", definition); + this.sig = definition.getSignature(); + this.mappings = mappingSet; + this.className = "Gen" + classCreator.incrementAndGet(); + this.fqcn = packageName + "." + className; try{ this.definition = definition; this.model = new JCodeModel(); this.clazz = model._package(packageName)._class(className); clazz._implements(definition.getInternalInterface()); - this.parentEvalBlock = new JBlock(); - this.parentSetupBlock = new JBlock(); + parentBlocks = new JBlock[sig.size()]; + for(int i =0; i < sig.size(); i++){ + parentBlocks[i] = new JBlock(false, false); + } + childBlocks = new JBlock[sig.size()]; this.evaluationVisitor = new EvaluationVisitor(funcRegistry); + rotateBlock(); } catch (JClassAlreadyExistsException e) { throw new IllegalStateException(e); } } - + public MappingSet getMappingSet(){ + return mappings; + } + + public void setMappingSet(MappingSet mappings){ + this.mappings = mappings; + } + + private GeneratorMapping getCurrentMapping(){ + return mappings.getCurrentMapping(); + } + + private JBlock getBlock(String methodName){ + JBlock blk = this.childBlocks[sig.get(methodName)]; + Preconditions.checkNotNull(blk, "Requested method name of %s was not available for signature %s.", methodName, this.sig); + return blk; + } + + public JBlock getSetupBlock(){ + return getBlock(getCurrentMapping().getSetup()); + } + public JBlock getEvalBlock(){ + return getBlock(getCurrentMapping().getEval()); + } +// public JBlock getResetBlock(){ +// return getBlock(getCurrentMapping().getReset()); +// } +// public JBlock getCleanupBlock(){ +// return getBlock(getCurrentMapping().getCleanup()); +// } + public JVar declareVectorValueSetupAndMember(String batchName, TypedFieldId fieldId){ Class valueVectorClass = TypeHelper.getValueVectorClass(fieldId.getType().getMinorType(), fieldId.getType().getMode()); JClass vvClass = model.ref(valueVectorClass); @@ -78,7 +121,6 @@ public JVar declareVectorValueSetupAndMember(String batchName, TypedFieldId fiel JVar vv = declareClassField("vv", retClass); JClass t = model.ref(SchemaChangeException.class); - JType wrapperClass = model.ref(VectorWrapper.class); JType objClass = model.ref(Object.class); JBlock b = getSetupBlock(); JVar obj = b.decl( // @@ -99,87 +141,51 @@ public JVar declareVectorValueSetupAndMember(String batchName, TypedFieldId fiel } public HoldingContainer addExpr(LogicalExpression ex){ + return addExpr(ex, true); + } + + public HoldingContainer addExpr(LogicalExpression ex, boolean rotate){ logger.debug("Adding next write {}", ex); - rotateBlock(); - return ex.accept(evaluationVisitor, this); + if(rotate) rotateBlock(); + return evaluationVisitor.addExpr(ex, this); } public void rotateBlock(){ - currentEvalBlock = new JBlock(); - parentEvalBlock.add(currentEvalBlock); - currentSetupBlock = new JBlock(); - parentSetupBlock.add(currentSetupBlock); - + for(int i =0; i < childBlocks.length; i++){ + this.childBlocks[i] = new JBlock(true, true); + + this.parentBlocks[i].add(childBlocks[i]); + } } - public JBlock getBlock() { - return currentEvalBlock; - } - public String getMaterializedClassName(){ return fqcn; } - - public JBlock getSetupBlock(){ - return currentSetupBlock; - } - - + public TemplateClassDefinition getDefinition() { return definition; } public String generate() throws IOException{ + int i =0; + for(CodeGeneratorMethod method : sig){ + JMethod m = clazz.method(JMod.PUBLIC, model._ref(method.getReturnType()), method.getMethodName()); + for(CodeGeneratorArgument arg : method){ + m.param(arg.getType(), arg.getName()); + } + for(Class c : method.getThrowsIterable()){ + m._throws(model.ref(c)); + } - { - //setup method - JMethod m = clazz.method(JMod.PUBLIC, model.VOID, "doSetup"); - m.param(model._ref(FragmentContext.class), "context"); - m.param(model._ref(RecordBatch.class), "incoming"); - m.param(model._ref(RecordBatch.class), "outgoing"); m._throws(SchemaChangeException.class); - m.body().add(parentSetupBlock); - } - - { - // eval method. - JType ret = definition.getEvalReturnType() == null ? model.VOID : model._ref(definition.getEvalReturnType()); - JMethod m = clazz.method(JMod.PUBLIC, ret, "doEval"); - m.param(model.INT, "inIndex"); - m.param(model.INT, "outIndex"); - m.body().add(parentEvalBlock); + m.body().add(this.parentBlocks[i++]); } SingleClassStringWriter w = new SingleClassStringWriter(); model.build(w); return w.getCode().toString(); } - - public String generateMultipleOutputs() throws IOException{ - - { - //setup method - JMethod m = clazz.method(JMod.PUBLIC, model.VOID, "doSetup"); - m.param(model._ref(FragmentContext.class), "context"); - m.param(model._ref(RecordBatch.class), "incoming"); - m.param(model._ref(OutgoingRecordBatch.class).array(), "outgoing"); - m._throws(SchemaChangeException.class); - m.body().add(parentSetupBlock); - } - - { - // eval method. - JType ret = definition.getEvalReturnType() == null ? model.VOID : model._ref(definition.getEvalReturnType()); - JMethod m = clazz.method(JMod.PUBLIC, ret, "doEval"); - m.param(model.INT, "inIndex"); - m.param(model.INT, "outIndex"); - m.body().add(parentEvalBlock); - } - - SingleClassStringWriter w = new SingleClassStringWriter(); - model.build(w); - return w.getCode().toString(); - } + public JCodeModel getModel() { return model; @@ -205,9 +211,9 @@ public HoldingContainer declare(MajorType t, boolean includeNewInstance){ JType holderType = getHolderType(t); JVar var; if(includeNewInstance){ - var = currentEvalBlock.decl(holderType, "out" + index, JExpr._new(holderType)); + var = getEvalBlock().decl(holderType, "out" + index, JExpr._new(holderType)); }else{ - var = currentEvalBlock.decl(holderType, "out" + index); + var = getEvalBlock().decl(holderType, "out" + index); } JFieldRef outputSet = null; if(t.getMode() == DataMode.OPTIONAL){ @@ -239,6 +245,10 @@ public JVar getHolder() { public JFieldRef getValue() { return value; } + + public MajorType getMajorType(){ + return type; + } public JFieldRef getIsSet() { Preconditions.checkNotNull(isSet, "You cannot access the isSet variable when operating on a non-nullable output value."); @@ -261,4 +271,4 @@ public TypeProtos.MinorType getMinorType() { public JType getHolderType(MajorType t){ return TypeHelper.getHolderType(model, t.getMinorType(), t.getMode()); } -} +} \ No newline at end of file diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java index 21106c1d3a9..0e66fd02a51 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java @@ -1,5 +1,7 @@ package org.apache.drill.exec.expr; +import java.util.Set; + import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.IfExpression; import org.apache.drill.common.expression.IfExpression.IfCondition; @@ -12,6 +14,7 @@ import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; +import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier; import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; import org.apache.drill.exec.expr.fn.FunctionHolder; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; @@ -24,190 +27,309 @@ import com.sun.codemodel.JInvocation; import com.sun.codemodel.JVar; -public class EvaluationVisitor extends AbstractExprVisitor, RuntimeException> { +public class EvaluationVisitor { - private FunctionImplementationRegistry registry; - + private final FunctionImplementationRegistry registry; public EvaluationVisitor(FunctionImplementationRegistry registry) { + super(); this.registry = registry; } - @Override - public HoldingContainer visitFunctionCall(FunctionCall call, CodeGenerator generator) throws RuntimeException { - HoldingContainer[] args = new HoldingContainer[call.args.size()]; - for(int i = 0; i < call.args.size(); i++){ - args[i] = call.args.get(i).accept(this, generator); - } - FunctionHolder holder = registry.getFunction(call); + public HoldingContainer addExpr(LogicalExpression e, CodeGenerator generator){ + return e.accept(new ConstantFilter(ConstantExpressionIdentifier.getConstantExpressionSet(e)), generator); - return holder.renderFunction(generator, args); } - - @Override - public HoldingContainer visitIfExpression(IfExpression ifExpr, CodeGenerator generator) throws RuntimeException { - JBlock local = generator.getBlock(); - - HoldingContainer output = generator.declare(ifExpr.getMajorType()); + + private class EvalVisitor extends AbstractExprVisitor, RuntimeException> { + - JConditional jc = null; - JBlock conditionalBlock = new JBlock(); - for (IfCondition c : ifExpr.conditions) { - HoldingContainer HoldingContainer = c.condition.accept(this, generator); - if (jc == null) { - if (HoldingContainer.isOptional()) { - jc = conditionalBlock._if(HoldingContainer.getIsSet().cand(HoldingContainer.getValue())); + @Override + public HoldingContainer visitFunctionCall(FunctionCall call, CodeGenerator generator) throws RuntimeException { + HoldingContainer[] args = new HoldingContainer[call.args.size()]; + for (int i = 0; i < call.args.size(); i++) { + args[i] = call.args.get(i).accept(this, generator); + } + FunctionHolder holder = registry.getFunction(call); + return holder.renderFunction(generator, args); + } + + @Override + public HoldingContainer visitIfExpression(IfExpression ifExpr, CodeGenerator generator) throws RuntimeException { + JBlock local = generator.getEvalBlock(); + + HoldingContainer output = generator.declare(ifExpr.getMajorType()); + + JConditional jc = null; + JBlock conditionalBlock = new JBlock(false, false); + for (IfCondition c : ifExpr.conditions) { + HoldingContainer HoldingContainer = c.condition.accept(this, generator); + if (jc == null) { + if (HoldingContainer.isOptional()) { + jc = conditionalBlock._if(HoldingContainer.getIsSet().cand(HoldingContainer.getValue())); + } else { + jc = conditionalBlock._if(HoldingContainer.getValue()); + } } else { - jc = conditionalBlock._if(HoldingContainer.getValue()); + if (HoldingContainer.isOptional()) { + jc = jc._else()._if(HoldingContainer.getIsSet().cand(HoldingContainer.getValue())); + } else { + jc = jc._else()._if(HoldingContainer.getValue()); + } } - } else { - if (HoldingContainer.isOptional()) { - jc = jc._else()._if(HoldingContainer.getIsSet().cand(HoldingContainer.getValue())); + + HoldingContainer thenExpr = c.expression.accept(this, generator); + if (thenExpr.isOptional()) { + JConditional newCond = jc._then()._if(thenExpr.getIsSet()); + JBlock b = newCond._then(); + b.assign(output.getValue(), thenExpr.getValue()); + b.assign(output.getIsSet(), thenExpr.getIsSet()); } else { - jc = jc._else()._if(HoldingContainer.getValue()); + jc._then().assign(output.getValue(), thenExpr.getValue()); } + } - HoldingContainer thenExpr = c.expression.accept(this, generator); - if (thenExpr.isOptional()) { - JConditional newCond = jc._then()._if(thenExpr.getIsSet()); + HoldingContainer elseExpr = ifExpr.elseExpression.accept(this, generator); + if (elseExpr.isOptional()) { + JConditional newCond = jc._else()._if(elseExpr.getIsSet()); JBlock b = newCond._then(); - b.assign(output.getValue(), thenExpr.getValue()); - b.assign(output.getIsSet(), thenExpr.getIsSet()); + b.assign(output.getValue(), elseExpr.getValue()); + b.assign(output.getIsSet(), elseExpr.getIsSet()); } else { - jc._then().assign(output.getValue(), thenExpr.getValue()); + jc._else().assign(output.getValue(), elseExpr.getValue()); + } + local.add(conditionalBlock); + return output; + } + @Override + public HoldingContainer visitSchemaPath(SchemaPath path, CodeGenerator generator) throws RuntimeException { + throw new UnsupportedOperationException("All schema paths should have been replaced with ValueVectorExpressions."); } - HoldingContainer elseExpr = ifExpr.elseExpression.accept(this, generator); - if (elseExpr.isOptional()) { - JConditional newCond = jc._else()._if(elseExpr.getIsSet()); - JBlock b = newCond._then(); - b.assign(output.getValue(), elseExpr.getValue()); - b.assign(output.getIsSet(), elseExpr.getIsSet()); - } else { - jc._else().assign(output.getValue(), elseExpr.getValue()); + @Override + public HoldingContainer visitLongConstant(LongExpression e, CodeGenerator generator) throws RuntimeException { + HoldingContainer out = generator.declare(e.getMajorType()); + generator.getEvalBlock().assign(out.getValue(), JExpr.lit(e.getLong())); + return out; + } + @Override + public HoldingContainer visitDoubleConstant(DoubleExpression e, CodeGenerator generator) throws RuntimeException { + HoldingContainer out = generator.declare(e.getMajorType()); + generator.getEvalBlock().assign(out.getValue(), JExpr.lit(e.getDouble())); + return out; } - local.add(conditionalBlock); - return output; - } - @Override - public HoldingContainer visitSchemaPath(SchemaPath path, CodeGenerator generator) throws RuntimeException { - throw new UnsupportedOperationException("All schema paths should have been replaced with ValueVectorExpressions."); - } + @Override + public HoldingContainer visitBooleanConstant(BooleanExpression e, CodeGenerator generator) + throws RuntimeException { + HoldingContainer out = generator.declare(e.getMajorType()); + generator.getEvalBlock().assign(out.getValue(), JExpr.lit(e.getBoolean() ? 1 : 0)); + return out; + } - @Override - public HoldingContainer visitLongConstant(LongExpression e, CodeGenerator generator) throws RuntimeException { - HoldingContainer out = generator.declare(e.getMajorType()); - generator.getBlock().assign(out.getValue(), JExpr.lit(e.getLong())); - return out; - } + @Override + public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator generator) throws RuntimeException { + if (e instanceof ValueVectorReadExpression) { + return visitValueVectorReadExpression((ValueVectorReadExpression) e, generator); + } else if (e instanceof ValueVectorWriteExpression) { + return visitValueVectorWriteExpression((ValueVectorWriteExpression) e, generator); + } else if (e instanceof ReturnValueExpression) { + return visitReturnValueExpression((ReturnValueExpression) e, generator); + }else if(e instanceof HoldingContainerExpression){ + return ((HoldingContainerExpression) e).getContainer(); + } else { + return super.visitUnknown(e, generator); + } - @Override - public HoldingContainer visitDoubleConstant(DoubleExpression e, CodeGenerator generator) throws RuntimeException { - HoldingContainer out = generator.declare(e.getMajorType()); - generator.getBlock().assign(out.getValue(), JExpr.lit(e.getDouble())); - return out; - } + } - @Override - public HoldingContainer visitBooleanConstant(BooleanExpression e, CodeGenerator generator) throws RuntimeException { - HoldingContainer out = generator.declare(e.getMajorType()); - generator.getBlock().assign(out.getValue(), JExpr.lit(e.getBoolean() ? 1 : 0)); - return out; - } - - @Override - public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator generator) throws RuntimeException { - if(e instanceof ValueVectorReadExpression){ - return visitValueVectorReadExpression((ValueVectorReadExpression) e, generator); - }else if(e instanceof ValueVectorWriteExpression){ - return visitValueVectorWriteExpression((ValueVectorWriteExpression) e, generator); - }else if(e instanceof ReturnValueExpression){ - return visitReturnValueExpression((ReturnValueExpression) e, generator); - }else{ - return super.visitUnknown(e, generator); + private HoldingContainer visitValueVectorWriteExpression(ValueVectorWriteExpression e, CodeGenerator generator) { + LogicalExpression child = e.getChild(); + HoldingContainer hc = child.accept(this, generator); + JBlock block = generator.getEvalBlock(); + JExpression outIndex = generator.getMappingSet().getValueWriteIndex(); + JVar vv = generator.declareVectorValueSetupAndMember("outgoing", e.getFieldId()); + + if (hc.isOptional()) { + vv.invoke("getMutator").invoke("set").arg(outIndex); + JConditional jc = block._if(hc.getIsSet().eq(JExpr.lit(0)).not()); + block = jc._then(); + } + if (hc.getMinorType() == TypeProtos.MinorType.VARCHAR || hc.getMinorType() == TypeProtos.MinorType.VARBINARY) { + block.add(vv.invoke("getMutator").invoke("set").arg(outIndex).arg(hc.getHolder())); + } else { + block.add(vv.invoke("getMutator").invoke("set").arg(outIndex).arg(hc.getValue())); + } + return null; } - - } - private HoldingContainer visitValueVectorWriteExpression(ValueVectorWriteExpression e, CodeGenerator generator){ - LogicalExpression child = e.getChild(); - HoldingContainer hc = child.accept(this, generator); - JBlock block = generator.getBlock(); - - JVar vv = generator.declareVectorValueSetupAndMember("outgoing", e.getFieldId()); - - if(hc.isOptional()){ - vv.invoke("getMutator").invoke("set").arg(JExpr.direct("outIndex")); - JConditional jc = block._if(hc.getIsSet().eq(JExpr.lit(0)).not()); - block = jc._then(); + private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpression e, CodeGenerator generator) + throws RuntimeException { + // declare value vector + + JVar vv1 = generator.declareVectorValueSetupAndMember("incoming", e.getFieldId()); + JExpression indexVariable = generator.getMappingSet().getValueReadIndex(); + + JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get"); + if (e.isSuperReader()) { + + getValueAccessor = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor") + .invoke("get"); + indexVariable = indexVariable.band(JExpr.lit((int) Character.MAX_VALUE)); + } + + // evaluation work. + HoldingContainer out = generator.declare(e.getMajorType()); + + if (out.isOptional()) { + JBlock blk = generator.getEvalBlock(); + blk.assign(out.getIsSet(), vv1.invoke("getAccessor").invoke("isSet").arg(indexVariable)); + JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1))); + if (Types.usesHolderForGet(e.getMajorType())) { + jc._then().add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); + } else { + jc._then().assign(out.getValue(), getValueAccessor.arg(indexVariable)); + } + } else { + if (Types.usesHolderForGet(e.getMajorType())) { + generator.getEvalBlock().add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); + } else { + generator.getEvalBlock().assign(out.getValue(), getValueAccessor.arg(indexVariable)); + } + } + return out; + } + + private HoldingContainer visitReturnValueExpression(ReturnValueExpression e, CodeGenerator generator) { + LogicalExpression child = e.getChild(); + // Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN)); + HoldingContainer hc = child.accept(this, generator); + generator.getEvalBlock()._return(hc.getValue().eq(JExpr.lit(1))); + return null; } - if (hc.getMinorType() == TypeProtos.MinorType.VARCHAR || hc.getMinorType() == TypeProtos.MinorType.VARBINARY) { - block.add(vv.invoke("getMutator").invoke("set").arg(JExpr.direct("outIndex")).arg(hc.getHolder())); - } else { - block.add(vv.invoke("getMutator").invoke("set").arg(JExpr.direct("outIndex")).arg(hc.getValue())); + + @Override + public HoldingContainer visitQuotedStringConstant(QuotedString e, CodeGenerator generator) + throws RuntimeException { + throw new UnsupportedOperationException( + "We don't yet support string literals as we need to build the string value holders."); + + // JExpr stringLiteral = JExpr.lit(e.value); + // CodeGenerator.block.decl(stringLiteral.invoke("getBytes").arg(JExpr.ref(Charsets.UTF_8)); } - return null; } - - private HoldingContainer visitValueVectorReadExpression(ValueVectorReadExpression e, CodeGenerator generator) throws RuntimeException{ - // declare value vector + + private class ConstantFilter extends EvalVisitor { + + private Set constantBoundaries; - JVar vv1 = generator.declareVectorValueSetupAndMember("incoming", e.getFieldId()); - final String indexVariableName = e.isNamed() ? e.getIndexVariableName() : "inIndex"; - JExpression indexVariable = JExpr.direct(indexVariableName); - JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get"); - if(e.isSuperReader()){ - - getValueAccessor = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor").invoke("get"); - indexVariable = indexVariable.band(JExpr.lit((int) Character.MAX_VALUE)); + public ConstantFilter(Set constantBoundaries) { + super(); + this.constantBoundaries = constantBoundaries; } - - - // evaluation work. - HoldingContainer out = generator.declare(e.getMajorType()); - - if(out.isOptional()){ - JBlock blk = generator.getBlock(); - blk.assign(out.getIsSet(), vv1.invoke("getAccessor").invoke("isSet").arg(indexVariable)); - JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1))); - if (Types.usesHolderForGet(e.getMajorType())) { - jc._then() - .add(getValueAccessor.arg(JExpr.direct("inIndex")).arg(out.getHolder())); + + @Override + public HoldingContainer visitFunctionCall(FunctionCall e, CodeGenerator generator) throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitFunctionCall(e, generator); + generator.getMappingSet().exitConstant(); + return c; } else { - jc._then() - .assign(out.getValue(), getValueAccessor.arg(indexVariable)); + return super.visitFunctionCall(e, generator); } - }else{ - if (Types.usesHolderForGet(e.getMajorType())) { - generator.getBlock().add(getValueAccessor.arg(indexVariable).arg(out.getHolder())); + } + + @Override + public HoldingContainer visitIfExpression(IfExpression e, CodeGenerator generator) throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitIfExpression(e, generator); + generator.getMappingSet().exitConstant(); + return c; } else { - generator.getBlock().assign(out.getValue(), getValueAccessor.arg(indexVariable)); + return super.visitIfExpression(e, generator); } } - return out; - } - - - private HoldingContainer visitReturnValueExpression(ReturnValueExpression e, CodeGenerator generator){ - LogicalExpression child = e.getChild(); -// Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN)); - HoldingContainer hc = child.accept(this, generator); - generator.getBlock()._return(hc.getValue().eq(JExpr.lit(1))); - return null; - } - - @Override - public HoldingContainer visitQuotedStringConstant(QuotedString e, CodeGenerator CodeGenerator) throws RuntimeException { - throw new UnsupportedOperationException("We don't yet support string literals as we need to build the string value holders."); - -// JExpr stringLiteral = JExpr.lit(e.value); -// CodeGenerator.block.decl(stringLiteral.invoke("getBytes").arg(JExpr.ref(Charsets.UTF_8)); - } + @Override + public HoldingContainer visitSchemaPath(SchemaPath e, CodeGenerator generator) throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitSchemaPath(e, generator); + generator.getMappingSet().exitConstant(); + return c; + } else { + return super.visitSchemaPath(e, generator); + } + } + + @Override + public HoldingContainer visitLongConstant(LongExpression e, CodeGenerator generator) throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitLongConstant(e, generator); + generator.getMappingSet().exitConstant(); + return c; + } else { + return super.visitLongConstant(e, generator); + } + } + + @Override + public HoldingContainer visitDoubleConstant(DoubleExpression e, CodeGenerator generator) throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitDoubleConstant(e, generator); + generator.getMappingSet().exitConstant(); + return c; + } else { + return super.visitDoubleConstant(e, generator); + } + } + + @Override + public HoldingContainer visitBooleanConstant(BooleanExpression e, CodeGenerator generator) + throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitBooleanConstant(e, generator); + generator.getMappingSet().exitConstant(); + return c; + } else { + return super.visitBooleanConstant(e, generator); + } + } + + @Override + public HoldingContainer visitUnknown(LogicalExpression e, CodeGenerator generator) throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitUnknown(e, generator); + generator.getMappingSet().exitConstant(); + return c; + } else { + return super.visitUnknown(e, generator); + } + } + + @Override + public HoldingContainer visitQuotedStringConstant(QuotedString e, CodeGenerator generator) + throws RuntimeException { + if (constantBoundaries.contains(e)) { + generator.getMappingSet().enterConstant(); + HoldingContainer c = super.visitQuotedStringConstant(e, generator); + generator.getMappingSet().exitConstant(); + return c; + } else { + return super.visitQuotedStringConstant(e, generator); + } + } + + } } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java index 07db72cc27c..cb2be68aa90 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java @@ -80,7 +80,7 @@ public LogicalExpression visitFunctionCall(FunctionCall call) { @Override public LogicalExpression visitIfExpression(IfExpression ifExpr) { - List conditions = Lists.newArrayList(ifExpr.iterator()); + List conditions = Lists.newArrayList(ifExpr.conditions); LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, null); for (int i = 0; i < conditions.size(); ++i) { diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HoldingContainerExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HoldingContainerExpression.java new file mode 100644 index 00000000000..da22bee57f5 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HoldingContainerExpression.java @@ -0,0 +1,46 @@ +package org.apache.drill.exec.expr; + +import java.util.Iterator; + +import org.apache.drill.common.expression.ExpressionPosition; +import org.apache.drill.common.expression.LogicalExpression; +import org.apache.drill.common.expression.visitors.ExprVisitor; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; + +import com.google.common.collect.Iterators; + +public class HoldingContainerExpression implements LogicalExpression{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HoldingContainerExpression.class); + + final HoldingContainer container; + + public HoldingContainerExpression(HoldingContainer container) { + this.container = container; + } + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + + @Override + public MajorType getMajorType() { + return container.getMajorType(); + } + + @Override + public T accept(ExprVisitor visitor, V value) throws E { + return visitor.visitUnknown(this, value); + } + + + public HoldingContainer getContainer() { + return container; + } + + @Override + public ExpressionPosition getPosition() { + return ExpressionPosition.UNKNOWN; + } +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java index f86bd29bd60..6848234e854 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorReadExpression.java @@ -1,11 +1,15 @@ package org.apache.drill.exec.expr; +import java.util.Iterator; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.visitors.ExprVisitor; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.record.TypedFieldId; +import com.google.common.collect.Iterators; + public class ValueVectorReadExpression implements LogicalExpression{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVectorReadExpression.class); @@ -58,6 +62,11 @@ public TypedFieldId getFieldId() { public ExpressionPosition getPosition() { return ExpressionPosition.UNKNOWN; } + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java index 58141d86bec..7f2d8477fe9 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ValueVectorWriteExpression.java @@ -1,5 +1,7 @@ package org.apache.drill.exec.expr; +import java.util.Iterator; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.visitors.ExprVisitor; @@ -7,6 +9,8 @@ import org.apache.drill.common.types.Types; import org.apache.drill.exec.record.TypedFieldId; +import com.google.common.collect.Iterators; + public class ValueVectorWriteExpression implements LogicalExpression { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVectorWriteExpression.class); @@ -47,5 +51,10 @@ public LogicalExpression getChild() { return child; } + @Override + public Iterator iterator() { + return Iterators.singletonIterator(child); + } + } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/MethodMap.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/MethodMap.java new file mode 100644 index 00000000000..3f3f45d6ac7 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/MethodMap.java @@ -0,0 +1,21 @@ +package org.apache.drill.exec.expr.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; + +/** + * Marker annotation to determine which fields should be included as parameters for the function. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface MethodMap { + + + String parentMethod(); + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java new file mode 100644 index 00000000000..a77b4b05eba --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java @@ -0,0 +1,36 @@ +package org.apache.drill.exec.expr.fn; + +import java.util.List; +import java.util.Map; + +import org.apache.drill.exec.expr.CodeGenerator; +import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; + +public class DrillAggFuncHolder extends DrillFuncHolder{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillAggFuncHolder.class); + + + public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, + String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, + Map methods, List imports) { + super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports); + + } + + public HoldingContainer renderFunction(CodeGenerator g, HoldingContainer[] inputVariables){ + g.getMappingSet().enterChild(); + renderInside(g, inputVariables); + g.getMappingSet().exitChild(); + return renderOutside(g); + } + + private void renderInside(CodeGenerator g, HoldingContainer[] inputVariables){ + + } + + private HoldingContainer renderOutside(CodeGenerator g){ + return null; + } +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java new file mode 100644 index 00000000000..8e696094dcc --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java @@ -0,0 +1,102 @@ +package org.apache.drill.exec.expr.fn; + +import java.util.List; +import java.util.Map; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.exec.expr.CodeGenerator; +import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; +import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.sun.codemodel.JBlock; +import com.sun.codemodel.JConditional; +import com.sun.codemodel.JExpr; +import com.sun.codemodel.JExpression; +import com.sun.codemodel.JMod; +import com.sun.codemodel.JVar; + +public class DrillFuncHolder extends FunctionHolder{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolder.class); + + private final String setupBody; + private final String evalBody; + + + public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, + String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, + Map methods, List imports) { + super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports); + setupBody = methods.get("setup"); + evalBody = methods.get("eval"); + Preconditions.checkNotNull(evalBody); + + } + + private void generateSetupBody(CodeGenerator g){ + if(!Strings.isNullOrEmpty(setupBody)){ + JBlock sub = new JBlock(true, true); + addProtectedBlock(g, sub, setupBody, null); + g.getSetupBlock().directStatement(String.format("/** start setup for function %s **/ ", functionName)); + g.getSetupBlock().add(sub); + g.getSetupBlock().directStatement(String.format("/** end setup for function %s **/ ", functionName)); + } + } + + + public HoldingContainer renderFunction(CodeGenerator g, HoldingContainer[] inputVariables){ + generateSetupBody(g); + return generateEvalBody(g, inputVariables); + } + + private HoldingContainer generateEvalBody(CodeGenerator g, HoldingContainer[] inputVariables){ + + //g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName)); + + JBlock sub = new JBlock(true, true); + JBlock topSub = sub; + HoldingContainer out = null; + MajorType returnValueType = returnValue.type; + + // add outside null handling if it is defined. + if(nullHandling == NullHandling.NULL_IF_NULL){ + JExpression e = null; + for(HoldingContainer v : inputVariables){ + if(v.isOptional()){ + if(e == null){ + e = v.getIsSet(); + }else{ + e = e.mul(v.getIsSet()); + } + } + } + + if(e != null){ + // if at least one expression must be checked, set up the conditional. + returnValueType = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build(); + out = g.declare(returnValueType); + e = e.eq(JExpr.lit(0)); + JConditional jc = sub._if(e); + jc._then().assign(out.getIsSet(), JExpr.lit(0)); + sub = jc._else(); + } + } + + if(out == null) out = g.declare(returnValueType); + + // add the subblock after the out declaration. + g.getEvalBlock().add(topSub); + + + JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValueType), returnValue.name, JExpr._new(g.getHolderType(returnValueType))); + addProtectedBlock(g, sub, evalBody, inputVariables); + if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode + sub.assign(out.getHolder(), internalOutput); + + return out; + } + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionBody.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionBody.java new file mode 100644 index 00000000000..04d6e4c8627 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionBody.java @@ -0,0 +1,12 @@ +package org.apache.drill.exec.expr.fn; + +public class FunctionBody { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionBody.class); + + + public static enum BodyType{ + SETUP, EVAL_INNER, EVAL_OUTER, + } + + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java index 5babc78038c..24ebeaa9db1 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java @@ -127,8 +127,18 @@ public FunctionHolder getHolder(Class clazz){ } try{ - FunctionHolder fh = new FunctionHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, works, methods, imports); - return fh; + switch(template.scope()){ + case POINT_AGGREGATE: + return new DrillAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, works, methods, imports); + case SIMPLE: + FunctionHolder fh = new DrillFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, works, methods, imports); + return fh; + + case HOLISTIC_AGGREGATE: + case RANGE_AGGREGATE: + default: + return failure("Unsupported Function Type.", clazz); + } }catch(Exception ex){ return failure("Failure while creating function holder.", ex, clazz); } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java index e91b904b121..0c9b97905a9 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java @@ -1,12 +1,10 @@ package org.apache.drill.exec.expr.fn; -import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.Types; import org.apache.drill.exec.expr.CodeGenerator; @@ -16,29 +14,25 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.sun.codemodel.JBlock; -import com.sun.codemodel.JConditional; -import com.sun.codemodel.JExpr; -import com.sun.codemodel.JExpression; import com.sun.codemodel.JMod; import com.sun.codemodel.JVar; -public class FunctionHolder { +public abstract class FunctionHolder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class); - private FunctionTemplate.FunctionScope scope; - private FunctionTemplate.NullHandling nullHandling; - private boolean isBinaryCommutative; - private String functionName; - private String evalBody; - private String addBody; - private String setupBody; - private List imports; - private WorkspaceReference[] workspaceVars; - private ValueReference[] parameters; - private ValueReference returnValue; + protected final FunctionTemplate.FunctionScope scope; + protected final FunctionTemplate.NullHandling nullHandling; + protected final boolean isBinaryCommutative; + protected final String functionName; + protected final ImmutableList imports; + protected final WorkspaceReference[] workspaceVars; + protected final ValueReference[] parameters; + protected final ValueReference returnValue; + protected final ImmutableMap methodMap; public FunctionHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map methods, List imports) { super(); @@ -47,14 +41,10 @@ public FunctionHolder(FunctionScope scope, NullHandling nullHandling, boolean is this.workspaceVars = workspaceVars; this.isBinaryCommutative = isBinaryCommutative; this.functionName = functionName; - this.setupBody = methods.get("setup"); - this.addBody = methods.get("add"); - this.evalBody = methods.get("eval"); - Preconditions.checkNotNull(evalBody); - Preconditions.checkArgument(!evalBody.isEmpty()); + this.methodMap = ImmutableMap.copyOf(methods); this.parameters = parameters; this.returnValue = returnValue; - this.imports = imports; + this.imports = ImmutableList.copyOf(imports); } @@ -62,15 +52,7 @@ public List getImports() { return imports; } - private void generateSetupBody(CodeGenerator g){ - if(!Strings.isNullOrEmpty(setupBody)){ - JBlock sub = new JBlock(true, true); - addProtectedBlock(g, sub, setupBody, null); - g.getSetupBlock().directStatement(String.format("/** start setup for function %s **/", functionName)); - g.getSetupBlock().add(sub); - g.getSetupBlock().directStatement(String.format("/** end setup for function %s **/", functionName)); - } - } + public abstract HoldingContainer renderFunction(CodeGenerator g, HoldingContainer[] inputVariables); public void addProtectedBlock(CodeGenerator g, JBlock sub, String body, HoldingContainer[] inputVariables){ @@ -102,57 +84,7 @@ public void addProtectedBlock(CodeGenerator g, JBlock sub, String body, Holdi } } - public HoldingContainer renderFunction(CodeGenerator g, HoldingContainer[] inputVariables){ - generateSetupBody(g); - return generateEvalBody(g, inputVariables); - } - - private HoldingContainer generateEvalBody(CodeGenerator g, HoldingContainer[] inputVariables){ - - //g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName)); - - JBlock sub = new JBlock(true, true); - JBlock topSub = sub; - HoldingContainer out = null; - - // add outside null handling if it is defined. - if(nullHandling == NullHandling.NULL_IF_NULL){ - JExpression e = null; - for(HoldingContainer v : inputVariables){ - if(v.isOptional()){ - if(e == null){ - e = v.getIsSet(); - }else{ - e = e.mul(v.getIsSet()); - } - } - } - - if(e != null){ - // if at least one expression must be checked, set up the conditional. - returnValue.type = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build(); - out = g.declare(returnValue.type); - e = e.eq(JExpr.lit(0)); - JConditional jc = sub._if(e); - jc._then().assign(out.getIsSet(), JExpr.lit(0)); - sub = jc._else(); - } - } - - if(out == null) out = g.declare(returnValue.type); - - // add the subblock after the out declaration. - g.getBlock().add(topSub); - - - JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type))); - addProtectedBlock(g, sub, evalBody, inputVariables); - if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode - sub.assign(out.getHolder(), internalOutput); - - return out; - } - + public boolean matches(FunctionCall call){ @@ -215,15 +147,7 @@ public WorkspaceReference(Class type, String name) { } } - @Override - public String toString() { - final int maxLen = 10; - return "FunctionHolder [scope=" + scope + ", isBinaryCommutative=" + isBinaryCommutative + ", functionName=" - + functionName + ", evalBody=" + evalBody + ", addBody=" + addBody + ", setupBody=" + setupBody - + ", parameters=" - + (parameters != null ? Arrays.asList(parameters).subList(0, Math.min(parameters.length, maxLen)) : null) - + ", returnValue=" + returnValue + "]"; - } + } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index c2dfca41bbb..bbe4cfbf9df 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -107,14 +107,6 @@ public T getImplementationClass(CodeGenerator cg) throws ClassTransformat return t; } - - public T getImplementationClassMultipleOutput(CodeGenerator cg) throws ClassTransformationException, IOException{ - long t1 = System.nanoTime(); - T t = transformer.getImplementationClass(this.loader, cg.getDefinition(), cg.generateMultipleOutputs(), cg.getMaterializedClassName()); - logger.debug("Compile time: {} millis.", (System.nanoTime() - t1)/1000/1000 ); - return t; - - } public void addMetricsToStatus(FragmentStatus.Builder stats){ stats.setBatchesCompleted(batchesCompleted.get()); diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java new file mode 100644 index 00000000000..f085cd3e819 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterSignature.java @@ -0,0 +1,14 @@ +package org.apache.drill.exec.physical.impl.filter; + +import javax.inject.Named; + +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface FilterSignature extends CodeGeneratorSignature{ + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing); + public boolean doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex); + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java index b270869c484..fb08ef3c34b 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/Filterer.java @@ -3,7 +3,6 @@ import org.apache.drill.exec.compile.TemplateClassDefinition; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.physical.impl.project.Projector; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.TransferPair; @@ -14,6 +13,6 @@ public interface Filterer { public void filterBatch(int recordCount); public static TemplateClassDefinition TEMPLATE_DEFINITION = new TemplateClassDefinition( // - Filterer.class, "org.apache.drill.exec.physical.impl.filter.FilterTemplate", FilterEvaluator.class, boolean.class); + Filterer.class, "org.apache.drill.exec.physical.impl.filter.FilterTemplate", FilterEvaluator.class, FilterSignature.class); } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java index a794d633826..cfe520dcf7d 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java @@ -1,11 +1,15 @@ package org.apache.drill.exec.physical.impl.filter; +import java.util.Iterator; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.visitors.ExprVisitor; import org.apache.drill.common.types.Types; import org.apache.drill.common.types.TypeProtos.MajorType; +import com.google.common.collect.Iterators; + public class ReturnValueExpression implements LogicalExpression{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReturnValueExpression.class); @@ -34,6 +38,10 @@ public ExpressionPosition getPosition() { return ExpressionPosition.UNKNOWN; } - + @Override + public Iterator iterator() { + return Iterators.singletonIterator(child); + } + } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java index 6d24e0b6131..f96e3cf1f5d 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java @@ -171,7 +171,7 @@ private void createPartitioner() throws SchemaChangeException { // generate evaluate expression to determine the hash CodeGenerator.HoldingContainer exprHolder = cg.addExpr(materializedExpr); - cg.getBlock().decl(JType.parse(cg.getModel(), "int"), "bucket", exprHolder.getValue().mod(JExpr.lit(outgoing.length))); + cg.getEvalBlock().decl(JType.parse(cg.getModel(), "int"), "bucket", exprHolder.getValue().mod(JExpr.lit(outgoing.length))); // declare and assign the array of outgoing record batches JVar outgoingBatches = cg.clazz.field(JMod.NONE, @@ -230,7 +230,7 @@ private void createPartitioner() throws SchemaChangeException { // ((IntVector) outgoingVectors[bucket][0]).copyFrom(inIndex, // outgoingBatches[bucket].getRecordCount(), // vv1); - cg.getBlock().add( + cg.getEvalBlock().add( ((JExpression) JExpr.cast(vvClass, ((JExpression) outgoingVectors @@ -244,11 +244,12 @@ private void createPartitioner() throws SchemaChangeException { ++fieldId; } // generate the OutgoingRecordBatch helper invocations - cg.getBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("incRecordCount")); - cg.getBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("flushIfNecessary")); + cg.getEvalBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("incRecordCount")); + cg.getEvalBlock().add(((JExpression) outgoingBatches.component(bucket)).invoke("flushIfNecessary")); try { // compile and setup generated code - partitioner = context.getImplementationClassMultipleOutput(cg); +// partitioner = context.getImplementationClassMultipleOutput(cg); + partitioner = context.getImplementationClass(cg); partitioner.setup(context, incoming, outgoing); } catch (ClassTransformationException | IOException e) { diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java index e8f2ca71609..992ffdfc47c 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/Partitioner.java @@ -35,5 +35,5 @@ public abstract void setup(FragmentContext context, new TemplateClassDefinition<>(Partitioner.class, "org.apache.drill.exec.physical.impl.partitionsender.PartitionerTemplate", PartitionerEvaluator.class, - null); + PartitionerInnerSignature.class); } \ No newline at end of file diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java new file mode 100644 index 00000000000..be209a95374 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionerInnerSignature.java @@ -0,0 +1,17 @@ +package org.apache.drill.exec.physical.impl.partitionsender; + +import javax.inject.Named; + +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface PartitionerInnerSignature extends CodeGeneratorSignature{ + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") OutgoingRecordBatch[] outgoing) throws SchemaChangeException; + public void doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex); + + + +} \ No newline at end of file diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java index 5fd1fb4db0b..75632e74bca 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectEvaluator.java @@ -8,5 +8,5 @@ public interface ProjectEvaluator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectEvaluator.class); public abstract void doSetup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException; - public abstract void doEval(int inIndex, int outIndex); + public abstract void doEval(int inIndex, int outIndex) throws SchemaChangeException; } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java index 0d1e201e3b3..ba83e619f82 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/Projector.java @@ -3,6 +3,7 @@ import java.util.List; import org.apache.drill.exec.compile.TemplateClassDefinition; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.record.RecordBatch; @@ -16,6 +17,6 @@ public interface Projector { public abstract int projectRecords(int recordCount, int firstOutputIndex); public static TemplateClassDefinition TEMPLATE_DEFINITION = new TemplateClassDefinition( // - Projector.class, "org.apache.drill.exec.physical.impl.project.ProjectorTemplate", ProjectEvaluator.class, null); + Projector.class, "org.apache.drill.exec.physical.impl.project.ProjectorTemplate", ProjectEvaluator.class); } \ No newline at end of file diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java index 83d43b2da79..02fffa5efd9 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/ReadIndexRewriter.java @@ -45,7 +45,7 @@ public LogicalExpression visitFunctionCall(FunctionCall call, String newIndexNam @Override public LogicalExpression visitIfExpression(IfExpression ifExpr, String newIndexName) { - List conditions = Lists.newArrayList(ifExpr.iterator()); + List conditions = Lists.newArrayList(ifExpr.conditions); LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, null); for (int i = 0; i < conditions.size(); ++i) { diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java index e361e38306a..c9bd55dc85b 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java @@ -14,6 +14,7 @@ import org.apache.drill.exec.expr.CodeGenerator; import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer; import org.apache.drill.exec.expr.ExpressionTreeMaterializer; +import org.apache.drill.exec.expr.HoldingContainerExpression; import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.config.Sort; @@ -123,22 +124,26 @@ public IterOutcome next() { } + private Sorter createNewSorter() throws ClassTransformationException, IOException, SchemaChangeException{ CodeGenerator g = new CodeGenerator(Sorter.TEMPLATE_DEFINITION, context.getFunctionRegistry()); + g.setMappingSet(SortSignature.MAIN_MAPPING); for(OrderDef od : popConfig.getOrderings()){ // first, we rewrite the evaluation stack for each side of the comparison. ErrorCollector collector = new ErrorCollectorImpl(); final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), this, collector); if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString()); - ReadIndexRewriter rewriter = new ReadIndexRewriter(); - LogicalExpression left = expr.accept(rewriter, "inIndex"); - LogicalExpression right = expr.accept(rewriter, "outIndex"); + g.setMappingSet(SortSignature.LEFT_MAPPING); + HoldingContainer left = g.addExpr(expr, false); + g.setMappingSet(SortSignature.RIGHT_MAPPING); + HoldingContainer right = g.addExpr(expr, false); + g.setMappingSet(SortSignature.MAIN_MAPPING); // next we wrap the two comparison sides and add the expression block for the comparison. - FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of(left, right), ExpressionPosition.UNKNOWN); - HoldingContainer out = g.addExpr(f); - JConditional jc = g.getBlock()._if(out.getValue().ne(JExpr.lit(0))); + FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(left), new HoldingContainerExpression(right)), ExpressionPosition.UNKNOWN); + HoldingContainer out = g.addExpr(f, false); + JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0))); //TODO: is this the right order... if(od.getDirection() == Direction.ASC){ @@ -148,7 +153,7 @@ private Sorter createNewSorter() throws ClassTransformationException, IOExceptio } } - g.getBlock()._return(JExpr.lit(0)); + g.getEvalBlock()._return(JExpr.lit(0)); return context.getImplementationClass(g); diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java new file mode 100644 index 00000000000..7614f3ec2c4 --- /dev/null +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortSignature.java @@ -0,0 +1,20 @@ +package org.apache.drill.exec.physical.impl.sort; + +import javax.inject.Named; + +import org.apache.drill.exec.compile.sig.CodeGeneratorSignature; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; +import org.apache.drill.exec.compile.sig.MappingSet; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.record.RecordBatch; + +public interface SortSignature extends CodeGeneratorSignature{ + + public static final MappingSet MAIN_MAPPING = new MappingSet("null", "null", DefaultGeneratorSignature.DEFAULT_SCALAR_MAP, DefaultGeneratorSignature.DEFAULT_SCALAR_MAP); + public static final MappingSet LEFT_MAPPING = new MappingSet("leftIndex", "null", DefaultGeneratorSignature.DEFAULT_SCALAR_MAP, DefaultGeneratorSignature.DEFAULT_SCALAR_MAP); + public static final MappingSet RIGHT_MAPPING = new MappingSet("rightIndex", "null", DefaultGeneratorSignature.DEFAULT_SCALAR_MAP, DefaultGeneratorSignature.DEFAULT_SCALAR_MAP); + + public void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing); + public int doEval(@Named("leftIndex") int leftIndex, @Named("rightIndex") int rightIndex); + +} diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java index c45f500618a..d312fb4b115 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortTemplate.java @@ -34,12 +34,12 @@ public void swap(int sv0, int sv1) { } @Override - public int compare(int inIndex, int outIndex) { - int sv1 = vector4.get(inIndex); - int sv2 = vector4.get(outIndex); + public int compare(int leftIndex, int rightIndex) { + int sv1 = vector4.get(leftIndex); + int sv2 = vector4.get(rightIndex); return doEval(sv1, sv2); } public abstract void doSetup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing) throws SchemaChangeException; - public abstract int doEval(int inIndex, int outIndex); + public abstract int doEval(int leftIndex, int rightIndex); } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java index bc4fae5b30a..1a764235500 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/Sorter.java @@ -12,6 +12,6 @@ public interface Sorter { public void sort(SelectionVector4 vector4, VectorContainer container); public static TemplateClassDefinition TEMPLATE_DEFINITION = new TemplateClassDefinition( // - Sorter.class, "org.apache.drill.exec.physical.impl.sort.SortTemplate", Comparator.class, int.class); + Sorter.class, "org.apache.drill.exec.physical.impl.sort.SortTemplate", Comparator.class, SortSignature.class); } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java index ce17a2b0311..363bbeeaf65 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/Copier.java @@ -1,18 +1,18 @@ package org.apache.drill.exec.physical.impl.svremover; import org.apache.drill.exec.compile.TemplateClassDefinition; +import org.apache.drill.exec.compile.sig.DefaultGeneratorSignature; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.VectorAllocator; import org.apache.drill.exec.record.RecordBatch; -import org.apache.drill.exec.record.selection.SelectionVector2; public interface Copier { public static TemplateClassDefinition TEMPLATE_DEFINITION2 = new TemplateClassDefinition( // - Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate2", CopyEvaluator.class, null); + Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate2", CopyEvaluator.class); public static TemplateClassDefinition TEMPLATE_DEFINITION4 = new TemplateClassDefinition( // - Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate4", CopyEvaluator.class, null); + Copier.class, "org.apache.drill.exec.physical.impl.svremover.CopierTemplate4", CopyEvaluator.class); public void setupRemover(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException; public abstract void copyRecords(); diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java index 64e89eef6c8..e4fd9a07d88 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java @@ -167,7 +167,7 @@ private void generateCopies(CodeGenerator g, boolean hyper){ if(hyper){ - g.getBlock().add( + g.getEvalBlock().add( outVV .invoke("copyFrom") .arg( @@ -178,7 +178,7 @@ private void generateCopies(CodeGenerator g, boolean hyper){ ) ); }else{ - g.getBlock().add(outVV.invoke("copyFrom").arg(inIndex).arg(outIndex).arg(inVV)); + g.getEvalBlock().add(outVV.invoke("copyFrom").arg(inIndex).arg(outIndex).arg(inVV)); } diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java index c2f2a699395..821a4b46fb0 100644 --- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java +++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/NullExpression.java @@ -1,12 +1,15 @@ package org.apache.drill.exec.record; +import java.util.Iterator; + import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.visitors.ExprVisitor; -import org.apache.drill.common.types.Types; -import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; + +import com.google.common.collect.Iterators; public class NullExpression implements LogicalExpression{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullExpression.class); @@ -29,5 +32,11 @@ public T accept(ExprVisitor visitor, V valu public ExpressionPosition getPosition() { return ExpressionPosition.UNKNOWN; } + + @Override + public Iterator iterator() { + return Iterators.emptyIterator(); + } + } diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index d2889ede185..16c993d7ab0 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -40,7 +40,7 @@ private void testBasicClassCompilation(boolean useJanino) throws ClassTransforma TemplateClassDefinition def = new TemplateClassDefinition( ExampleExternalInterface.class, "org.apache.drill.exec.compile.ExampleTemplate", - ExampleInternalInterface.class, null); + ExampleInternalInterface.class); ClassTransformer ct = new ClassTransformer(); diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java index 4129079f7e7..a25e234e73d 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHashToRandomExchange.java @@ -18,8 +18,10 @@ package org.apache.drill.exec.physical.impl; -import com.google.common.base.Charsets; -import com.google.common.io.Files; +import static org.junit.Assert.assertEquals; + +import java.util.List; + import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.pop.PopUnitTestBase; @@ -27,11 +29,12 @@ import org.apache.drill.exec.rpc.user.QueryResultBatch; import org.apache.drill.exec.server.Drillbit; import org.apache.drill.exec.server.RemoteServiceSet; +import org.junit.Ignore; import org.junit.Test; -import java.nio.charset.Charset; -import java.util.List; -import static org.junit.Assert.assertEquals; +import com.google.common.base.Charsets; +import com.google.common.io.Files; + public class TestHashToRandomExchange extends PopUnitTestBase { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHashToRandomExchange.class); diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java index 8a1736c2cb6..f0d9901aa35 100644 --- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java +++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java @@ -30,7 +30,7 @@ import org.apache.drill.exec.proto.SchemaDefProtos.NamePart; import org.junit.Test; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; public class ExpressionTreeMaterializerTest { @@ -42,16 +42,16 @@ public class ExpressionTreeMaterializerTest { final MajorType intType = MajorType.newBuilder().setMode(DataMode.REQUIRED).setMinorType(MinorType.INT).build(); private MaterializedField getField(int fieldId, String name, MajorType type) { - return new MaterializedField(FieldDef.newBuilder().setMajorType(type) - .addName(NamePart.newBuilder().setName(name)).build()); + return new MaterializedField(FieldDef.newBuilder().setMajorType(type).addName(NamePart.newBuilder().setName(name)) + .build()); } - @Test public void testMaterializingConstantTree(@Injectable RecordBatch batch) throws SchemaChangeException { - + ErrorCollector ec = new ErrorCollectorImpl(); - LogicalExpression expr = ExpressionTreeMaterializer.materialize(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN), batch, ec); + LogicalExpression expr = ExpressionTreeMaterializer.materialize(new ValueExpressions.LongExpression(1L, + ExpressionPosition.UNKNOWN), batch, ec); assertTrue(expr instanceof ValueExpressions.LongExpression); assertEquals(1L, ValueExpressions.LongExpression.class.cast(expr).getLong()); assertFalse(ec.hasErrors()); @@ -62,13 +62,14 @@ public void testMaterializingLateboundField(final @Injectable RecordBatch batch) final SchemaBuilder builder = BatchSchema.newBuilder(); builder.addField(getField(2, "test", bigIntType)); final BatchSchema schema = builder.build(); - + new NonStrictExpectations() { { - batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); + batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); } }; - + ErrorCollector ec = new ErrorCollectorImpl(); LogicalExpression expr = ExpressionTreeMaterializer.materialize(new FieldReference("test", ExpressionPosition.UNKNOWN), batch, ec); @@ -80,23 +81,26 @@ public void testMaterializingLateboundField(final @Injectable RecordBatch batch) public void testMaterializingLateboundTree(final @Injectable RecordBatch batch) throws SchemaChangeException { new NonStrictExpectations() { { - batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIT), -4); - batch.getValueVectorId(new FieldReference("test1", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); + batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIT), -4); + batch.getValueVectorId(new FieldReference("test1", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); } }; - + ErrorCollector ec = new ErrorCollectorImpl(); - - LogicalExpression expr = new IfExpression.Builder() + LogicalExpression expr = new IfExpression.Builder() .addCondition( new IfExpression.IfCondition( // new FieldReference("test", ExpressionPosition.UNKNOWN), // - new IfExpression.Builder() // + new IfExpression.Builder() + // .addCondition( // - new IfExpression.IfCondition( // - new ValueExpressions.BooleanExpression("true", ExpressionPosition.UNKNOWN), new FieldReference( - "test1", ExpressionPosition.UNKNOWN))) + new IfExpression.IfCondition( + // + new ValueExpressions.BooleanExpression("true", ExpressionPosition.UNKNOWN), + new FieldReference("test1", ExpressionPosition.UNKNOWN))) .setElse(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN)).build()) // ) // .setElse(new ValueExpressions.LongExpression(1L, ExpressionPosition.UNKNOWN)).build(); @@ -111,7 +115,8 @@ public void testMaterializingLateboundTree(final @Injectable RecordBatch batch) ifCondition = newIfExpr.conditions.get(0); assertEquals(bigIntType, ifCondition.expression.getMajorType()); assertEquals(true, ((ValueExpressions.BooleanExpression) ifCondition.condition).value); - if (ec.hasErrors()) System.out.println(ec.toErrorString()); + if (ec.hasErrors()) + System.out.println(ec.toErrorString()); assertFalse(ec.hasErrors()); } @@ -126,8 +131,8 @@ public void addGeneralError(ExpressionPosition expr, String s) { } @Override - public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, MajorType[] expected, - int argumentIndex) { + public void addUnexpectedArgumentType(ExpressionPosition expr, String name, MajorType actual, + MajorType[] expected, int argumentIndex) { errorCount++; } @@ -174,14 +179,17 @@ public int getErrorCount() { new NonStrictExpectations() { { - batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); + batch.getValueVectorId(new FieldReference("test", ExpressionPosition.UNKNOWN)); + result = new TypedFieldId(Types.required(MinorType.BIGINT), -5); } }; + LogicalExpression functionCallExpr = new FunctionCall(FunctionDefinition.simple("testFunc", new ArgumentValidator() { @Override - public void validateArguments(ExpressionPosition expr, List expressions, ErrorCollector errors) { + public void validateArguments(ExpressionPosition expr, List expressions, + ErrorCollector errors) { errors.addGeneralError(expr, "Error!"); } @@ -189,8 +197,8 @@ public void validateArguments(ExpressionPosition expr, List e public String[] getArgumentNamesByPosition() { return new String[0]; } - }, OutputTypeDeterminer.FIXED_BIT), Lists.newArrayList((LogicalExpression) new FieldReference("test", - ExpressionPosition.UNKNOWN)), ExpressionPosition.UNKNOWN); + }, OutputTypeDeterminer.FIXED_BIT), ImmutableList.of((LogicalExpression) // + new FieldReference("test", ExpressionPosition.UNKNOWN) ), ExpressionPosition.UNKNOWN); LogicalExpression newExpr = ExpressionTreeMaterializer.materialize(functionCallExpr, batch, ec); assertTrue(newExpr instanceof FunctionCall); FunctionCall funcExpr = (FunctionCall) newExpr;