From 2ec6e684d3f7df7cfda195a1355d9c1e45eedc4b Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 11 Aug 2014 00:59:47 +0900 Subject: [PATCH 01/31] initial work for JitVecTestBase. --- pom.xml | 1 + tajo-jitvec-tests/pom.xml | 358 ++++++++++++++++++ .../apache/tajo/jitvec/JitVecTestServer.java | 172 +++++++++ .../src/main/proto/JitVecTestProtocol.proto | 40 ++ tajo-project/pom.xml | 6 + .../apache/tajo/rpc/ProtoPipelineFactory.java | 7 +- 6 files changed, 582 insertions(+), 2 deletions(-) create mode 100644 tajo-jitvec-tests/pom.xml create mode 100644 tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java create mode 100644 tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto diff --git a/pom.xml b/pom.xml index bf33c69305..fa184a3ba9 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,7 @@ tajo-storage tajo-yarn-pullserver tajo-dist + tajo-jitvec-tests diff --git a/tajo-jitvec-tests/pom.xml b/tajo-jitvec-tests/pom.xml new file mode 100644 index 0000000000..a44cc5b9e7 --- /dev/null +++ b/tajo-jitvec-tests/pom.xml @@ -0,0 +1,358 @@ + + + + + + tajo-project + org.apache.tajo + 0.9.0-SNAPSHOT + ../tajo-project + + 4.0.0 + + tajo-jitvec-tests + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + ${project.build.sourceEncoding} + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + + TRUE + + -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 + + + + maven-deploy-plugin + + + org.apache.maven.plugins + maven-jar-plugin + 2.3.1 + + + LICENSE + + + false + + + + + package + + test-jar + + + + + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../tajo-common/src/main/proto + --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto + --proto_path=../tajo-client/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/JitVecTestProtocol.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + runtime + ${project.build.directory}/lib + false + false + true + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 2.7.1 + + + + + + + org.apache.tajo + tajo-common + + + org.apache.tajo + tajo-algebra + + + org.apache.tajo + tajo-catalog-common + + + org.apache.tajo + tajo-catalog-client + + + org.apache.tajo + tajo-catalog-server + + + org.apache.tajo + tajo-storage + + + org.apache.tajo + tajo-yarn-pullserver + + + org.apache.tajo + tajo-client + + + org.apache.tajo + tajo-jdbc + + + org.apache.tajo + tajo-rpc + + + org.apache.tajo + tajo-core + jar + + + org.apache.tajo + tajo-core + jar + + + org.apache.tajo + tajo-core + compile + test-jar + + + + org.apache.hadoop + hadoop-common + compile + + + org.apache.hadoop + hadoop-minicluster + compile + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + + + org.apache.hadoop + hadoop-hdfs + compile + + + commons-el + commons-el + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + org.mortbay.jetty + jsp-2.1-jetty + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + + + + + org.apache.hadoop + hadoop-yarn-api + compile + + + org.apache.hadoop + hadoop-yarn-common + compile + + + org.apache.hadoop + hadoop-yarn-server-common + compile + + + + com.google.protobuf + protobuf-java + + + org.antlr + antlr4 + + + junit + junit + test + + + org.mockito + mockito-core + test + + + org.dspace.dependencies + dspace-geoip + 1.2.3 + + + org.eclipse.jdt + core + 3.1.1 + + + org.mortbay.jetty + jetty + 6.1.14 + + + org.mortbay.jetty + jsp-2.1 + 6.1.14 + + + info.ganglia.gmetric4j + gmetric4j + 1.0.3 + + + + \ No newline at end of file diff --git a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java new file mode 100644 index 0000000000..15413b06a6 --- /dev/null +++ b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.jitvec; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; +import com.sun.org.apache.commons.logging.Log; +import com.sun.org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.AbstractService; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.TpchTestBase; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.engine.exception.VerifyException; +import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.planner.*; +import org.apache.tajo.master.session.Session; +import org.apache.tajo.rpc.BlockingRpcServer; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; +import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.ProtoUtil; + +import java.net.InetSocketAddress; + +public class JitVecTestServer extends AbstractService { + private static final Log LOG = LogFactory.getLog(JitVecTestServer.class); + + TajoTestingCluster testingCluster; + + private TajoConf conf; + private SQLAnalyzer analyzer; + private CatalogService catalog; + private PreLogicalPlanVerifier preVerifier; + private LogicalPlanner planner; + private LogicalOptimizer optimizer; + private LogicalPlanVerifier annotatedPlanVerifier; + + private BlockingRpcServer rpcServer; + private JitVecTestServerProtocol.JitVecTestServerProtocolService.BlockingInterface handler; + + public JitVecTestServer() { + super(JitVecTestServer.class.getSimpleName()); + testingCluster = TpchTestBase.getInstance().getTestingCluster(); + this.conf = testingCluster.getConfiguration(); + catalog = testingCluster.getMaster().getCatalog(); + + analyzer = new SQLAnalyzer(); + preVerifier = new PreLogicalPlanVerifier(catalog); + planner = new LogicalPlanner(catalog); + optimizer = new LogicalOptimizer(conf); + annotatedPlanVerifier = new LogicalPlanVerifier(conf, catalog); + + handler = new ProtocolHandler(); + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + rpcServer = new BlockingRpcServer(JitVecTestServerProtocol.class, handler, + new InetSocketAddress("0.0.0.0", 30060), 1); + + super.serviceInit(conf); + } + + @Override + public void serviceStart() throws Exception { + rpcServer.start(); + + super.serviceStart(); + + InetSocketAddress bindAddr = rpcServer.getListenAddress(); + System.out.println("\n============================================================================="); + System.out.println("Test Server Addr: " + NetUtils.normalizeInetSocketAddress(bindAddr)); + System.out.println("HDFS Namenode Addr: " + testingCluster.getMiniDFSCluster().getFileSystem().getUri()); + System.out.println("Catalog Server Addr: " + + NetUtils.normalizeInetSocketAddress(testingCluster.getMaster().getCatalogServer().getBindAddress())); + System.out.println("=============================================================================\n"); + } + + public void serviceStop() throws Exception { + rpcServer.shutdown(); + + super.serviceStop(); + } + + public class ProtocolHandler implements JitVecTestServerProtocol.JitVecTestServerProtocolService.BlockingInterface { + @Override + public JitVecTestServerProtocol.PlanResponse requestPlan(RpcController controller, + JitVecTestServerProtocol.RequestPlan request) throws ServiceException { + + JitVecTestServerProtocol.PlanResponse.Builder builder = JitVecTestServerProtocol.PlanResponse.newBuilder(); + + Session session = new Session("00", "tajo", "default"); + try { + LOG.info("Request is received: " + request.getSql()); + Expr expr = analyzer.parse(request.getSql()); + VerificationState state = new VerificationState(); + preVerifier.verify(session, state, expr); + if (!state.verified()) { + StringBuilder sb = new StringBuilder(); + for (String error : state.getErrorMessages()) { + sb.append(error).append("\n"); + } + throw new VerifyException(sb.toString()); + } + + LogicalPlan plan = planner.createPlan(session, expr); + if (LOG.isDebugEnabled()) { + LOG.debug("============================================="); + LOG.debug("Non Optimized Query: \n" + plan.toString()); + LOG.debug("============================================="); + } + LOG.info("Non Optimized Query: \n" + plan.toString()); + optimizer.optimize(session, plan); + LOG.info("============================================="); + LOG.info("Optimized Query: \n" + plan.toString()); + LOG.info("============================================="); + + annotatedPlanVerifier.verify(session, state, plan); + + if (!state.verified()) { + StringBuilder sb = new StringBuilder(); + for (String error : state.getErrorMessages()) { + sb.append(error).append("\n"); + } + throw new VerifyException(sb.toString()); + } + + builder.setSerializedPlan(plan.getRootBlock().getRoot().toJson()); + } catch (PlanningException e) { + if (e.getMessage() != null) { + LOG.error(e.getMessage()); + builder.setErrorMessage(e.getMessage()); + } else { + e.printStackTrace(); + builder.setErrorMessage("Internal Error"); + } + } + + return builder.build(); + } + } + + public static void startServer(String [] args) throws Exception { + TajoConf conf = new TajoConf(); + JitVecTestServer server = new JitVecTestServer(); + server.init(conf); + server.start(); + } + + public static void main(String [] args) throws Exception { + startServer(args); + } +} diff --git a/tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto b/tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto new file mode 100644 index 0000000000..643064f130 --- /dev/null +++ b/tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// JitVecEngine -> JetVecTestServer + +package tajo; +option cc_generic_services = true; + +option java_package = "org.apache.tajo.jitvec"; +option java_outer_classname = "JitVecTestServerProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + +message RequestPlan { + required string sql = 1; +} + +message PlanResponse { + optional string serializedPlan = 1; + optional string errorMessage = 2; +} + +service JitVecTestServerProtocolService { + rpc requestPlan(RequestPlan) returns (PlanResponse); +} \ No newline at end of file diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 7c0da53174..d102331e1c 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -743,6 +743,12 @@ ${tajo.version} test-jar + + org.apache.tajo + tajo-jitvec-tests + ${tajo.version} + jar + org.apache.hadoop diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java index 7aa03e787e..fdecfb660f 100644 --- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java +++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java @@ -23,6 +23,8 @@ import org.jboss.netty.channel.ChannelPipelineFactory; import org.jboss.netty.channel.ChannelUpstreamHandler; import org.jboss.netty.channel.Channels; +import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder; +import org.jboss.netty.handler.codec.frame.LengthFieldPrepender; import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder; import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder; import org.jboss.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; @@ -40,9 +42,10 @@ public ProtoPipelineFactory(ChannelUpstreamHandler handlerFactory, public ChannelPipeline getPipeline() throws Exception { ChannelPipeline p = Channels.pipeline(); - p.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); + p.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(1048576*2, 0, 4, + 0, 4)); p.addLast("protobufDecoder", new ProtobufDecoder(defaultInstance)); - p.addLast("frameEncoder", new ProtobufVarint32LengthFieldPrepender()); + p.addLast("frameEncoder", new LengthFieldPrepender(4)); p.addLast("protobufEncoder", new ProtobufEncoder()); p.addLast("handler", handler); return p; From 7df2c6192a3158fc225b54b424f50f0272ceef5c Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 12 Aug 2014 03:55:34 +0900 Subject: [PATCH 02/31] Added more utility methods to Eval, and Added basic layout of LogicalPlanConvertor. --- tajo-core/pom.xml | 1 + .../tajo/engine/eval/AlgebraicUtil.java | 6 +- .../engine/eval/BetweenPredicateEval.java | 18 ++ .../apache/tajo/engine/eval/BinaryEval.java | 6 + .../apache/tajo/engine/eval/CaseWhenEval.java | 32 +++ .../apache/tajo/engine/eval/ConstEval.java | 12 +- .../org/apache/tajo/engine/eval/EvalNode.java | 4 + .../apache/tajo/engine/eval/EvalTreeUtil.java | 2 +- .../org/apache/tajo/engine/eval/EvalType.java | 35 ++- .../apache/tajo/engine/eval/FieldEval.java | 12 +- .../apache/tajo/engine/eval/FunctionEval.java | 14 + .../tajo/engine/eval/RowConstantEval.java | 10 + .../apache/tajo/engine/eval/UnaryEval.java | 11 + .../engine/plan/LogicalPlanConvertor.java | 265 ++++++++++++++++++ tajo-core/src/main/proto/Plan.proto | 165 +++++++++++ .../apache/tajo/engine/eval/TestEvalTree.java | 22 +- .../engine/plan/TestLogicalPlanConvertor.java | 141 ++++++++++ .../apache/tajo/jitvec/JitVecTestServer.java | 5 + 18 files changed, 748 insertions(+), 13 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java create mode 100644 tajo-core/src/main/proto/Plan.proto create mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 32ae266c14..364c428aa5 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -144,6 +144,7 @@ src/main/proto/TajoMasterProtocol.proto src/main/proto/TajoWorkerProtocol.proto src/main/proto/InternalTypes.proto + src/main/proto/Plan.proto diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java index a72e2a8a17..518b72f0c9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java @@ -72,7 +72,7 @@ private static EvalNode _transpose(BinaryEval _expr, Column target) { EvalNode lTerm = null; EvalNode rTerm = null; - if (EvalType.isArithmeticOperator(left)) { // we can ensure that left is binary. + if (EvalType.isArithmeticOperator(left.getType())) { // we can ensure that left is binary. // If the left-left term is a variable, the left-right term is transposed. if (EvalTreeUtil.containColumnRef(((BinaryEval)left).getLeftExpr(), target)) { @@ -234,7 +234,7 @@ public static boolean containSingleVar(EvalNode expr) { */ public static PartialBinaryExpr splitLeftTerm(BinaryEval binary) { - if (!(EvalType.isArithmeticOperator(binary))) { + if (!(EvalType.isArithmeticOperator(binary.getType()))) { throw new AlgebraicException("Invalid algebraic operation: " + binary); } @@ -259,7 +259,7 @@ public static PartialBinaryExpr splitLeftTerm(BinaryEval binary) { */ public static PartialBinaryExpr splitRightTerm(BinaryEval binary) { - if (!(EvalType.isArithmeticOperator(binary))) { + if (!(EvalType.isArithmeticOperator(binary.getType()))) { throw new AlgebraicException("Invalid algebraic operation: " + binary); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java index 0b9c7c196c..1a752f1cf6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java @@ -169,6 +169,24 @@ public TajoDataTypes.DataType getValueType() { return RES_TYPE; } + @Override + public int childNum() { + return 3; + } + + @Override + public EvalNode getExpr(int idx) { + if (idx == 0) { + return predicand; + } else if (idx == 1) { + return begin; + } else if (idx == 2) { + return end; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + @Override public String getName() { return "between"; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java index 70d6bb1b9b..3d02b59ac8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java @@ -88,6 +88,7 @@ public T getRightExpr() { return (T) this.rightExpr; } + @Override public EvalNode getExpr(int id) { if (id == 0) { return this.leftExpr; @@ -153,6 +154,11 @@ public DataType getValueType() { return returnType; } + @Override + public int childNum() { + return 2; + } + @Deprecated public void preOrder(EvalNodeVisitor visitor) { visitor.visit(this); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java index cf1acdf64a..fc24c0e956 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java @@ -79,6 +79,22 @@ public DataType getValueType() { return NullDatum.getDataType(); } + @Override + public int childNum() { + return whens.size() + (elseResult != null ? 1 : 0); + } + + @Override + public EvalNode getExpr(int idx) { + if (idx < whens.size()) { + return whens.get(idx); + } else if (idx == whens.size()) { + return elseResult; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + @Override public String getName() { return "?"; @@ -174,6 +190,22 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.BOOLEAN); } + @Override + public int childNum() { + return 2; + } + + @Override + public EvalNode getExpr(int idx) { + if (idx == 0) { + return condition; + } else if (idx == 1) { + return result; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + @Override public String getName() { return "when?"; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java index 2cb530dff0..f164030059 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java @@ -52,7 +52,17 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(datum.type()); } - @Override + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getExpr(int idx) { + return null; + } + + @Override public String getName() { return this.datum.toString(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java index a30d27fca8..9b02103146 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java @@ -46,6 +46,10 @@ public EvalType getType() { } public abstract DataType getValueType(); + + public abstract int childNum(); + + public abstract EvalNode getExpr(int idx); public abstract String getName(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java index 3921a7d81f..cfd832c0b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java @@ -242,7 +242,7 @@ public static boolean isJoinQual(EvalNode expr, boolean includeThetaJoin) { if (expr instanceof BinaryEval) { boolean joinComparator; if (includeThetaJoin) { - joinComparator = EvalType.isComparisonOperator(expr); + joinComparator = EvalType.isComparisonOperator(expr.getType()); } else { joinComparator = expr.getType() == EvalType.EQUAL; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java index 549f8d0fd0..e578e2891a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java @@ -79,8 +79,33 @@ public enum EvalType { this.operatorName = text; } - public static boolean isLogicalOperator(EvalNode evalNode) { - EvalType type = evalNode.getType(); + public static boolean isUnaryOperator(EvalType type) { + boolean match = false; + + match |= type == CAST; + match |= type == IS_NULL; + match |= type == NOT; + match |= type == SIGNED; + + return match; + } + + public static boolean isBinaryOperator(EvalType type) { + boolean match = false; + + match |= isArithmeticOperator(type); + match |= isLogicalOperator(type) && type != NOT; + match |= isComparisonOperator(type) && type != BETWEEN; + + match |= type == IN; + match |= type == LIKE; + match |= type == REGEX; + match |= type == SIMILAR_TO; + + return match; + } + + public static boolean isLogicalOperator(EvalType type) { boolean match = false; match |= type == AND; @@ -90,8 +115,7 @@ public static boolean isLogicalOperator(EvalNode evalNode) { return match; } - public static boolean isComparisonOperator(EvalNode evalNode) { - EvalType type = evalNode.getType(); + public static boolean isComparisonOperator(EvalType type) { boolean match = false; match |= type == EQUAL; @@ -105,8 +129,7 @@ public static boolean isComparisonOperator(EvalNode evalNode) { return match; } - public static boolean isArithmeticOperator(EvalNode evalNode) { - EvalType type = evalNode.getType(); + public static boolean isArithmeticOperator(EvalType type) { boolean match = false; match |= type == PLUS; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java index 20af8542a6..e0a71b75c3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java @@ -59,7 +59,17 @@ public Datum eval(Schema schema, Tuple tuple) { public DataType getValueType() { return column.getDataType(); } - + + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getExpr(int idx) { + return null; + } + public Column getColumnRef() { return column; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java index 0cc8d987c6..32b7f5715e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java @@ -74,6 +74,20 @@ public boolean isDistinct() { public void setArgs(EvalNode [] args) { this.argEvals = args; } + + @Override + public int childNum() { + if (argEvals != null) { + return argEvals.length; + } else { + return 0; + } + } + + @Override + public EvalNode getExpr(int idx) { + return argEvals[idx]; + } public DataType getValueType() { return this.funcDesc.getReturnType(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java index ffaf6b503b..288ffee210 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java @@ -41,6 +41,16 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(values[0].type()); } + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getExpr(int idx) { + return null; + } + @Override public String getName() { return "ROW"; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java index 7c6833fb98..d2b7e9a2b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.eval; import com.google.common.base.Objects; +import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; @@ -37,6 +38,16 @@ public UnaryEval(EvalType type, EvalNode child) { this.child = child; } + @Override + public int childNum() { + return 1; + } + + public EvalNode getExpr(int idx) { + Preconditions.checkArgument(idx != 0, "UnaryEval always has one child."); + return child; + } + public void setChild(EvalNode child) { this.child = child; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java new file mode 100644 index 0000000000..679694998b --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; +import org.apache.tajo.engine.planner.LogicalPlan; + +import java.util.Iterator; +import java.util.Map; +import java.util.SortedMap; +import java.util.Stack; + +public class LogicalPlanConvertor { + public PlanProto.LogicalPlan convert(LogicalPlan plan) { + PlanProto.LogicalPlan.Builder builder = PlanProto.LogicalPlan.newBuilder(); + + return null; + } + + + + public static class PlanProtoBuilder extends BasicLogicalPlanVisitor { + + } + + public static class EvalTreeProtoBuilderContext { + private int seqId = 0; + private Map idMap = Maps.newHashMap(); + private PlanProto.EvalTree.Builder evalTreeBuilder = PlanProto.EvalTree.newBuilder(); + } + + public static class EvalTreeProtoDeserializer { + public EvalNode deserialize(PlanProto.EvalTree tree) { + SortedMap protoMap = Maps.newTreeMap(); + Map evalNodeMap = Maps.newHashMap(); + + for (PlanProto.EvalNode node : tree.getNodesList()) { + protoMap.put(node.getId(), node); + } + + EvalNode current = null; + + Iterator> it = protoMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + + PlanProto.EvalNode protoNode = entry.getValue(); + + EvalType type = EvalType.valueOf(protoNode.getType().name()); + + if (EvalType.isUnaryOperator(type)) { + PlanProto.UnaryEval unaryProto = protoNode.getUnary(); + EvalNode child = evalNodeMap.get(unaryProto.getChildId()); + + switch (type) { + case NOT: + current = new NotEval(child); + break; + case IS_NULL: + current = new IsNullEval(unaryProto.getNegative(), child); + break; + case CAST: + current = new CastEval(child, unaryProto.getCastingType()); + break; + case SIGNED: + current = new SignedEval(unaryProto.getNegative(), child); + break; + default: + throw new RuntimeException("Unknown EvalType: " + type.name()); + } + } else if (EvalType.isBinaryOperator(type)) { + PlanProto.BinaryEval binProto = protoNode.getBinary(); + EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); + EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); + current = new BinaryEval(type, lhs, rhs); + evalNodeMap.put(protoNode.getId(), current); + } else { + throw new RuntimeException("Unknown EvalType: " + type.name()); + } + } + + return current; + } + } + + public static class EvalTreeProtoSerializer extends SimpleEvalNodeVisitor { + + private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalNode) { + int selfId; + if (context.idMap.containsKey(evalNode)) { + selfId = context.idMap.get(evalNode); + } else { + selfId = context.seqId++; + context.idMap.put(evalNode, selfId); + } + + return selfId; + } + + private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { + int [] childIds = new int[evalNode.childNum()]; + for (int i = 0; i < evalNode.childNum(); i++) { + if (context.idMap.containsKey(evalNode.getExpr(i))) { + childIds[i] = context.idMap.get(evalNode.getExpr(i)); + } else { + childIds[i] = context.seqId++; + } + } + return childIds; + } + + public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unaryEval) { + // visiting childs + stack.push(unaryEval); + super.visitUnaryEval(context, stack, unaryEval); + stack.pop(); + + // register childs + int [] childIds = registerGetChildIds(context, unaryEval); + PlanProto.UnaryEval.Builder unaryBuilder = PlanProto.UnaryEval.newBuilder(); + unaryBuilder.setChildId(childIds[0]); + if (unaryEval.getType() == EvalType.IS_NULL) { + IsNullEval isNullEval = (IsNullEval) unaryEval; + unaryBuilder.setNegative(isNullEval.isNot()); + } else if (unaryEval.getType() == EvalType.SIGNED) { + SignedEval signedEval = (SignedEval) unaryEval; + unaryBuilder.setNegative(signedEval.isNegative()); + } else if (unaryEval.getType() == EvalType.CAST) { + CastEval castEval = (CastEval) unaryEval; + unaryBuilder.setCastingType(castEval.getValueType()); + } + + // register itself + int selfId = registerAndGetId(context, unaryEval); + + // build node itself + PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); + nodeBuilder.setId(selfId); + nodeBuilder.setDataType(unaryEval.getValueType()); + nodeBuilder.setType(PlanProto.EvalType.valueOf(unaryEval.getType().name())); + nodeBuilder.setUnary(unaryBuilder); + + // add node to eval tree + context.evalTreeBuilder.addNodes(nodeBuilder); + return unaryEval; + } + + public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binaryEval) { + + // visiting childs + stack.push(binaryEval); + super.visitBinaryEval(context, stack, binaryEval); + stack.pop(); + + // register childs + int [] childIds = registerGetChildIds(context, binaryEval); + PlanProto.BinaryEval.Builder binaryBuilder = PlanProto.BinaryEval.newBuilder(); + binaryBuilder.setLhsId(childIds[0]); + binaryBuilder.setRhsId(childIds[1]); + + // register itself + int selfId = registerAndGetId(context, binaryEval); + + // build node itself + PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); + nodeBuilder.setId(selfId); + nodeBuilder.setDataType(binaryEval.getValueType()); + nodeBuilder.setType(PlanProto.EvalType.valueOf(binaryEval.getType().name())); + nodeBuilder.setBinary(binaryBuilder); + + // add node to eval tree + context.evalTreeBuilder.addNodes(nodeBuilder); + return binaryEval; + } + } + + public static Datum deserialize(PlanProto.Datum datum) { + switch (datum.getType()) { + case BOOLEAN: + return DatumFactory.createBool(datum.getBoolean()); + case INT1: + case INT2: + return DatumFactory.createInt2((short) datum.getInt4()); + case INT4: + return DatumFactory.createInt4(datum.getInt4()); + case INT8: + return DatumFactory.createInt8(datum.getInt8()); + case FLOAT4: + return DatumFactory.createFloat4(datum.getFloat4()); + case FLOAT8: + return DatumFactory.createFloat8(datum.getFloat8()); + case CHAR: + case VARCHAR: + case TEXT: + return DatumFactory.createText(datum.getText()); + case BINARY: + case BLOB: + return DatumFactory.createBlob(datum.getBlob().toByteArray()); + default: + throw new RuntimeException("Unknown data type: " + datum.getType().name()); + } + + } + + public static PlanProto.Datum serialize(Datum datum) { + PlanProto.Datum.Builder builder = PlanProto.Datum.newBuilder(); + + builder.setType(datum.type()); + + switch (datum.type()) { + case BOOLEAN: + builder.setBoolean(datum.asBool()); + break; + case INT1: + case INT2: + case INT4: + builder.setInt4(datum.asInt4()); + break; + case INT8: + builder.setInt8(datum.asInt4()); + break; + case FLOAT4: + builder.setFloat4(datum.asFloat4()); + break; + case FLOAT8: + builder.setFloat8(datum.asFloat8()); + break; + case CHAR: + case VARCHAR: + case TEXT: + builder.setText(datum.asChars()); + break; + case BINARY: + case BLOB: + builder.setBlob(ByteString.copyFrom(datum.asByteArray())); + break; + default: + throw new RuntimeException("Unknown data type: " + datum.type().name()); + } + + return builder.build(); + } +} diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto new file mode 100644 index 0000000000..e175938019 --- /dev/null +++ b/tajo-core/src/main/proto/Plan.proto @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tajo.plan; +option cc_generic_services = false; +option java_package = "org.apache.tajo.engine.plan.proto"; +option java_outer_classname = "PlanProto"; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "PrimitiveProtos.proto"; +import "CatalogProtos.proto"; +import "DataTypes.proto"; + +enum NodeType { + BST_INDEX_SCAN = 0; + EXCEPT = 1; + EXPRS = 2; + DISTINCT_GROUP_BY = 3; + GROUP_BY = 4; + HAVING = 5; + JOIN = 6; + INSERT = 7; + INTERSECT = 8; + LIMIT = 9; + PARTITIONS_SCAN = 10; + PROJECTION = 11; + ROOT = 12; + SCAN = 13; + SELECTION = 14; + SORT = 15; + STORE = 16; + TABLE_SUBQUERY = 17; + UNION = 18; + WINDOW_AGG = 19; + + CREATE_DATABASE = 20; + DROP_DATABASE = 21; + CREATE_TABLE = 22; + DROP_TABLE = 23; + ALTER_TABLESPACE = 24; + ALTER_TABLE = 25; + TRUNCATE_TABLE = 26; +} + +message LogicalPlan { + required KeyValueSetProto adjacentList = 1; +} + +message LogicalNode { + required int32 pid = 1; + required NodeType type = 2; + required SchemaProto in_schema = 3; + required SchemaProto out_schema = 4; + required NodeSpec spec = 5; +} + +message NodeSpec { + optional ScanNode scan = 1; +} + +message ScanNode { + required TableDescProto table = 1; + optional string alias = 2; + required SchemaProto schema = 3; +} + + +enum EvalType { + NOT = 0; + AND = 1; + OR = 2; + EQUAL = 3; + IS_NULL = 4; + NOT_EQUAL = 5; + LTH = 6; + LEQ = 7; + GTH = 8; + GEQ = 9; + PLUS = 10; + MINUS = 11; + MODULAR = 12; + MULTIPLY = 13; + DIVIDE = 14; + + // Binary Bitwise expressions + BIT_AND = 15; + BIT_OR = 16; + BIT_XOR = 17; + + // Function + WINDOW_FUNCTION = 18; + AGG_FUNCTION = 19; + FUNCTION = 20; + + // String operator or pattern matching predicates + LIKE = 21; + SIMILAR_TO = 22; + REGEX = 23; + CONCATENATE = 24; + + // Other predicates + BETWEEN = 25; + CASE = 26; + IF_THEN = 27; + IN = 28; + + // Value or Reference + SIGNED = 29; + CAST = 30; + ROW_CONSTANT = 31; + FIELD = 32; + CONST = 33; +} + +message EvalTree { + repeated EvalNode nodes = 1; +} + +message EvalNode { + required int32 id = 1; + required EvalType type = 2; + required DataType data_type = 3; + + optional UnaryEval unary = 4; // NOT + optional BinaryEval binary = 5; + +} + +message UnaryEval { + required int32 child_id = 1; + required DataType castingType = 2; + required bool negative = 3; +} + +message BinaryEval { + required int32 lhs_id = 1; + required int32 rhs_id = 2; +} + +message Datum { + required Type type = 1; + optional bool boolean = 2; + optional int32 int4 = 3; + optional int64 int8 = 4; + optional float float4 = 5; + optional double float8 = 6; + optional string text = 7; + optional bytes blob = 8; +} \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index da43025f50..ec9887eddf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -31,7 +31,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.*; import static org.junit.Assert.*; -public class TestEvalTree extends ExprTestBase{ +public class TestEvalTree extends ExprTestBase { @Test public void testTupleEval() throws CloneNotSupportedException { ConstEval e1 = new ConstEval(DatumFactory.createInt4(1)); @@ -89,6 +89,16 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(BOOLEAN); } + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getExpr(int idx) { + return null; + } + } public static class MockFalseExpr extends EvalNode { @@ -126,6 +136,16 @@ public void postOrder(EvalNodeVisitor visitor) { public DataType getValueType() { return CatalogUtil.newSimpleDataType(BOOLEAN); } + + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getExpr(int idx) { + return null; + } } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java new file mode 100644 index 0000000000..9a4814d6c1 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import org.apache.tajo.LocalTajoTestingUtility; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.algebra.Expr; +import org.apache.tajo.algebra.OpType; +import org.apache.tajo.algebra.Selection; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.BooleanDatum; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; +import org.apache.tajo.master.TajoMaster; +import org.apache.tajo.master.session.Session; +import org.apache.tajo.util.CommonTestingUtil; +import org.junit.*; + +import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; +import static org.junit.Assert.assertEquals; + +public class TestLogicalPlanConvertor { + static TajoTestingCluster util; + static CatalogService catalog = null; + static SQLAnalyzer analyzer; + static LogicalPlanner planner; + static Session session = LocalTajoTestingUtility.createDummySession(); + + @BeforeClass + public static void setUp() throws Exception { + util = new TajoTestingCluster(); + util.startCatalogCluster(); + catalog = util.getMiniCatalogCluster().getCatalog(); + for (FunctionDesc funcDesc : TajoMaster.initBuiltinFunctions()) { + catalog.createFunction(funcDesc); + } + catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234/warehouse"); + catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); + + Schema schema = new Schema(); + schema.addColumn("name", TajoDataTypes.Type.TEXT); + schema.addColumn("score", TajoDataTypes.Type.INT4); + schema.addColumn("age", TajoDataTypes.Type.INT4); + + TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); + TableDesc desc = new TableDesc( + CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, + CommonTestingUtil.getTestDir()); + catalog.createTable(desc); + + analyzer = new SQLAnalyzer(); + planner = new LogicalPlanner(catalog); + } + + @AfterClass + public static void tearDown() throws Exception { + util.shutdownCatalogCluster(); + } + + public static Target[] getRawTargets(String query) { + Expr expr = analyzer.parse(query); + LogicalPlan plan = null; + try { + plan = planner.createPlan(session, expr); + } catch (PlanningException e) { + e.printStackTrace(); + } + + return plan.getRootBlock().getRawTargets(); + } + + public static EvalNode getRootSelection(String query) throws PlanningException { + Expr block = analyzer.parse(query); + LogicalPlan plan = null; + try { + plan = planner.createPlan(session, block); + } catch (PlanningException e) { + e.printStackTrace(); + } + + Selection selection = plan.getRootBlock().getSingletonExpr(OpType.Filter); + return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual(), + NameResolvingMode.RELS_AND_SUBEXPRS); + } + + @Test + public void testConvert() throws Exception { + Target [] targets = getRawTargets("select 1 + 2"); + + } + + @Test + public void testDatumConvert() throws Exception { + assertSerializationDatum(DatumFactory.createBool(true)); + assertSerializationDatum(DatumFactory.createBool(false)); + assertSerializationDatum(DatumFactory.createInt2((short) 1)); + assertSerializationDatum(DatumFactory.createInt4(1980)); + assertSerializationDatum(DatumFactory.createInt8(19800401)); + assertSerializationDatum(DatumFactory.createFloat4(3.14f)); + assertSerializationDatum(DatumFactory.createFloat8(3.141592d)); + assertSerializationDatum(DatumFactory.createText("Apache Tajo")); + assertSerializationDatum(DatumFactory.createBlob("Apache Tajo".getBytes())); + } + + private static void assertSerializationDatum(Datum datum) { + PlanProto.Datum converted = LogicalPlanConvertor.serialize(datum); + assertEquals(datum, LogicalPlanConvertor.deserialize(converted)); + } + + @Test + public void testConvertDatum() throws Exception { + + } +} diff --git a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java index 15413b06a6..107285aea9 100644 --- a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java +++ b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java @@ -31,6 +31,7 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.exception.VerifyException; +import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.master.session.Session; @@ -145,6 +146,10 @@ public JitVecTestServerProtocol.PlanResponse requestPlan(RpcController controlle } builder.setSerializedPlan(plan.getRootBlock().getRoot().toJson()); + + System.out.println("\n\n======================================================================\n\n"); + System.out.println(CoreGsonHelper.getPrettyInstance().toJson(plan.getRootBlock().getRoot())); + System.out.println("\n\n======================================================================\n\n"); } catch (PlanningException e) { if (e.getMessage() != null) { LOG.error(e.getMessage()); From 0e910132b3d3349e8af18632f6759fbbb5a04429 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 13 Aug 2014 01:55:44 +0900 Subject: [PATCH 03/31] Added (de)serializer for const eval. --- .../engine/plan/LogicalPlanConvertor.java | 63 ++++++++++++------- tajo-core/src/main/proto/Plan.proto | 5 ++ .../engine/plan/TestLogicalPlanConvertor.java | 8 +++ 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index 679694998b..f45f9d4f6b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -45,6 +45,17 @@ public static class PlanProtoBuilder extends BasicLogicalPlanVisitor { } + public static PlanProto.EvalTree serialize(EvalNode evalNode) { + EvalTreeProtoBuilderContext context = new EvalTreeProtoBuilderContext(); + EvalTreeProtoSerializer serializer = new EvalTreeProtoSerializer(); + serializer.visit(context, evalNode, new Stack()); + return context.evalTreeBuilder.build(); + } + + public static EvalNode deserialize(PlanProto.EvalTree evalTree) { + return EvalTreeProtoDeserializer.deserialize(evalTree); + } + public static class EvalTreeProtoBuilderContext { private int seqId = 0; private Map idMap = Maps.newHashMap(); @@ -52,7 +63,7 @@ public static class EvalTreeProtoBuilderContext { } public static class EvalTreeProtoDeserializer { - public EvalNode deserialize(PlanProto.EvalTree tree) { + public static EvalNode deserialize(PlanProto.EvalTree tree) { SortedMap protoMap = Maps.newTreeMap(); Map evalNodeMap = Maps.newHashMap(); @@ -96,6 +107,10 @@ public EvalNode deserialize(PlanProto.EvalTree tree) { EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); current = new BinaryEval(type, lhs, rhs); evalNodeMap.put(protoNode.getId(), current); + } else if (type == EvalType.CONST) { + PlanProto.ConstEval constProto = protoNode.getConst(); + current = new ConstEval(LogicalPlanConvertor.deserialize(constProto.getValue())); + evalNodeMap.put(protoNode.getId(), current); } else { throw new RuntimeException("Unknown EvalType: " + type.name()); } @@ -131,6 +146,14 @@ private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalN return childIds; } + private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { + PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); + nodeBuilder.setId(id); + nodeBuilder.setDataType(eval.getValueType()); + nodeBuilder.setType(PlanProto.EvalType.valueOf(eval.getType().name())); + return nodeBuilder; + } + public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unaryEval) { // visiting childs stack.push(unaryEval); @@ -152,18 +175,11 @@ public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack) { + int selfId = registerAndGetId(context, constEval); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constEval); + builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constEval.getValue()))); + context.evalTreeBuilder.addNodes(builder); + return constEval; + } } public static Datum deserialize(PlanProto.Datum datum) { @@ -221,7 +237,6 @@ public static Datum deserialize(PlanProto.Datum datum) { default: throw new RuntimeException("Unknown data type: " + datum.getType().name()); } - } public static PlanProto.Datum serialize(Datum datum) { diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index e175938019..97e37828c4 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -139,6 +139,7 @@ message EvalNode { optional UnaryEval unary = 4; // NOT optional BinaryEval binary = 5; + optional ConstEval const = 6; } @@ -148,6 +149,10 @@ message UnaryEval { required bool negative = 3; } +message ConstEval { + required Datum value = 1; +} + message BinaryEval { required int32 lhs_id = 1; required int32 rhs_id = 2; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java index 9a4814d6c1..1e388496db 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java @@ -113,7 +113,10 @@ public static EvalNode getRootSelection(String query) throws PlanningException { @Test public void testConvert() throws Exception { Target [] targets = getRawTargets("select 1 + 2"); + assertSerializationOfEvalNode(targets[0].getEvalTree()); + targets = getRawTargets("select l_orderkey + l_partkey from lineitem"); + assertSerializationOfEvalNode(targets[0].getEvalTree()); } @Test @@ -134,6 +137,11 @@ private static void assertSerializationDatum(Datum datum) { assertEquals(datum, LogicalPlanConvertor.deserialize(converted)); } + private static void assertSerializationOfEvalNode(EvalNode evalNode) { + PlanProto.EvalTree converted = LogicalPlanConvertor.serialize(evalNode); + assertEquals(evalNode, LogicalPlanConvertor.deserialize(converted)); + } + @Test public void testConvertDatum() throws Exception { From 1bde69303e69582b81ecaeec7f08aa0551abe8fb Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 13 Aug 2014 02:20:15 +0900 Subject: [PATCH 04/31] Added (de)serializer for field eval. --- .../engine/plan/LogicalPlanConvertor.java | 21 ++++++++++++++-- tajo-core/src/main/proto/Plan.proto | 9 ++++--- .../engine/plan/TestLogicalPlanConvertor.java | 25 +++---------------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index f45f9d4f6b..d47c3b0aa2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -20,6 +20,8 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.eval.*; @@ -63,6 +65,7 @@ public static class EvalTreeProtoBuilderContext { } public static class EvalTreeProtoDeserializer { + public static EvalNode deserialize(PlanProto.EvalTree tree) { SortedMap protoMap = Maps.newTreeMap(); Map evalNodeMap = Maps.newHashMap(); @@ -101,19 +104,25 @@ public static EvalNode deserialize(PlanProto.EvalTree tree) { default: throw new RuntimeException("Unknown EvalType: " + type.name()); } + } else if (EvalType.isBinaryOperator(type)) { PlanProto.BinaryEval binProto = protoNode.getBinary(); EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); current = new BinaryEval(type, lhs, rhs); - evalNodeMap.put(protoNode.getId(), current); + } else if (type == EvalType.CONST) { PlanProto.ConstEval constProto = protoNode.getConst(); current = new ConstEval(LogicalPlanConvertor.deserialize(constProto.getValue())); - evalNodeMap.put(protoNode.getId(), current); + + } else if (type == EvalType.FIELD) { + CatalogProtos.ColumnProto columnProto = protoNode.getField(); + current = new FieldEval(new Column(columnProto)); } else { throw new RuntimeException("Unknown EvalType: " + type.name()); } + + evalNodeMap.put(protoNode.getId(), current); } return current; @@ -210,6 +219,14 @@ public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval constE context.evalTreeBuilder.addNodes(builder); return constEval; } + + public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval fieldEval) { + int selfId = registerAndGetId(context, fieldEval); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, fieldEval); + builder.setField(fieldEval.getColumnRef().getProto()); + context.evalTreeBuilder.addNodes(builder); + return fieldEval; + } } public static Datum deserialize(PlanProto.Datum datum) { diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index 97e37828c4..9b9868b704 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -140,6 +140,7 @@ message EvalNode { optional UnaryEval unary = 4; // NOT optional BinaryEval binary = 5; optional ConstEval const = 6; + optional ColumnProto field = 7; // field eval } @@ -149,15 +150,15 @@ message UnaryEval { required bool negative = 3; } -message ConstEval { - required Datum value = 1; -} - message BinaryEval { required int32 lhs_id = 1; required int32 rhs_id = 2; } +message ConstEval { + required Datum value = 1; +} + message Datum { required Type type = 1; optional bool boolean = 2; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java index 1e388496db..cc6f3b8886 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java @@ -21,6 +21,7 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.TpchTestBase; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.OpType; import org.apache.tajo.algebra.Selection; @@ -55,25 +56,8 @@ public class TestLogicalPlanConvertor { @BeforeClass public static void setUp() throws Exception { - util = new TajoTestingCluster(); - util.startCatalogCluster(); - catalog = util.getMiniCatalogCluster().getCatalog(); - for (FunctionDesc funcDesc : TajoMaster.initBuiltinFunctions()) { - catalog.createFunction(funcDesc); - } - catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:1234/warehouse"); - catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - - Schema schema = new Schema(); - schema.addColumn("name", TajoDataTypes.Type.TEXT); - schema.addColumn("score", TajoDataTypes.Type.INT4); - schema.addColumn("age", TajoDataTypes.Type.INT4); - - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); - TableDesc desc = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, - CommonTestingUtil.getTestDir()); - catalog.createTable(desc); + util = TpchTestBase.getInstance().getTestingCluster(); + catalog = util.getMaster().getCatalog(); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); @@ -81,14 +65,13 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { - util.shutdownCatalogCluster(); } public static Target[] getRawTargets(String query) { Expr expr = analyzer.parse(query); LogicalPlan plan = null; try { - plan = planner.createPlan(session, expr); + plan = planner.createPlan(session, expr, true); } catch (PlanningException e) { e.printStackTrace(); } From a5e3317f88f87a1182a2a1dbc9d6a805bc47b5a4 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 13 Aug 2014 03:12:27 +0900 Subject: [PATCH 05/31] Added (de)serializer for function. --- .../java/org/apache/tajo/datum/Datum.java | 3 +- .../org/apache/tajo/engine/eval/EvalType.java | 10 ++ .../apache/tajo/engine/eval/UnaryEval.java | 2 +- .../engine/plan/LogicalPlanConvertor.java | 132 +++++++++++++----- tajo-core/src/main/proto/Plan.proto | 21 ++- .../apache/tajo/engine/eval/ExprTestBase.java | 14 ++ .../engine/plan/TestLogicalPlanConvertor.java | 33 ++--- 7 files changed, 155 insertions(+), 60 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java index 19acafc243..a6c97727b9 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java @@ -69,9 +69,8 @@ public short asInt2() { throw new InvalidCastException(type, Type.INT2); } public int asInt4() { - throw new InvalidCastException(type, Type.INT1); + throw new InvalidCastException(type, Type.INT4); } - public long asInt8() { throw new InvalidCastException(type, Type.INT8); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java index e578e2891a..7a3eca09bf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java @@ -141,6 +141,16 @@ public static boolean isArithmeticOperator(EvalType type) { return match; } + public static boolean isFunction(EvalType type) { + boolean match = false; + + match |= type == FUNCTION; + match |= type == AGG_FUNCTION; + match |= type == WINDOW_FUNCTION; + + return match; + } + public String getOperatorName() { return operatorName != null ? operatorName : name(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java index d2b7e9a2b9..6b0bf6d09e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java @@ -44,7 +44,7 @@ public int childNum() { } public EvalNode getExpr(int idx) { - Preconditions.checkArgument(idx != 0, "UnaryEval always has one child."); + Preconditions.checkArgument(idx == 0, "UnaryEval always has one child."); return child; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index d47c3b0aa2..b677e4f381 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -21,13 +21,21 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.function.AggFunction; +import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.exception.InternalException; +import org.apache.tajo.util.datetime.DateTimeUtil; import java.util.Iterator; import java.util.Map; @@ -118,6 +126,34 @@ public static EvalNode deserialize(PlanProto.EvalTree tree) { } else if (type == EvalType.FIELD) { CatalogProtos.ColumnProto columnProto = protoNode.getField(); current = new FieldEval(new Column(columnProto)); + + } else if (EvalType.isFunction(type)) { + PlanProto.FunctionEval funcProto = protoNode.getFunction(); + + EvalNode [] params = new EvalNode[funcProto.getParamIdsCount()]; + for (int i = 0; i < funcProto.getParamIdsCount(); i++) { + params[i] = evalNodeMap.get(funcProto.getParamIds(i)); + } + + FunctionDesc funcDesc = null; + try { + funcDesc = new FunctionDesc(funcProto.getFuncion()); + if (type == EvalType.FUNCTION) { + GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); + current = new GeneralFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { + AggFunction instance = (AggFunction) funcDesc.newInstance(); + if (type == EvalType.AGG_FUNCTION) { + current = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + } else { + current = new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params, null); + } + } + } catch (ClassNotFoundException cnfe) { + throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + } catch (InternalException ie) { + throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + } } else { throw new RuntimeException("Unknown EvalType: " + type.name()); } @@ -163,69 +199,91 @@ private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { return nodeBuilder; } - public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unaryEval) { + public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unary) { // visiting childs - stack.push(unaryEval); - super.visitUnaryEval(context, stack, unaryEval); + stack.push(unary); + super.visitUnaryEval(context, stack, unary); stack.pop(); // register childs - int [] childIds = registerGetChildIds(context, unaryEval); + int [] childIds = registerGetChildIds(context, unary); PlanProto.UnaryEval.Builder unaryBuilder = PlanProto.UnaryEval.newBuilder(); unaryBuilder.setChildId(childIds[0]); - if (unaryEval.getType() == EvalType.IS_NULL) { - IsNullEval isNullEval = (IsNullEval) unaryEval; + if (unary.getType() == EvalType.IS_NULL) { + IsNullEval isNullEval = (IsNullEval) unary; unaryBuilder.setNegative(isNullEval.isNot()); - } else if (unaryEval.getType() == EvalType.SIGNED) { - SignedEval signedEval = (SignedEval) unaryEval; + } else if (unary.getType() == EvalType.SIGNED) { + SignedEval signedEval = (SignedEval) unary; unaryBuilder.setNegative(signedEval.isNegative()); - } else if (unaryEval.getType() == EvalType.CAST) { - CastEval castEval = (CastEval) unaryEval; + } else if (unary.getType() == EvalType.CAST) { + CastEval castEval = (CastEval) unary; unaryBuilder.setCastingType(castEval.getValueType()); } // register itself and add eval to eval tree - int selfId = registerAndGetId(context, unaryEval); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, unaryEval); + int selfId = registerAndGetId(context, unary); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, unary); builder.setUnary(unaryBuilder); context.evalTreeBuilder.addNodes(builder); - return unaryEval; + return unary; } - public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binaryEval) { + public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binary) { // visiting childs - stack.push(binaryEval); - super.visitBinaryEval(context, stack, binaryEval); + stack.push(binary); + super.visitBinaryEval(context, stack, binary); stack.pop(); // register childs - int [] childIds = registerGetChildIds(context, binaryEval); + int [] childIds = registerGetChildIds(context, binary); PlanProto.BinaryEval.Builder binaryBuilder = PlanProto.BinaryEval.newBuilder(); binaryBuilder.setLhsId(childIds[0]); binaryBuilder.setRhsId(childIds[1]); - int selfId = registerAndGetId(context, binaryEval); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, binaryEval); + int selfId = registerAndGetId(context, binary); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, binary); builder.setBinary(binaryBuilder); context.evalTreeBuilder.addNodes(builder); - return binaryEval; + return binary; + } + + public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval constant, Stack stack) { + int selfId = registerAndGetId(context, constant); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constant); + builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constant.getValue()))); + context.evalTreeBuilder.addNodes(builder); + return constant; } - public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval constEval, Stack stack) { - int selfId = registerAndGetId(context, constEval); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constEval); - builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constEval.getValue()))); + public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { + int selfId = registerAndGetId(context, field); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, field); + builder.setField(field.getColumnRef().getProto()); context.evalTreeBuilder.addNodes(builder); - return constEval; + return field; } - public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval fieldEval) { - int selfId = registerAndGetId(context, fieldEval); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, fieldEval); - builder.setField(fieldEval.getColumnRef().getProto()); + public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { + + stack.push(function); + super.visitFuncCall(context, function, stack); + stack.pop(); + + // register childs + int [] childIds = registerGetChildIds(context, function); + PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); + funcBuilder.setFuncion(function.getFuncDesc().getProto()); + for (int i = 0; i < childIds.length; i++) { + funcBuilder.addParamIds(i); + } + + int selfId = registerAndGetId(context, function); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, function); + builder.setFunction(funcBuilder); context.evalTreeBuilder.addNodes(builder); - return fieldEval; + + return function; } } @@ -233,6 +291,8 @@ public static Datum deserialize(PlanProto.Datum datum) { switch (datum.getType()) { case BOOLEAN: return DatumFactory.createBool(datum.getBoolean()); + case CHAR: + return DatumFactory.createChar(datum.getText()); case INT1: case INT2: return DatumFactory.createInt2((short) datum.getInt4()); @@ -244,10 +304,15 @@ public static Datum deserialize(PlanProto.Datum datum) { return DatumFactory.createFloat4(datum.getFloat4()); case FLOAT8: return DatumFactory.createFloat8(datum.getFloat8()); - case CHAR: case VARCHAR: case TEXT: return DatumFactory.createText(datum.getText()); + case TIMESTAMP: + return new TimestampDatum(datum.getInt8()); + case DATE: + return DatumFactory.createDate(datum.getInt4()); + case TIME: + return DatumFactory.createTime(datum.getInt8()); case BINARY: case BLOB: return DatumFactory.createBlob(datum.getBlob().toByteArray()); @@ -268,10 +333,13 @@ public static PlanProto.Datum serialize(Datum datum) { case INT1: case INT2: case INT4: + case DATE: builder.setInt4(datum.asInt4()); break; case INT8: - builder.setInt8(datum.asInt4()); + case TIMESTAMP: + case TIME: + builder.setInt8(datum.asInt8()); break; case FLOAT4: builder.setFloat4(datum.asFloat4()); diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index 9b9868b704..a59e943a38 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -141,13 +141,13 @@ message EvalNode { optional BinaryEval binary = 5; optional ConstEval const = 6; optional ColumnProto field = 7; // field eval - + optional FunctionEval function = 8; } message UnaryEval { required int32 child_id = 1; - required DataType castingType = 2; - required bool negative = 3; + optional DataType castingType = 2; + optional bool negative = 3; } message BinaryEval { @@ -159,13 +159,24 @@ message ConstEval { required Datum value = 1; } +message FunctionEval { + required FunctionDescProto funcion = 1; + repeated int32 paramIds = 2; +} + message Datum { required Type type = 1; optional bool boolean = 2; - optional int32 int4 = 3; - optional int64 int8 = 4; + optional int32 int4 = 3; // int4, date + optional int64 int8 = 4; // int8, timestamp, and time optional float float4 = 5; optional double float8 = 6; optional string text = 7; optional bytes blob = 8; + optional Interval interval = 12; +} + +message Interval { + optional int32 month = 1 [default = 0]; + optional int64 msec = 2; } \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index ad80ddfafe..567f5a5111 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -31,6 +31,7 @@ import org.apache.tajo.datum.*; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.plan.TestLogicalPlanConvertor; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.TajoMaster; @@ -52,6 +53,7 @@ import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; public class ExprTestBase { private static TajoTestingCluster util; @@ -132,9 +134,21 @@ private static Target[] getRawTargets(String query, boolean condition) throws Pl if (targets == null) { throw new PlanningException("Wrong query statement or query plan: " + parsedResults.get(0).getHistoryStatement()); } + + // Trying regression test for cloning, (de)serialization for json and protocol buffer + for (Target t : targets) { + try { + assertEquals(t.getEvalTree(), t.getEvalTree().clone()); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + } for (Target t : targets) { assertJsonSerDer(t.getEvalTree()); } + for (Target t : targets) { + TestLogicalPlanConvertor.assertEvalNodeProtoSerder(t.getEvalTree()); + } return targets; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java index cc6f3b8886..b4c94c6bcc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java @@ -19,16 +19,12 @@ package org.apache.tajo.engine.plan; import org.apache.tajo.LocalTajoTestingUtility; -import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.OpType; import org.apache.tajo.algebra.Selection; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.BooleanDatum; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.eval.EvalNode; @@ -39,12 +35,9 @@ import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.engine.planner.Target; import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; -import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; -import org.apache.tajo.util.CommonTestingUtil; import org.junit.*; -import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.junit.Assert.assertEquals; public class TestLogicalPlanConvertor { @@ -96,31 +89,31 @@ public static EvalNode getRootSelection(String query) throws PlanningException { @Test public void testConvert() throws Exception { Target [] targets = getRawTargets("select 1 + 2"); - assertSerializationOfEvalNode(targets[0].getEvalTree()); + assertEvalNodeProtoSerder(targets[0].getEvalTree()); targets = getRawTargets("select l_orderkey + l_partkey from lineitem"); - assertSerializationOfEvalNode(targets[0].getEvalTree()); + assertEvalNodeProtoSerder(targets[0].getEvalTree()); } @Test public void testDatumConvert() throws Exception { - assertSerializationDatum(DatumFactory.createBool(true)); - assertSerializationDatum(DatumFactory.createBool(false)); - assertSerializationDatum(DatumFactory.createInt2((short) 1)); - assertSerializationDatum(DatumFactory.createInt4(1980)); - assertSerializationDatum(DatumFactory.createInt8(19800401)); - assertSerializationDatum(DatumFactory.createFloat4(3.14f)); - assertSerializationDatum(DatumFactory.createFloat8(3.141592d)); - assertSerializationDatum(DatumFactory.createText("Apache Tajo")); - assertSerializationDatum(DatumFactory.createBlob("Apache Tajo".getBytes())); + assertDatumProtoSerder(DatumFactory.createBool(true)); + assertDatumProtoSerder(DatumFactory.createBool(false)); + assertDatumProtoSerder(DatumFactory.createInt2((short) 1)); + assertDatumProtoSerder(DatumFactory.createInt4(1980)); + assertDatumProtoSerder(DatumFactory.createInt8(19800401)); + assertDatumProtoSerder(DatumFactory.createFloat4(3.14f)); + assertDatumProtoSerder(DatumFactory.createFloat8(3.141592d)); + assertDatumProtoSerder(DatumFactory.createText("Apache Tajo")); + assertDatumProtoSerder(DatumFactory.createBlob("Apache Tajo".getBytes())); } - private static void assertSerializationDatum(Datum datum) { + public static void assertDatumProtoSerder(Datum datum) { PlanProto.Datum converted = LogicalPlanConvertor.serialize(datum); assertEquals(datum, LogicalPlanConvertor.deserialize(converted)); } - private static void assertSerializationOfEvalNode(EvalNode evalNode) { + public static void assertEvalNodeProtoSerder(EvalNode evalNode) { PlanProto.EvalTree converted = LogicalPlanConvertor.serialize(evalNode); assertEquals(evalNode, LogicalPlanConvertor.deserialize(converted)); } From ecbdb340954940b01ab48a602191c5d579bc7a84 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 14 Aug 2014 07:33:22 +0900 Subject: [PATCH 06/31] Added (de)serializer for in clause and rowconstant. --- .../engine/plan/LogicalPlanConvertor.java | 63 +++++++++++++------ tajo-core/src/main/proto/Plan.proto | 8 +++ 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index b677e4f381..a63577deae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -117,12 +117,25 @@ public static EvalNode deserialize(PlanProto.EvalTree tree) { PlanProto.BinaryEval binProto = protoNode.getBinary(); EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); - current = new BinaryEval(type, lhs, rhs); + + if (type == EvalType.IN) { + current = new InEval(lhs, (RowConstantEval) rhs, binProto.getNegative()); + } else { + current = new BinaryEval(type, lhs, rhs); + } } else if (type == EvalType.CONST) { PlanProto.ConstEval constProto = protoNode.getConst(); current = new ConstEval(LogicalPlanConvertor.deserialize(constProto.getValue())); + } else if (type == EvalType.ROW_CONSTANT) { + PlanProto.RowConstEval rowConstProto = protoNode.getRowConst(); + Datum [] values = new Datum[rowConstProto.getValuesCount()]; + for (int i = 0; i < rowConstProto.getValuesCount(); i++) { + values[i] = LogicalPlanConvertor.deserialize(rowConstProto.getValues(i)); + } + current = new RowConstantEval(values); + } else if (type == EvalType.FIELD) { CatalogProtos.ColumnProto columnProto = protoNode.getField(); current = new FieldEval(new Column(columnProto)); @@ -199,14 +212,13 @@ private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { return nodeBuilder; } + @Override public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unary) { - // visiting childs - stack.push(unary); + // visiting and registering childs super.visitUnaryEval(context, stack, unary); - stack.pop(); - - // register childs int [] childIds = registerGetChildIds(context, unary); + + // building itself PlanProto.UnaryEval.Builder unaryBuilder = PlanProto.UnaryEval.newBuilder(); unaryBuilder.setChildId(childIds[0]); if (unary.getType() == EvalType.IS_NULL) { @@ -220,7 +232,7 @@ public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binary) { - - // visiting childs - stack.push(binary); + // visiting and registering childs super.visitBinaryEval(context, stack, binary); - stack.pop(); - - // register childs int [] childIds = registerGetChildIds(context, binary); + + // building itself PlanProto.BinaryEval.Builder binaryBuilder = PlanProto.BinaryEval.newBuilder(); binaryBuilder.setLhsId(childIds[0]); binaryBuilder.setRhsId(childIds[1]); + // registering itself and building EvalNode int selfId = registerAndGetId(context, binary); PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, binary); builder.setBinary(binaryBuilder); @@ -248,6 +259,7 @@ public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack) { int selfId = registerAndGetId(context, constant); PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constant); @@ -256,6 +268,22 @@ public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval consta return constant; } + @Override + public EvalNode visitRowConstant(EvalTreeProtoBuilderContext context, RowConstantEval rowConst, + Stack stack) { + + PlanProto.RowConstEval.Builder rowConstBuilder = PlanProto.RowConstEval.newBuilder(); + for (Datum d : rowConst.getValues()) { + rowConstBuilder.addValues(LogicalPlanConvertor.serialize(d)); + } + + int selfId = registerAndGetId(context, rowConst); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, rowConst); + builder.setRowConst(rowConstBuilder); + context.evalTreeBuilder.addNodes(builder); + return rowConst; + } + public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { int selfId = registerAndGetId(context, field); PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, field); @@ -265,19 +293,18 @@ public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack } public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { - - stack.push(function); + // visiting and registering childs super.visitFuncCall(context, function, stack); - stack.pop(); - - // register childs int [] childIds = registerGetChildIds(context, function); + + // building itself PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); funcBuilder.setFuncion(function.getFuncDesc().getProto()); for (int i = 0; i < childIds.length; i++) { funcBuilder.addParamIds(i); } + // registering itself and building EvalNode int selfId = registerAndGetId(context, function); PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, function); builder.setFunction(funcBuilder); diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index a59e943a38..7da054470c 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -142,6 +142,7 @@ message EvalNode { optional ConstEval const = 6; optional ColumnProto field = 7; // field eval optional FunctionEval function = 8; + optional RowConstEval rowConst = 9; } message UnaryEval { @@ -153,12 +154,19 @@ message UnaryEval { message BinaryEval { required int32 lhs_id = 1; required int32 rhs_id = 2; + optional bool negative = 3 [default = false]; } + + message ConstEval { required Datum value = 1; } +message RowConstEval { + repeated Datum values = 1; +} + message FunctionEval { required FunctionDescProto funcion = 1; repeated int32 paramIds = 2; From 95cd8ffa358da732d81c927a0dedb20742376d63 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 15 Aug 2014 12:50:03 +0900 Subject: [PATCH 07/31] Fixed (de)serialization bug for function, and implemented IntervalDatum. --- .../org/apache/tajo/engine/eval/EvalType.java | 1 + .../engine/plan/LogicalPlanConvertor.java | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java index 7a3eca09bf..500928a869 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java @@ -97,6 +97,7 @@ public static boolean isBinaryOperator(EvalType type) { match |= isLogicalOperator(type) && type != NOT; match |= isComparisonOperator(type) && type != BETWEEN; + match |= type == CONCATENATE; match |= type == IN; match |= type == LIKE; match |= type == REGEX; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index a63577deae..54fc3cb3af 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -24,9 +24,7 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.TimestampDatum; +import org.apache.tajo.datum.*; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.function.AggFunction; import org.apache.tajo.engine.function.GeneralFunction; @@ -118,9 +116,11 @@ public static EvalNode deserialize(PlanProto.EvalTree tree) { EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); - if (type == EvalType.IN) { + switch (type) { + case IN: current = new InEval(lhs, (RowConstantEval) rhs, binProto.getNegative()); - } else { + break; + default: current = new BinaryEval(type, lhs, rhs); } @@ -301,7 +301,7 @@ public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); funcBuilder.setFuncion(function.getFuncDesc().getProto()); for (int i = 0; i < childIds.length; i++) { - funcBuilder.addParamIds(i); + funcBuilder.addParamIds(childIds[i]); } // registering itself and building EvalNode @@ -343,6 +343,10 @@ public static Datum deserialize(PlanProto.Datum datum) { case BINARY: case BLOB: return DatumFactory.createBlob(datum.getBlob().toByteArray()); + case INTERVAL: + return new IntervalDatum(datum.getInterval().getMonth(), datum.getInterval().getMsec()); + case NULL_TYPE: + return NullDatum.get(); default: throw new RuntimeException("Unknown data type: " + datum.getType().name()); } @@ -354,6 +358,8 @@ public static PlanProto.Datum serialize(Datum datum) { builder.setType(datum.type()); switch (datum.type()) { + case NULL_TYPE: + break; case BOOLEAN: builder.setBoolean(datum.asBool()); break; @@ -383,6 +389,13 @@ public static PlanProto.Datum serialize(Datum datum) { case BLOB: builder.setBlob(ByteString.copyFrom(datum.asByteArray())); break; + case INTERVAL: + IntervalDatum interval = (IntervalDatum) datum; + PlanProto.Interval.Builder intervalBuilder = PlanProto.Interval.newBuilder(); + intervalBuilder.setMonth(interval.getMonths()); + intervalBuilder.setMsec(interval.getMilliSeconds()); + builder.setInterval(intervalBuilder); + break; default: throw new RuntimeException("Unknown data type: " + datum.type().name()); } From c58f469aa8aa31df1109374ceb9c158d9a77213c Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 15 Aug 2014 20:25:14 +0900 Subject: [PATCH 08/31] Added (de)serialization for between. --- .../engine/eval/BetweenPredicateEval.java | 20 ++++++++++++ .../engine/plan/LogicalPlanConvertor.java | 32 +++++++++++++++++++ tajo-core/src/main/proto/Plan.proto | 9 +++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java index 1a752f1cf6..86af4ba357 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java @@ -46,6 +46,26 @@ public BetweenPredicateEval(boolean not, boolean symmetric, EvalNode predicand, this.end = end; } + public boolean isNot() { + return not; + } + + public boolean isSymmetric() { + return symmetric; + } + + public void setExpr(int idx, EvalNode node) { + if (idx == 0) { + predicand = node; + } else if (idx == 1) { + begin = node; + } else if (idx == 2) { + end = node; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + public void setPredicand(EvalNode predicand) { this.predicand = predicand; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index 54fc3cb3af..99f86dd0da 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.plan; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import org.apache.tajo.catalog.Column; @@ -140,6 +141,13 @@ public static EvalNode deserialize(PlanProto.EvalTree tree) { CatalogProtos.ColumnProto columnProto = protoNode.getField(); current = new FieldEval(new Column(columnProto)); + } else if (type == EvalType.BETWEEN) { + PlanProto.BetweenEval betweenProto = protoNode.getBetween(); + current = new BetweenPredicateEval(betweenProto.getNegative(), betweenProto.getSymmetric(), + evalNodeMap.get(betweenProto.getPredicand()), + evalNodeMap.get(betweenProto.getBegin()), + evalNodeMap.get(betweenProto.getEnd())); + } else if (EvalType.isFunction(type)) { PlanProto.FunctionEval funcProto = protoNode.getFunction(); @@ -292,6 +300,30 @@ public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack return field; } + public EvalNode visitBetween(EvalTreeProtoBuilderContext context, BetweenPredicateEval between, + Stack stack) { + // visiting and registering childs + super.visitBetween(context, between, stack); + int [] childIds = registerGetChildIds(context, between); + Preconditions.checkState(childIds.length == 3, "Between must have three childs, but there are " + childIds.length + + " child nodes"); + + // building itself + PlanProto.BetweenEval.Builder betweenBuilder = PlanProto.BetweenEval.newBuilder(); + betweenBuilder.setNegative(between.isNot()); + betweenBuilder.setSymmetric(between.isSymmetric()); + betweenBuilder.setPredicand(childIds[0]); + betweenBuilder.setBegin(childIds[1]); + betweenBuilder.setEnd(childIds[2]); + + int selfId = registerAndGetId(context, between); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, between); + builder.setBetween(betweenBuilder); + context.evalTreeBuilder.addNodes(builder); + return between; + + } + public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { // visiting and registering childs super.visitFuncCall(context, function, stack); diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index 7da054470c..a8776cdf3a 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -143,6 +143,7 @@ message EvalNode { optional ColumnProto field = 7; // field eval optional FunctionEval function = 8; optional RowConstEval rowConst = 9; + optional BetweenEval between = 10; } message UnaryEval { @@ -157,7 +158,13 @@ message BinaryEval { optional bool negative = 3 [default = false]; } - +message BetweenEval { + required int32 predicand = 1; + required int32 begin = 2; + required int32 end = 3; + optional bool negative = 4 [default = false]; + optional bool symmetric = 5 [default = false]; +} message ConstEval { required Datum value = 1; From 621967232d90dd0768fcfbd9e78aa64497fb0478 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 15 Aug 2014 21:41:30 +0900 Subject: [PATCH 09/31] Added (de)serialization for case when. --- .../apache/tajo/engine/eval/CaseWhenEval.java | 7 ++- .../engine/plan/LogicalPlanConvertor.java | 62 ++++++++++++++++++- .../tajo/engine/planner/ExprAnnotator.java | 2 +- tajo-core/src/main/proto/Plan.proto | 12 ++++ 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java index fc24c0e956..27769a4087 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java @@ -28,6 +28,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.json.CoreGsonHelper; +import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.json.GsonObject; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.TUtil; @@ -43,7 +44,11 @@ public CaseWhenEval() { super(EvalType.CASE); } - public void addWhen(EvalNode condition, EvalNode result) { + public void addIfCond(IfThenEval ifCond) { + whens.add(ifCond); + } + + public void addIfCond(EvalNode condition, EvalNode result) { whens.add(new IfThenEval(condition, result)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index 99f86dd0da..9168d9d36a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -32,9 +32,7 @@ import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.exception.InternalException; -import org.apache.tajo.util.datetime.DateTimeUtil; import java.util.Iterator; import java.util.Map; @@ -148,6 +146,22 @@ public static EvalNode deserialize(PlanProto.EvalTree tree) { evalNodeMap.get(betweenProto.getBegin()), evalNodeMap.get(betweenProto.getEnd())); + } else if (type == EvalType.CASE) { + PlanProto.CaseWhenEval caseWhenProto = protoNode.getCasewhen(); + CaseWhenEval caseWhenEval = new CaseWhenEval(); + for (int i = 0; i < caseWhenProto.getIfCondsCount(); i++) { + caseWhenEval.addIfCond((CaseWhenEval.IfThenEval) evalNodeMap.get(caseWhenProto.getIfConds(i))); + } + if (caseWhenProto.hasElse()) { + caseWhenEval.setElseResult(evalNodeMap.get(caseWhenProto.getElse())); + } + current = caseWhenEval; + + } else if (type == EvalType.IF_THEN) { + PlanProto.IfCondEval ifCondProto = protoNode.getIfCond(); + current = new CaseWhenEval.IfThenEval(evalNodeMap.get(ifCondProto.getCondition()), + evalNodeMap.get(ifCondProto.getThen())); + } else if (EvalType.isFunction(type)) { PlanProto.FunctionEval funcProto = protoNode.getFunction(); @@ -321,7 +335,51 @@ public EvalNode visitBetween(EvalTreeProtoBuilderContext context, BetweenPredica builder.setBetween(betweenBuilder); context.evalTreeBuilder.addNodes(builder); return between; + } + + public EvalNode visitCaseWhen(EvalTreeProtoBuilderContext context, CaseWhenEval caseWhen, Stack stack) { + // visiting and registering childs + super.visitCaseWhen(context, caseWhen, stack); + int [] childIds = registerGetChildIds(context, caseWhen); + Preconditions.checkState(childIds.length > 0, "Case When must have at least one child, but there is no child"); + + // building itself + PlanProto.CaseWhenEval.Builder caseWhenBuilder = PlanProto.CaseWhenEval.newBuilder(); + int ifCondsNum = childIds.length - (caseWhen.hasElse() ? 1 : 0); + for (int i = 0; i < ifCondsNum; i++) { + caseWhenBuilder.addIfConds(childIds[i]); + } + if (caseWhen.hasElse()) { + caseWhenBuilder.setElse(childIds[childIds.length - 1]); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, caseWhen); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, caseWhen); + builder.setCasewhen(caseWhenBuilder); + context.evalTreeBuilder.addNodes(builder); + + return caseWhen; + } + + public EvalNode visitIfThen(EvalTreeProtoBuilderContext context, CaseWhenEval.IfThenEval ifCond, + Stack stack) { + // visiting and registering childs + super.visitIfThen(context, ifCond, stack); + int [] childIds = registerGetChildIds(context, ifCond); + + // building itself + PlanProto.IfCondEval.Builder ifCondBuilder = PlanProto.IfCondEval.newBuilder(); + ifCondBuilder.setCondition(childIds[0]); + ifCondBuilder.setThen(childIds[1]); + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, ifCond); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, ifCond); + builder.setIfCond(ifCondBuilder); + context.evalTreeBuilder.addNodes(builder); + return ifCond; } public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 39fd08acf5..c0434025d6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -352,7 +352,7 @@ public EvalNode visitCaseWhen(Context ctx, Stack stack, CaseWhenPredicate for (CaseWhenPredicate.WhenExpr when : caseWhen.getWhens()) { condition = visit(ctx, stack, when.getCondition()); result = visit(ctx, stack, when.getResult()); - caseWhenEval.addWhen(condition, result); + caseWhenEval.addIfCond(condition, result); } if (caseWhen.hasElseResult()) { diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index a8776cdf3a..fa8cbb8c44 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -144,6 +144,8 @@ message EvalNode { optional FunctionEval function = 8; optional RowConstEval rowConst = 9; optional BetweenEval between = 10; + optional CaseWhenEval casewhen = 11; + optional IfCondEval ifCond = 12; } message UnaryEval { @@ -166,6 +168,16 @@ message BetweenEval { optional bool symmetric = 5 [default = false]; } +message CaseWhenEval { + repeated int32 ifConds = 1; + optional int32 else = 2; +} + +message IfCondEval { + required int32 condition = 1; + required int32 then = 2; +} + message ConstEval { required Datum value = 1; } From 0b257fd924a747075c68200a676e29e352b15c99 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 15 Aug 2014 22:03:32 +0900 Subject: [PATCH 10/31] Renamed setExpr to setChild. --- .../org/apache/tajo/engine/eval/BetweenPredicateEval.java | 2 +- .../src/main/java/org/apache/tajo/engine/eval/BinaryEval.java | 2 +- .../main/java/org/apache/tajo/engine/eval/CaseWhenEval.java | 4 ++-- .../src/main/java/org/apache/tajo/engine/eval/ConstEval.java | 2 +- .../src/main/java/org/apache/tajo/engine/eval/EvalNode.java | 2 +- .../src/main/java/org/apache/tajo/engine/eval/FieldEval.java | 2 +- .../main/java/org/apache/tajo/engine/eval/FunctionEval.java | 2 +- .../java/org/apache/tajo/engine/eval/RowConstantEval.java | 2 +- .../src/main/java/org/apache/tajo/engine/eval/UnaryEval.java | 2 +- .../org/apache/tajo/engine/plan/LogicalPlanConvertor.java | 4 ++-- .../test/java/org/apache/tajo/engine/eval/TestEvalTree.java | 4 ++-- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java index 86af4ba357..e42838eca0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java @@ -195,7 +195,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { if (idx == 0) { return predicand; } else if (idx == 1) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java index 7cc13de291..9c76ec06d3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java @@ -89,7 +89,7 @@ public T getRightExpr() { } @Override - public EvalNode getExpr(int id) { + public EvalNode getChild(int id) { if (id == 0) { return this.leftExpr; } else if (id == 1) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java index 27769a4087..a1a6ba1e3f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java @@ -90,7 +90,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { if (idx < whens.size()) { return whens.get(idx); } else if (idx == whens.size()) { @@ -201,7 +201,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { if (idx == 0) { return condition; } else if (idx == 1) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java index f164030059..323a4e98b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java @@ -58,7 +58,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { return null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java index 9b02103146..48ab5167ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java @@ -49,7 +49,7 @@ public EvalType getType() { public abstract int childNum(); - public abstract EvalNode getExpr(int idx); + public abstract EvalNode getChild(int idx); public abstract String getName(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java index e0a71b75c3..5cedbca38f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java @@ -66,7 +66,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { return null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java index 8e20a36815..9447ef26b6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java @@ -89,7 +89,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { return argEvals[idx]; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java index 288ffee210..5b21be046f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java @@ -47,7 +47,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { return null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java index 6b0bf6d09e..e7ae112893 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java @@ -43,7 +43,7 @@ public int childNum() { return 1; } - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { Preconditions.checkArgument(idx == 0, "UnaryEval always has one child."); return child; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java index 9168d9d36a..b6fdab0144 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java @@ -217,8 +217,8 @@ private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalN private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { int [] childIds = new int[evalNode.childNum()]; for (int i = 0; i < evalNode.childNum(); i++) { - if (context.idMap.containsKey(evalNode.getExpr(i))) { - childIds[i] = context.idMap.get(evalNode.getExpr(i)); + if (context.idMap.containsKey(evalNode.getChild(i))) { + childIds[i] = context.idMap.get(evalNode.getChild(i)); } else { childIds[i] = context.seqId++; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index ec9887eddf..56595ee9fd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -95,7 +95,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { return null; } @@ -143,7 +143,7 @@ public int childNum() { } @Override - public EvalNode getExpr(int idx) { + public EvalNode getChild(int idx) { return null; } } From f5e994df1f64980ae48a2d9cb5e13c68dc0fe611 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 15 Aug 2014 22:21:43 +0900 Subject: [PATCH 11/31] Refactored eval tree proto. --- .../engine/eval/SimpleEvalNodeVisitor.java | 24 +- .../plan/EvalTreeProtoDeserializer.java | 204 ++++++++ .../engine/plan/EvalTreeProtoSerializer.java | 305 +++++++++++ .../engine/plan/LogicalPlanConvertor.java | 495 ------------------ .../apache/tajo/engine/eval/ExprTestBase.java | 10 +- .../engine/plan/TestLogicalPlanConvertor.java | 80 --- .../apache/tajo/jitvec/JitVecTestServer.java | 11 +- 7 files changed, 537 insertions(+), 592 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java delete mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java index 93f1f74e56..15e34de6fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java @@ -26,7 +26,7 @@ * It provides simple visitor methods for an expression tree. Since SimpleEvalNodeVisitor provides * fewer visitor methods, it allows users to write a simple rewriter for expression trees. */ -public class SimpleEvalNodeVisitor { +public abstract class SimpleEvalNodeVisitor { public EvalNode visit(CONTEXT context, EvalNode evalNode, Stack stack) { EvalNode result; @@ -83,14 +83,14 @@ public EvalNode visit(CONTEXT context, EvalNode evalNode, Stack stack) return result; } - public EvalNode visitUnaryEval(CONTEXT context, Stack stack, UnaryEval unaryEval) { + protected EvalNode visitUnaryEval(CONTEXT context, Stack stack, UnaryEval unaryEval) { stack.push(unaryEval); visit(context, unaryEval.getChild(), stack); stack.pop(); return unaryEval; } - public EvalNode visitBinaryEval(CONTEXT context, Stack stack, BinaryEval binaryEval) { + protected EvalNode visitBinaryEval(CONTEXT context, Stack stack, BinaryEval binaryEval) { stack.push(binaryEval); visit(context, binaryEval.getLeftExpr(), stack); visit(context, binaryEval.getRightExpr(), stack); @@ -98,7 +98,7 @@ public EvalNode visitBinaryEval(CONTEXT context, Stack stack, BinaryEv return binaryEval; } - private EvalNode visitDefaultFunctionEval(CONTEXT context, Stack stack, FunctionEval functionEval) { + protected EvalNode visitDefaultFunctionEval(CONTEXT context, Stack stack, FunctionEval functionEval) { stack.push(functionEval); if (functionEval.getArgs() != null) { for (EvalNode arg : functionEval.getArgs()) { @@ -113,15 +113,15 @@ private EvalNode visitDefaultFunctionEval(CONTEXT context, Stack stack // Value and Literal /////////////////////////////////////////////////////////////////////////////////////////////// - public EvalNode visitConst(CONTEXT context, ConstEval evalNode, Stack stack) { + protected EvalNode visitConst(CONTEXT context, ConstEval evalNode, Stack stack) { return evalNode; } - public EvalNode visitRowConstant(CONTEXT context, RowConstantEval evalNode, Stack stack) { + protected EvalNode visitRowConstant(CONTEXT context, RowConstantEval evalNode, Stack stack) { return evalNode; } - public EvalNode visitField(CONTEXT context, Stack stack, FieldEval evalNode) { + protected EvalNode visitField(CONTEXT context, Stack stack, FieldEval evalNode) { return evalNode; } @@ -130,7 +130,7 @@ public EvalNode visitField(CONTEXT context, Stack stack, FieldEval eva // SQL standard predicates /////////////////////////////////////////////////////////////////////////////////////////////// - public EvalNode visitBetween(CONTEXT context, BetweenPredicateEval evalNode, Stack stack) { + protected EvalNode visitBetween(CONTEXT context, BetweenPredicateEval evalNode, Stack stack) { stack.push(evalNode); visit(context, evalNode.getPredicand(), stack); visit(context, evalNode.getBegin(), stack); @@ -138,7 +138,7 @@ public EvalNode visitBetween(CONTEXT context, BetweenPredicateEval evalNode, Sta return evalNode; } - public EvalNode visitCaseWhen(CONTEXT context, CaseWhenEval evalNode, Stack stack) { + protected EvalNode visitCaseWhen(CONTEXT context, CaseWhenEval evalNode, Stack stack) { stack.push(evalNode); for (CaseWhenEval.IfThenEval ifThenEval : evalNode.getIfThenEvals()) { visitIfThen(context, ifThenEval, stack); @@ -150,7 +150,7 @@ public EvalNode visitCaseWhen(CONTEXT context, CaseWhenEval evalNode, Stack stack) { + protected EvalNode visitIfThen(CONTEXT context, CaseWhenEval.IfThenEval evalNode, Stack stack) { stack.push(evalNode); visit(context, evalNode.getCondition(), stack); visit(context, evalNode.getResult(), stack); @@ -158,7 +158,7 @@ public EvalNode visitIfThen(CONTEXT context, CaseWhenEval.IfThenEval evalNode, S return evalNode; } - public EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack stack) { + protected EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack stack) { return visitBinaryEval(context, stack, evalNode); } @@ -166,7 +166,7 @@ public EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack stack) { + protected EvalNode visitFuncCall(CONTEXT context, FunctionEval evalNode, Stack stack) { return visitDefaultFunctionEval(context, stack, evalNode); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java new file mode 100644 index 0000000000..1421cda428 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java @@ -0,0 +1,204 @@ +/* + * Lisensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.collect.Maps; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.exception.NoSuchFunctionException; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.datum.*; +import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.function.AggFunction; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.exception.InternalException; + +import java.util.Iterator; +import java.util.Map; +import java.util.SortedMap; + +public class EvalTreeProtoDeserializer { + + public static EvalNode deserialize(PlanProto.EvalTree tree) { + SortedMap protoMap = Maps.newTreeMap(); + Map evalNodeMap = Maps.newHashMap(); + + for (PlanProto.EvalNode node : tree.getNodesList()) { + protoMap.put(node.getId(), node); + } + + EvalNode current = null; + + Iterator> it = protoMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + + PlanProto.EvalNode protoNode = entry.getValue(); + + EvalType type = EvalType.valueOf(protoNode.getType().name()); + + if (EvalType.isUnaryOperator(type)) { + PlanProto.UnaryEval unaryProto = protoNode.getUnary(); + EvalNode child = evalNodeMap.get(unaryProto.getChildId()); + + switch (type) { + case NOT: + current = new NotEval(child); + break; + case IS_NULL: + current = new IsNullEval(unaryProto.getNegative(), child); + break; + case CAST: + current = new CastEval(child, unaryProto.getCastingType()); + break; + case SIGNED: + current = new SignedEval(unaryProto.getNegative(), child); + break; + default: + throw new RuntimeException("Unknown EvalType: " + type.name()); + } + + } else if (EvalType.isBinaryOperator(type)) { + PlanProto.BinaryEval binProto = protoNode.getBinary(); + EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); + EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); + + switch (type) { + case IN: + current = new InEval(lhs, (RowConstantEval) rhs, binProto.getNegative()); + break; + default: + current = new BinaryEval(type, lhs, rhs); + } + + } else if (type == EvalType.CONST) { + PlanProto.ConstEval constProto = protoNode.getConst(); + current = new ConstEval(deserialize(constProto.getValue())); + + } else if (type == EvalType.ROW_CONSTANT) { + PlanProto.RowConstEval rowConstProto = protoNode.getRowConst(); + Datum[] values = new Datum[rowConstProto.getValuesCount()]; + for (int i = 0; i < rowConstProto.getValuesCount(); i++) { + values[i] = deserialize(rowConstProto.getValues(i)); + } + current = new RowConstantEval(values); + + } else if (type == EvalType.FIELD) { + CatalogProtos.ColumnProto columnProto = protoNode.getField(); + current = new FieldEval(new Column(columnProto)); + + } else if (type == EvalType.BETWEEN) { + PlanProto.BetweenEval betweenProto = protoNode.getBetween(); + current = new BetweenPredicateEval(betweenProto.getNegative(), betweenProto.getSymmetric(), + evalNodeMap.get(betweenProto.getPredicand()), + evalNodeMap.get(betweenProto.getBegin()), + evalNodeMap.get(betweenProto.getEnd())); + + } else if (type == EvalType.CASE) { + PlanProto.CaseWhenEval caseWhenProto = protoNode.getCasewhen(); + CaseWhenEval caseWhenEval = new CaseWhenEval(); + for (int i = 0; i < caseWhenProto.getIfCondsCount(); i++) { + caseWhenEval.addIfCond((CaseWhenEval.IfThenEval) evalNodeMap.get(caseWhenProto.getIfConds(i))); + } + if (caseWhenProto.hasElse()) { + caseWhenEval.setElseResult(evalNodeMap.get(caseWhenProto.getElse())); + } + current = caseWhenEval; + + } else if (type == EvalType.IF_THEN) { + PlanProto.IfCondEval ifCondProto = protoNode.getIfCond(); + current = new CaseWhenEval.IfThenEval(evalNodeMap.get(ifCondProto.getCondition()), + evalNodeMap.get(ifCondProto.getThen())); + + } else if (EvalType.isFunction(type)) { + PlanProto.FunctionEval funcProto = protoNode.getFunction(); + + EvalNode [] params = new EvalNode[funcProto.getParamIdsCount()]; + for (int i = 0; i < funcProto.getParamIdsCount(); i++) { + params[i] = evalNodeMap.get(funcProto.getParamIds(i)); + } + + FunctionDesc funcDesc = null; + try { + funcDesc = new FunctionDesc(funcProto.getFuncion()); + if (type == EvalType.FUNCTION) { + GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); + current = new GeneralFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { + AggFunction instance = (AggFunction) funcDesc.newInstance(); + if (type == EvalType.AGG_FUNCTION) { + current = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + } else { + current = new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params, null); + } + } + } catch (ClassNotFoundException cnfe) { + throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + } catch (InternalException ie) { + throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + } + } else { + throw new RuntimeException("Unknown EvalType: " + type.name()); + } + + evalNodeMap.put(protoNode.getId(), current); + } + + return current; + } + + public static Datum deserialize(PlanProto.Datum datum) { + switch (datum.getType()) { + case BOOLEAN: + return DatumFactory.createBool(datum.getBoolean()); + case CHAR: + return DatumFactory.createChar(datum.getText()); + case INT1: + case INT2: + return DatumFactory.createInt2((short) datum.getInt4()); + case INT4: + return DatumFactory.createInt4(datum.getInt4()); + case INT8: + return DatumFactory.createInt8(datum.getInt8()); + case FLOAT4: + return DatumFactory.createFloat4(datum.getFloat4()); + case FLOAT8: + return DatumFactory.createFloat8(datum.getFloat8()); + case VARCHAR: + case TEXT: + return DatumFactory.createText(datum.getText()); + case TIMESTAMP: + return new TimestampDatum(datum.getInt8()); + case DATE: + return DatumFactory.createDate(datum.getInt4()); + case TIME: + return DatumFactory.createTime(datum.getInt8()); + case BINARY: + case BLOB: + return DatumFactory.createBlob(datum.getBlob().toByteArray()); + case INTERVAL: + return new IntervalDatum(datum.getInterval().getMonth(), datum.getInterval().getMsec()); + case NULL_TYPE: + return NullDatum.get(); + default: + throw new RuntimeException("Unknown data type: " + datum.getType().name()); + } + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java new file mode 100644 index 0000000000..6c843c848c --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java @@ -0,0 +1,305 @@ +/* + * Lisensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.IntervalDatum; +import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.plan.proto.PlanProto; + +import java.util.Map; +import java.util.Stack; + +public class EvalTreeProtoSerializer + extends SimpleEvalNodeVisitor { + + private static final EvalTreeProtoSerializer instance; + + static { + instance = new EvalTreeProtoSerializer(); + } + + public static class EvalTreeProtoBuilderContext { + private int seqId = 0; + private Map idMap = Maps.newHashMap(); + private PlanProto.EvalTree.Builder evalTreeBuilder = PlanProto.EvalTree.newBuilder(); + } + + public static PlanProto.EvalTree serialize(EvalNode evalNode) { + EvalTreeProtoSerializer.EvalTreeProtoBuilderContext context = + new EvalTreeProtoSerializer.EvalTreeProtoBuilderContext(); + instance.visit(context, evalNode, new Stack()); + return context.evalTreeBuilder.build(); + } + + private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalNode) { + int selfId; + if (context.idMap.containsKey(evalNode)) { + selfId = context.idMap.get(evalNode); + } else { + selfId = context.seqId++; + context.idMap.put(evalNode, selfId); + } + + return selfId; + } + + private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { + int [] childIds = new int[evalNode.childNum()]; + for (int i = 0; i < evalNode.childNum(); i++) { + if (context.idMap.containsKey(evalNode.getChild(i))) { + childIds[i] = context.idMap.get(evalNode.getChild(i)); + } else { + childIds[i] = context.seqId++; + } + } + return childIds; + } + + private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { + PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); + nodeBuilder.setId(id); + nodeBuilder.setDataType(eval.getValueType()); + nodeBuilder.setType(PlanProto.EvalType.valueOf(eval.getType().name())); + return nodeBuilder; + } + + @Override + public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unary) { + // visiting and registering childs + super.visitUnaryEval(context, stack, unary); + int [] childIds = registerGetChildIds(context, unary); + + // building itself + PlanProto.UnaryEval.Builder unaryBuilder = PlanProto.UnaryEval.newBuilder(); + unaryBuilder.setChildId(childIds[0]); + if (unary.getType() == EvalType.IS_NULL) { + IsNullEval isNullEval = (IsNullEval) unary; + unaryBuilder.setNegative(isNullEval.isNot()); + } else if (unary.getType() == EvalType.SIGNED) { + SignedEval signedEval = (SignedEval) unary; + unaryBuilder.setNegative(signedEval.isNegative()); + } else if (unary.getType() == EvalType.CAST) { + CastEval castEval = (CastEval) unary; + unaryBuilder.setCastingType(castEval.getValueType()); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, unary); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, unary); + builder.setUnary(unaryBuilder); + context.evalTreeBuilder.addNodes(builder); + return unary; + } + + @Override + public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binary) { + // visiting and registering childs + super.visitBinaryEval(context, stack, binary); + int [] childIds = registerGetChildIds(context, binary); + + // building itself + PlanProto.BinaryEval.Builder binaryBuilder = PlanProto.BinaryEval.newBuilder(); + binaryBuilder.setLhsId(childIds[0]); + binaryBuilder.setRhsId(childIds[1]); + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, binary); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, binary); + builder.setBinary(binaryBuilder); + context.evalTreeBuilder.addNodes(builder); + return binary; + } + + @Override + public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval constant, Stack stack) { + int selfId = registerAndGetId(context, constant); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constant); + builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constant.getValue()))); + context.evalTreeBuilder.addNodes(builder); + return constant; + } + + @Override + public EvalNode visitRowConstant(EvalTreeProtoBuilderContext context, RowConstantEval rowConst, + Stack stack) { + + PlanProto.RowConstEval.Builder rowConstBuilder = PlanProto.RowConstEval.newBuilder(); + for (Datum d : rowConst.getValues()) { + rowConstBuilder.addValues(serialize(d)); + } + + int selfId = registerAndGetId(context, rowConst); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, rowConst); + builder.setRowConst(rowConstBuilder); + context.evalTreeBuilder.addNodes(builder); + return rowConst; + } + + public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { + int selfId = registerAndGetId(context, field); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, field); + builder.setField(field.getColumnRef().getProto()); + context.evalTreeBuilder.addNodes(builder); + return field; + } + + public EvalNode visitBetween(EvalTreeProtoBuilderContext context, BetweenPredicateEval between, + Stack stack) { + // visiting and registering childs + super.visitBetween(context, between, stack); + int [] childIds = registerGetChildIds(context, between); + Preconditions.checkState(childIds.length == 3, "Between must have three childs, but there are " + childIds.length + + " child nodes"); + + // building itself + PlanProto.BetweenEval.Builder betweenBuilder = PlanProto.BetweenEval.newBuilder(); + betweenBuilder.setNegative(between.isNot()); + betweenBuilder.setSymmetric(between.isSymmetric()); + betweenBuilder.setPredicand(childIds[0]); + betweenBuilder.setBegin(childIds[1]); + betweenBuilder.setEnd(childIds[2]); + + int selfId = registerAndGetId(context, between); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, between); + builder.setBetween(betweenBuilder); + context.evalTreeBuilder.addNodes(builder); + return between; + } + + public EvalNode visitCaseWhen(EvalTreeProtoBuilderContext context, CaseWhenEval caseWhen, Stack stack) { + // visiting and registering childs + super.visitCaseWhen(context, caseWhen, stack); + int [] childIds = registerGetChildIds(context, caseWhen); + Preconditions.checkState(childIds.length > 0, "Case When must have at least one child, but there is no child"); + + // building itself + PlanProto.CaseWhenEval.Builder caseWhenBuilder = PlanProto.CaseWhenEval.newBuilder(); + int ifCondsNum = childIds.length - (caseWhen.hasElse() ? 1 : 0); + for (int i = 0; i < ifCondsNum; i++) { + caseWhenBuilder.addIfConds(childIds[i]); + } + if (caseWhen.hasElse()) { + caseWhenBuilder.setElse(childIds[childIds.length - 1]); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, caseWhen); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, caseWhen); + builder.setCasewhen(caseWhenBuilder); + context.evalTreeBuilder.addNodes(builder); + + return caseWhen; + } + + public EvalNode visitIfThen(EvalTreeProtoBuilderContext context, CaseWhenEval.IfThenEval ifCond, + Stack stack) { + // visiting and registering childs + super.visitIfThen(context, ifCond, stack); + int [] childIds = registerGetChildIds(context, ifCond); + + // building itself + PlanProto.IfCondEval.Builder ifCondBuilder = PlanProto.IfCondEval.newBuilder(); + ifCondBuilder.setCondition(childIds[0]); + ifCondBuilder.setThen(childIds[1]); + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, ifCond); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, ifCond); + builder.setIfCond(ifCondBuilder); + context.evalTreeBuilder.addNodes(builder); + + return ifCond; + } + + public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { + // visiting and registering childs + super.visitFuncCall(context, function, stack); + int [] childIds = registerGetChildIds(context, function); + + // building itself + PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); + funcBuilder.setFuncion(function.getFuncDesc().getProto()); + for (int i = 0; i < childIds.length; i++) { + funcBuilder.addParamIds(childIds[i]); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, function); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, function); + builder.setFunction(funcBuilder); + context.evalTreeBuilder.addNodes(builder); + + return function; + } + + public static PlanProto.Datum serialize(Datum datum) { + PlanProto.Datum.Builder builder = PlanProto.Datum.newBuilder(); + + builder.setType(datum.type()); + + switch (datum.type()) { + case NULL_TYPE: + break; + case BOOLEAN: + builder.setBoolean(datum.asBool()); + break; + case INT1: + case INT2: + case INT4: + case DATE: + builder.setInt4(datum.asInt4()); + break; + case INT8: + case TIMESTAMP: + case TIME: + builder.setInt8(datum.asInt8()); + break; + case FLOAT4: + builder.setFloat4(datum.asFloat4()); + break; + case FLOAT8: + builder.setFloat8(datum.asFloat8()); + break; + case CHAR: + case VARCHAR: + case TEXT: + builder.setText(datum.asChars()); + break; + case BINARY: + case BLOB: + builder.setBlob(ByteString.copyFrom(datum.asByteArray())); + break; + case INTERVAL: + IntervalDatum interval = (IntervalDatum) datum; + PlanProto.Interval.Builder intervalBuilder = PlanProto.Interval.newBuilder(); + intervalBuilder.setMonth(interval.getMonths()); + intervalBuilder.setMsec(interval.getMilliSeconds()); + builder.setInterval(intervalBuilder); + break; + default: + throw new RuntimeException("Unknown data type: " + datum.type().name()); + } + + return builder.build(); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java deleted file mode 100644 index b6fdab0144..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalPlanConvertor.java +++ /dev/null @@ -1,495 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.plan; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.google.protobuf.ByteString; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.catalog.exception.NoSuchFunctionException; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.datum.*; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.GeneralFunction; -import org.apache.tajo.engine.plan.proto.PlanProto; -import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.exception.InternalException; - -import java.util.Iterator; -import java.util.Map; -import java.util.SortedMap; -import java.util.Stack; - -public class LogicalPlanConvertor { - public PlanProto.LogicalPlan convert(LogicalPlan plan) { - PlanProto.LogicalPlan.Builder builder = PlanProto.LogicalPlan.newBuilder(); - - return null; - } - - - - public static class PlanProtoBuilder extends BasicLogicalPlanVisitor { - - } - - public static PlanProto.EvalTree serialize(EvalNode evalNode) { - EvalTreeProtoBuilderContext context = new EvalTreeProtoBuilderContext(); - EvalTreeProtoSerializer serializer = new EvalTreeProtoSerializer(); - serializer.visit(context, evalNode, new Stack()); - return context.evalTreeBuilder.build(); - } - - public static EvalNode deserialize(PlanProto.EvalTree evalTree) { - return EvalTreeProtoDeserializer.deserialize(evalTree); - } - - public static class EvalTreeProtoBuilderContext { - private int seqId = 0; - private Map idMap = Maps.newHashMap(); - private PlanProto.EvalTree.Builder evalTreeBuilder = PlanProto.EvalTree.newBuilder(); - } - - public static class EvalTreeProtoDeserializer { - - public static EvalNode deserialize(PlanProto.EvalTree tree) { - SortedMap protoMap = Maps.newTreeMap(); - Map evalNodeMap = Maps.newHashMap(); - - for (PlanProto.EvalNode node : tree.getNodesList()) { - protoMap.put(node.getId(), node); - } - - EvalNode current = null; - - Iterator> it = protoMap.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = it.next(); - - PlanProto.EvalNode protoNode = entry.getValue(); - - EvalType type = EvalType.valueOf(protoNode.getType().name()); - - if (EvalType.isUnaryOperator(type)) { - PlanProto.UnaryEval unaryProto = protoNode.getUnary(); - EvalNode child = evalNodeMap.get(unaryProto.getChildId()); - - switch (type) { - case NOT: - current = new NotEval(child); - break; - case IS_NULL: - current = new IsNullEval(unaryProto.getNegative(), child); - break; - case CAST: - current = new CastEval(child, unaryProto.getCastingType()); - break; - case SIGNED: - current = new SignedEval(unaryProto.getNegative(), child); - break; - default: - throw new RuntimeException("Unknown EvalType: " + type.name()); - } - - } else if (EvalType.isBinaryOperator(type)) { - PlanProto.BinaryEval binProto = protoNode.getBinary(); - EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); - EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); - - switch (type) { - case IN: - current = new InEval(lhs, (RowConstantEval) rhs, binProto.getNegative()); - break; - default: - current = new BinaryEval(type, lhs, rhs); - } - - } else if (type == EvalType.CONST) { - PlanProto.ConstEval constProto = protoNode.getConst(); - current = new ConstEval(LogicalPlanConvertor.deserialize(constProto.getValue())); - - } else if (type == EvalType.ROW_CONSTANT) { - PlanProto.RowConstEval rowConstProto = protoNode.getRowConst(); - Datum [] values = new Datum[rowConstProto.getValuesCount()]; - for (int i = 0; i < rowConstProto.getValuesCount(); i++) { - values[i] = LogicalPlanConvertor.deserialize(rowConstProto.getValues(i)); - } - current = new RowConstantEval(values); - - } else if (type == EvalType.FIELD) { - CatalogProtos.ColumnProto columnProto = protoNode.getField(); - current = new FieldEval(new Column(columnProto)); - - } else if (type == EvalType.BETWEEN) { - PlanProto.BetweenEval betweenProto = protoNode.getBetween(); - current = new BetweenPredicateEval(betweenProto.getNegative(), betweenProto.getSymmetric(), - evalNodeMap.get(betweenProto.getPredicand()), - evalNodeMap.get(betweenProto.getBegin()), - evalNodeMap.get(betweenProto.getEnd())); - - } else if (type == EvalType.CASE) { - PlanProto.CaseWhenEval caseWhenProto = protoNode.getCasewhen(); - CaseWhenEval caseWhenEval = new CaseWhenEval(); - for (int i = 0; i < caseWhenProto.getIfCondsCount(); i++) { - caseWhenEval.addIfCond((CaseWhenEval.IfThenEval) evalNodeMap.get(caseWhenProto.getIfConds(i))); - } - if (caseWhenProto.hasElse()) { - caseWhenEval.setElseResult(evalNodeMap.get(caseWhenProto.getElse())); - } - current = caseWhenEval; - - } else if (type == EvalType.IF_THEN) { - PlanProto.IfCondEval ifCondProto = protoNode.getIfCond(); - current = new CaseWhenEval.IfThenEval(evalNodeMap.get(ifCondProto.getCondition()), - evalNodeMap.get(ifCondProto.getThen())); - - } else if (EvalType.isFunction(type)) { - PlanProto.FunctionEval funcProto = protoNode.getFunction(); - - EvalNode [] params = new EvalNode[funcProto.getParamIdsCount()]; - for (int i = 0; i < funcProto.getParamIdsCount(); i++) { - params[i] = evalNodeMap.get(funcProto.getParamIds(i)); - } - - FunctionDesc funcDesc = null; - try { - funcDesc = new FunctionDesc(funcProto.getFuncion()); - if (type == EvalType.FUNCTION) { - GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); - current = new GeneralFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params); - } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { - AggFunction instance = (AggFunction) funcDesc.newInstance(); - if (type == EvalType.AGG_FUNCTION) { - current = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), instance, params); - } else { - current = new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params, null); - } - } - } catch (ClassNotFoundException cnfe) { - throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); - } catch (InternalException ie) { - throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); - } - } else { - throw new RuntimeException("Unknown EvalType: " + type.name()); - } - - evalNodeMap.put(protoNode.getId(), current); - } - - return current; - } - } - - public static class EvalTreeProtoSerializer extends SimpleEvalNodeVisitor { - - private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalNode) { - int selfId; - if (context.idMap.containsKey(evalNode)) { - selfId = context.idMap.get(evalNode); - } else { - selfId = context.seqId++; - context.idMap.put(evalNode, selfId); - } - - return selfId; - } - - private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { - int [] childIds = new int[evalNode.childNum()]; - for (int i = 0; i < evalNode.childNum(); i++) { - if (context.idMap.containsKey(evalNode.getChild(i))) { - childIds[i] = context.idMap.get(evalNode.getChild(i)); - } else { - childIds[i] = context.seqId++; - } - } - return childIds; - } - - private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { - PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); - nodeBuilder.setId(id); - nodeBuilder.setDataType(eval.getValueType()); - nodeBuilder.setType(PlanProto.EvalType.valueOf(eval.getType().name())); - return nodeBuilder; - } - - @Override - public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unary) { - // visiting and registering childs - super.visitUnaryEval(context, stack, unary); - int [] childIds = registerGetChildIds(context, unary); - - // building itself - PlanProto.UnaryEval.Builder unaryBuilder = PlanProto.UnaryEval.newBuilder(); - unaryBuilder.setChildId(childIds[0]); - if (unary.getType() == EvalType.IS_NULL) { - IsNullEval isNullEval = (IsNullEval) unary; - unaryBuilder.setNegative(isNullEval.isNot()); - } else if (unary.getType() == EvalType.SIGNED) { - SignedEval signedEval = (SignedEval) unary; - unaryBuilder.setNegative(signedEval.isNegative()); - } else if (unary.getType() == EvalType.CAST) { - CastEval castEval = (CastEval) unary; - unaryBuilder.setCastingType(castEval.getValueType()); - } - - // registering itself and building EvalNode - int selfId = registerAndGetId(context, unary); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, unary); - builder.setUnary(unaryBuilder); - context.evalTreeBuilder.addNodes(builder); - return unary; - } - - @Override - public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binary) { - // visiting and registering childs - super.visitBinaryEval(context, stack, binary); - int [] childIds = registerGetChildIds(context, binary); - - // building itself - PlanProto.BinaryEval.Builder binaryBuilder = PlanProto.BinaryEval.newBuilder(); - binaryBuilder.setLhsId(childIds[0]); - binaryBuilder.setRhsId(childIds[1]); - - // registering itself and building EvalNode - int selfId = registerAndGetId(context, binary); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, binary); - builder.setBinary(binaryBuilder); - context.evalTreeBuilder.addNodes(builder); - return binary; - } - - @Override - public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval constant, Stack stack) { - int selfId = registerAndGetId(context, constant); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constant); - builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constant.getValue()))); - context.evalTreeBuilder.addNodes(builder); - return constant; - } - - @Override - public EvalNode visitRowConstant(EvalTreeProtoBuilderContext context, RowConstantEval rowConst, - Stack stack) { - - PlanProto.RowConstEval.Builder rowConstBuilder = PlanProto.RowConstEval.newBuilder(); - for (Datum d : rowConst.getValues()) { - rowConstBuilder.addValues(LogicalPlanConvertor.serialize(d)); - } - - int selfId = registerAndGetId(context, rowConst); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, rowConst); - builder.setRowConst(rowConstBuilder); - context.evalTreeBuilder.addNodes(builder); - return rowConst; - } - - public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { - int selfId = registerAndGetId(context, field); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, field); - builder.setField(field.getColumnRef().getProto()); - context.evalTreeBuilder.addNodes(builder); - return field; - } - - public EvalNode visitBetween(EvalTreeProtoBuilderContext context, BetweenPredicateEval between, - Stack stack) { - // visiting and registering childs - super.visitBetween(context, between, stack); - int [] childIds = registerGetChildIds(context, between); - Preconditions.checkState(childIds.length == 3, "Between must have three childs, but there are " + childIds.length - + " child nodes"); - - // building itself - PlanProto.BetweenEval.Builder betweenBuilder = PlanProto.BetweenEval.newBuilder(); - betweenBuilder.setNegative(between.isNot()); - betweenBuilder.setSymmetric(between.isSymmetric()); - betweenBuilder.setPredicand(childIds[0]); - betweenBuilder.setBegin(childIds[1]); - betweenBuilder.setEnd(childIds[2]); - - int selfId = registerAndGetId(context, between); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, between); - builder.setBetween(betweenBuilder); - context.evalTreeBuilder.addNodes(builder); - return between; - } - - public EvalNode visitCaseWhen(EvalTreeProtoBuilderContext context, CaseWhenEval caseWhen, Stack stack) { - // visiting and registering childs - super.visitCaseWhen(context, caseWhen, stack); - int [] childIds = registerGetChildIds(context, caseWhen); - Preconditions.checkState(childIds.length > 0, "Case When must have at least one child, but there is no child"); - - // building itself - PlanProto.CaseWhenEval.Builder caseWhenBuilder = PlanProto.CaseWhenEval.newBuilder(); - int ifCondsNum = childIds.length - (caseWhen.hasElse() ? 1 : 0); - for (int i = 0; i < ifCondsNum; i++) { - caseWhenBuilder.addIfConds(childIds[i]); - } - if (caseWhen.hasElse()) { - caseWhenBuilder.setElse(childIds[childIds.length - 1]); - } - - // registering itself and building EvalNode - int selfId = registerAndGetId(context, caseWhen); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, caseWhen); - builder.setCasewhen(caseWhenBuilder); - context.evalTreeBuilder.addNodes(builder); - - return caseWhen; - } - - public EvalNode visitIfThen(EvalTreeProtoBuilderContext context, CaseWhenEval.IfThenEval ifCond, - Stack stack) { - // visiting and registering childs - super.visitIfThen(context, ifCond, stack); - int [] childIds = registerGetChildIds(context, ifCond); - - // building itself - PlanProto.IfCondEval.Builder ifCondBuilder = PlanProto.IfCondEval.newBuilder(); - ifCondBuilder.setCondition(childIds[0]); - ifCondBuilder.setThen(childIds[1]); - - // registering itself and building EvalNode - int selfId = registerAndGetId(context, ifCond); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, ifCond); - builder.setIfCond(ifCondBuilder); - context.evalTreeBuilder.addNodes(builder); - - return ifCond; - } - - public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { - // visiting and registering childs - super.visitFuncCall(context, function, stack); - int [] childIds = registerGetChildIds(context, function); - - // building itself - PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); - funcBuilder.setFuncion(function.getFuncDesc().getProto()); - for (int i = 0; i < childIds.length; i++) { - funcBuilder.addParamIds(childIds[i]); - } - - // registering itself and building EvalNode - int selfId = registerAndGetId(context, function); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, function); - builder.setFunction(funcBuilder); - context.evalTreeBuilder.addNodes(builder); - - return function; - } - } - - public static Datum deserialize(PlanProto.Datum datum) { - switch (datum.getType()) { - case BOOLEAN: - return DatumFactory.createBool(datum.getBoolean()); - case CHAR: - return DatumFactory.createChar(datum.getText()); - case INT1: - case INT2: - return DatumFactory.createInt2((short) datum.getInt4()); - case INT4: - return DatumFactory.createInt4(datum.getInt4()); - case INT8: - return DatumFactory.createInt8(datum.getInt8()); - case FLOAT4: - return DatumFactory.createFloat4(datum.getFloat4()); - case FLOAT8: - return DatumFactory.createFloat8(datum.getFloat8()); - case VARCHAR: - case TEXT: - return DatumFactory.createText(datum.getText()); - case TIMESTAMP: - return new TimestampDatum(datum.getInt8()); - case DATE: - return DatumFactory.createDate(datum.getInt4()); - case TIME: - return DatumFactory.createTime(datum.getInt8()); - case BINARY: - case BLOB: - return DatumFactory.createBlob(datum.getBlob().toByteArray()); - case INTERVAL: - return new IntervalDatum(datum.getInterval().getMonth(), datum.getInterval().getMsec()); - case NULL_TYPE: - return NullDatum.get(); - default: - throw new RuntimeException("Unknown data type: " + datum.getType().name()); - } - } - - public static PlanProto.Datum serialize(Datum datum) { - PlanProto.Datum.Builder builder = PlanProto.Datum.newBuilder(); - - builder.setType(datum.type()); - - switch (datum.type()) { - case NULL_TYPE: - break; - case BOOLEAN: - builder.setBoolean(datum.asBool()); - break; - case INT1: - case INT2: - case INT4: - case DATE: - builder.setInt4(datum.asInt4()); - break; - case INT8: - case TIMESTAMP: - case TIME: - builder.setInt8(datum.asInt8()); - break; - case FLOAT4: - builder.setFloat4(datum.asFloat4()); - break; - case FLOAT8: - builder.setFloat8(datum.asFloat8()); - break; - case CHAR: - case VARCHAR: - case TEXT: - builder.setText(datum.asChars()); - break; - case BINARY: - case BLOB: - builder.setBlob(ByteString.copyFrom(datum.asByteArray())); - break; - case INTERVAL: - IntervalDatum interval = (IntervalDatum) datum; - PlanProto.Interval.Builder intervalBuilder = PlanProto.Interval.newBuilder(); - intervalBuilder.setMonth(interval.getMonths()); - intervalBuilder.setMsec(interval.getMilliSeconds()); - builder.setInterval(intervalBuilder); - break; - default: - throw new RuntimeException("Unknown data type: " + datum.type().name()); - } - - return builder.build(); - } -} diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 90f7276707..f615bcc836 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -31,7 +31,10 @@ import org.apache.tajo.datum.*; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.plan.EvalTreeProtoDeserializer; +import org.apache.tajo.engine.plan.EvalTreeProtoSerializer; import org.apache.tajo.engine.plan.TestLogicalPlanConvertor; +import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.SchemaUtil; @@ -150,7 +153,7 @@ private static Target[] getRawTargets(String query, boolean condition) throws Pl assertJsonSerDer(t.getEvalTree()); } for (Target t : targets) { - TestLogicalPlanConvertor.assertEvalNodeProtoSerder(t.getEvalTree()); + assertEvalTreeProtoSerDer(t.getEvalTree()); } return targets; } @@ -240,4 +243,9 @@ public void testEval(Schema schema, String tableName, String csvTuple, String qu } } } + + public static void assertEvalTreeProtoSerDer(EvalNode evalNode) { + PlanProto.EvalTree converted = EvalTreeProtoSerializer.serialize(evalNode); + assertEquals(evalNode, EvalTreeProtoDeserializer.deserialize(converted)); + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java index b4c94c6bcc..582384a65b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/plan/TestLogicalPlanConvertor.java @@ -41,85 +41,5 @@ import static org.junit.Assert.assertEquals; public class TestLogicalPlanConvertor { - static TajoTestingCluster util; - static CatalogService catalog = null; - static SQLAnalyzer analyzer; - static LogicalPlanner planner; - static Session session = LocalTajoTestingUtility.createDummySession(); - @BeforeClass - public static void setUp() throws Exception { - util = TpchTestBase.getInstance().getTestingCluster(); - catalog = util.getMaster().getCatalog(); - - analyzer = new SQLAnalyzer(); - planner = new LogicalPlanner(catalog); - } - - @AfterClass - public static void tearDown() throws Exception { - } - - public static Target[] getRawTargets(String query) { - Expr expr = analyzer.parse(query); - LogicalPlan plan = null; - try { - plan = planner.createPlan(session, expr, true); - } catch (PlanningException e) { - e.printStackTrace(); - } - - return plan.getRootBlock().getRawTargets(); - } - - public static EvalNode getRootSelection(String query) throws PlanningException { - Expr block = analyzer.parse(query); - LogicalPlan plan = null; - try { - plan = planner.createPlan(session, block); - } catch (PlanningException e) { - e.printStackTrace(); - } - - Selection selection = plan.getRootBlock().getSingletonExpr(OpType.Filter); - return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual(), - NameResolvingMode.RELS_AND_SUBEXPRS); - } - - @Test - public void testConvert() throws Exception { - Target [] targets = getRawTargets("select 1 + 2"); - assertEvalNodeProtoSerder(targets[0].getEvalTree()); - - targets = getRawTargets("select l_orderkey + l_partkey from lineitem"); - assertEvalNodeProtoSerder(targets[0].getEvalTree()); - } - - @Test - public void testDatumConvert() throws Exception { - assertDatumProtoSerder(DatumFactory.createBool(true)); - assertDatumProtoSerder(DatumFactory.createBool(false)); - assertDatumProtoSerder(DatumFactory.createInt2((short) 1)); - assertDatumProtoSerder(DatumFactory.createInt4(1980)); - assertDatumProtoSerder(DatumFactory.createInt8(19800401)); - assertDatumProtoSerder(DatumFactory.createFloat4(3.14f)); - assertDatumProtoSerder(DatumFactory.createFloat8(3.141592d)); - assertDatumProtoSerder(DatumFactory.createText("Apache Tajo")); - assertDatumProtoSerder(DatumFactory.createBlob("Apache Tajo".getBytes())); - } - - public static void assertDatumProtoSerder(Datum datum) { - PlanProto.Datum converted = LogicalPlanConvertor.serialize(datum); - assertEquals(datum, LogicalPlanConvertor.deserialize(converted)); - } - - public static void assertEvalNodeProtoSerder(EvalNode evalNode) { - PlanProto.EvalTree converted = LogicalPlanConvertor.serialize(evalNode); - assertEquals(evalNode, LogicalPlanConvertor.deserialize(converted)); - } - - @Test - public void testConvertDatum() throws Exception { - - } } diff --git a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java index 107285aea9..c275a085d2 100644 --- a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java +++ b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java @@ -34,6 +34,7 @@ import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.rpc.BlockingRpcServer; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; @@ -110,11 +111,13 @@ public JitVecTestServerProtocol.PlanResponse requestPlan(RpcController controlle JitVecTestServerProtocol.PlanResponse.Builder builder = JitVecTestServerProtocol.PlanResponse.newBuilder(); Session session = new Session("00", "tajo", "default"); + QueryContext queryContext = new QueryContext(conf, session); + try { LOG.info("Request is received: " + request.getSql()); Expr expr = analyzer.parse(request.getSql()); VerificationState state = new VerificationState(); - preVerifier.verify(session, state, expr); + preVerifier.verify(queryContext, state, expr); if (!state.verified()) { StringBuilder sb = new StringBuilder(); for (String error : state.getErrorMessages()) { @@ -123,19 +126,19 @@ public JitVecTestServerProtocol.PlanResponse requestPlan(RpcController controlle throw new VerifyException(sb.toString()); } - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(queryContext, expr); if (LOG.isDebugEnabled()) { LOG.debug("============================================="); LOG.debug("Non Optimized Query: \n" + plan.toString()); LOG.debug("============================================="); } LOG.info("Non Optimized Query: \n" + plan.toString()); - optimizer.optimize(session, plan); + optimizer.optimize(queryContext, plan); LOG.info("============================================="); LOG.info("Optimized Query: \n" + plan.toString()); LOG.info("============================================="); - annotatedPlanVerifier.verify(session, state, plan); + annotatedPlanVerifier.verify(queryContext, state, plan); if (!state.verified()) { StringBuilder sb = new StringBuilder(); From ade69f784e2369aa2f6950ce4f95dd316f4526e3 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 15 Aug 2014 22:51:40 +0900 Subject: [PATCH 12/31] TAJO-1008: Protocol buffer De/Serialization for EvalNode. --- .../java/org/apache/tajo/datum/Datum.java | 3 +- tajo-core/pom.xml | 1 + .../tajo/engine/eval/AlgebraicUtil.java | 6 +- .../engine/eval/BetweenPredicateEval.java | 26 ++ .../apache/tajo/engine/eval/BinaryEval.java | 6 + .../apache/tajo/engine/eval/CaseWhenEval.java | 38 ++- .../apache/tajo/engine/eval/ConstEval.java | 12 +- .../org/apache/tajo/engine/eval/EvalNode.java | 4 + .../apache/tajo/engine/eval/EvalTreeUtil.java | 2 +- .../org/apache/tajo/engine/eval/EvalType.java | 46 ++- .../apache/tajo/engine/eval/FieldEval.java | 12 +- .../apache/tajo/engine/eval/FunctionEval.java | 14 + .../tajo/engine/eval/RowConstantEval.java | 10 + .../engine/eval/SimpleEvalNodeVisitor.java | 24 +- .../apache/tajo/engine/eval/UnaryEval.java | 11 + .../plan/EvalTreeProtoDeserializer.java | 218 ++++++++++++ .../engine/plan/EvalTreeProtoSerializer.java | 310 ++++++++++++++++++ .../tajo/engine/planner/ExprAnnotator.java | 2 +- tajo-core/src/main/proto/Plan.proto | 209 ++++++++++++ .../apache/tajo/engine/eval/ExprTestBase.java | 21 ++ .../apache/tajo/engine/eval/TestEvalTree.java | 21 +- 21 files changed, 967 insertions(+), 29 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java create mode 100644 tajo-core/src/main/proto/Plan.proto diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java index 874004b5ba..f21e3d7a54 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java @@ -69,9 +69,8 @@ public short asInt2() { throw new InvalidCastException(type, Type.INT2); } public int asInt4() { - throw new InvalidCastException(type, Type.INT1); + throw new InvalidCastException(type, Type.INT4); } - public long asInt8() { throw new InvalidCastException(type, Type.INT8); } diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 32ae266c14..364c428aa5 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -144,6 +144,7 @@ src/main/proto/TajoMasterProtocol.proto src/main/proto/TajoWorkerProtocol.proto src/main/proto/InternalTypes.proto + src/main/proto/Plan.proto diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java index a72e2a8a17..518b72f0c9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java @@ -72,7 +72,7 @@ private static EvalNode _transpose(BinaryEval _expr, Column target) { EvalNode lTerm = null; EvalNode rTerm = null; - if (EvalType.isArithmeticOperator(left)) { // we can ensure that left is binary. + if (EvalType.isArithmeticOperator(left.getType())) { // we can ensure that left is binary. // If the left-left term is a variable, the left-right term is transposed. if (EvalTreeUtil.containColumnRef(((BinaryEval)left).getLeftExpr(), target)) { @@ -234,7 +234,7 @@ public static boolean containSingleVar(EvalNode expr) { */ public static PartialBinaryExpr splitLeftTerm(BinaryEval binary) { - if (!(EvalType.isArithmeticOperator(binary))) { + if (!(EvalType.isArithmeticOperator(binary.getType()))) { throw new AlgebraicException("Invalid algebraic operation: " + binary); } @@ -259,7 +259,7 @@ public static PartialBinaryExpr splitLeftTerm(BinaryEval binary) { */ public static PartialBinaryExpr splitRightTerm(BinaryEval binary) { - if (!(EvalType.isArithmeticOperator(binary))) { + if (!(EvalType.isArithmeticOperator(binary.getType()))) { throw new AlgebraicException("Invalid algebraic operation: " + binary); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java index 0b9c7c196c..07f9a6f424 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java @@ -46,6 +46,14 @@ public BetweenPredicateEval(boolean not, boolean symmetric, EvalNode predicand, this.end = end; } + public boolean isNot() { + return not; + } + + public boolean isSymmetric() { + return symmetric; + } + public void setPredicand(EvalNode predicand) { this.predicand = predicand; } @@ -169,6 +177,24 @@ public TajoDataTypes.DataType getValueType() { return RES_TYPE; } + @Override + public int childNum() { + return 3; + } + + @Override + public EvalNode getChild(int idx) { + if (idx == 0) { + return predicand; + } else if (idx == 1) { + return begin; + } else if (idx == 2) { + return end; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + @Override public String getName() { return "between"; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java index 24300207ba..4c94f4a2d2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java @@ -88,6 +88,12 @@ public T getRightExpr() { return (T) this.rightExpr; } + @Override + public int childNum() { + return 2; + } + + @Override public EvalNode getChild(int id) { if (id == 0) { return this.leftExpr; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java index cf1acdf64a..3eb3a69ce0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java @@ -43,7 +43,11 @@ public CaseWhenEval() { super(EvalType.CASE); } - public void addWhen(EvalNode condition, EvalNode result) { + public void addIfCond(IfThenEval ifCond) { + whens.add(ifCond); + } + + public void addIfCond(EvalNode condition, EvalNode result) { whens.add(new IfThenEval(condition, result)); } @@ -79,6 +83,22 @@ public DataType getValueType() { return NullDatum.getDataType(); } + @Override + public int childNum() { + return whens.size() + (elseResult != null ? 1 : 0); + } + + @Override + public EvalNode getChild(int idx) { + if (idx < whens.size()) { + return whens.get(idx); + } else if (idx == whens.size()) { + return elseResult; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + @Override public String getName() { return "?"; @@ -174,6 +194,22 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(TajoDataTypes.Type.BOOLEAN); } + @Override + public int childNum() { + return 2; + } + + @Override + public EvalNode getChild(int idx) { + if (idx == 0) { + return condition; + } else if (idx == 1) { + return result; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } + @Override public String getName() { return "when?"; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java index 2cb530dff0..323a4e98b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java @@ -52,7 +52,17 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(datum.type()); } - @Override + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getChild(int idx) { + return null; + } + + @Override public String getName() { return this.datum.toString(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java index a30d27fca8..48ab5167ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java @@ -46,6 +46,10 @@ public EvalType getType() { } public abstract DataType getValueType(); + + public abstract int childNum(); + + public abstract EvalNode getChild(int idx); public abstract String getName(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java index 3921a7d81f..cfd832c0b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java @@ -242,7 +242,7 @@ public static boolean isJoinQual(EvalNode expr, boolean includeThetaJoin) { if (expr instanceof BinaryEval) { boolean joinComparator; if (includeThetaJoin) { - joinComparator = EvalType.isComparisonOperator(expr); + joinComparator = EvalType.isComparisonOperator(expr.getType()); } else { joinComparator = expr.getType() == EvalType.EQUAL; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java index 549f8d0fd0..500928a869 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java @@ -79,8 +79,34 @@ public enum EvalType { this.operatorName = text; } - public static boolean isLogicalOperator(EvalNode evalNode) { - EvalType type = evalNode.getType(); + public static boolean isUnaryOperator(EvalType type) { + boolean match = false; + + match |= type == CAST; + match |= type == IS_NULL; + match |= type == NOT; + match |= type == SIGNED; + + return match; + } + + public static boolean isBinaryOperator(EvalType type) { + boolean match = false; + + match |= isArithmeticOperator(type); + match |= isLogicalOperator(type) && type != NOT; + match |= isComparisonOperator(type) && type != BETWEEN; + + match |= type == CONCATENATE; + match |= type == IN; + match |= type == LIKE; + match |= type == REGEX; + match |= type == SIMILAR_TO; + + return match; + } + + public static boolean isLogicalOperator(EvalType type) { boolean match = false; match |= type == AND; @@ -90,8 +116,7 @@ public static boolean isLogicalOperator(EvalNode evalNode) { return match; } - public static boolean isComparisonOperator(EvalNode evalNode) { - EvalType type = evalNode.getType(); + public static boolean isComparisonOperator(EvalType type) { boolean match = false; match |= type == EQUAL; @@ -105,8 +130,7 @@ public static boolean isComparisonOperator(EvalNode evalNode) { return match; } - public static boolean isArithmeticOperator(EvalNode evalNode) { - EvalType type = evalNode.getType(); + public static boolean isArithmeticOperator(EvalType type) { boolean match = false; match |= type == PLUS; @@ -118,6 +142,16 @@ public static boolean isArithmeticOperator(EvalNode evalNode) { return match; } + public static boolean isFunction(EvalType type) { + boolean match = false; + + match |= type == FUNCTION; + match |= type == AGG_FUNCTION; + match |= type == WINDOW_FUNCTION; + + return match; + } + public String getOperatorName() { return operatorName != null ? operatorName : name(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java index 20af8542a6..5cedbca38f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java @@ -59,7 +59,17 @@ public Datum eval(Schema schema, Tuple tuple) { public DataType getValueType() { return column.getDataType(); } - + + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getChild(int idx) { + return null; + } + public Column getColumnRef() { return column; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java index 376ee2e898..9447ef26b6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java @@ -79,6 +79,20 @@ public void setArgs(EvalNode [] args) { this.argEvals = args; } + @Override + public int childNum() { + if (argEvals != null) { + return argEvals.length; + } else { + return 0; + } + } + + @Override + public EvalNode getChild(int idx) { + return argEvals[idx]; + } + public DataType getValueType() { return this.funcDesc.getReturnType(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java index ffaf6b503b..5b21be046f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java @@ -41,6 +41,16 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(values[0].type()); } + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getChild(int idx) { + return null; + } + @Override public String getName() { return "ROW"; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java index 93f1f74e56..15e34de6fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java @@ -26,7 +26,7 @@ * It provides simple visitor methods for an expression tree. Since SimpleEvalNodeVisitor provides * fewer visitor methods, it allows users to write a simple rewriter for expression trees. */ -public class SimpleEvalNodeVisitor { +public abstract class SimpleEvalNodeVisitor { public EvalNode visit(CONTEXT context, EvalNode evalNode, Stack stack) { EvalNode result; @@ -83,14 +83,14 @@ public EvalNode visit(CONTEXT context, EvalNode evalNode, Stack stack) return result; } - public EvalNode visitUnaryEval(CONTEXT context, Stack stack, UnaryEval unaryEval) { + protected EvalNode visitUnaryEval(CONTEXT context, Stack stack, UnaryEval unaryEval) { stack.push(unaryEval); visit(context, unaryEval.getChild(), stack); stack.pop(); return unaryEval; } - public EvalNode visitBinaryEval(CONTEXT context, Stack stack, BinaryEval binaryEval) { + protected EvalNode visitBinaryEval(CONTEXT context, Stack stack, BinaryEval binaryEval) { stack.push(binaryEval); visit(context, binaryEval.getLeftExpr(), stack); visit(context, binaryEval.getRightExpr(), stack); @@ -98,7 +98,7 @@ public EvalNode visitBinaryEval(CONTEXT context, Stack stack, BinaryEv return binaryEval; } - private EvalNode visitDefaultFunctionEval(CONTEXT context, Stack stack, FunctionEval functionEval) { + protected EvalNode visitDefaultFunctionEval(CONTEXT context, Stack stack, FunctionEval functionEval) { stack.push(functionEval); if (functionEval.getArgs() != null) { for (EvalNode arg : functionEval.getArgs()) { @@ -113,15 +113,15 @@ private EvalNode visitDefaultFunctionEval(CONTEXT context, Stack stack // Value and Literal /////////////////////////////////////////////////////////////////////////////////////////////// - public EvalNode visitConst(CONTEXT context, ConstEval evalNode, Stack stack) { + protected EvalNode visitConst(CONTEXT context, ConstEval evalNode, Stack stack) { return evalNode; } - public EvalNode visitRowConstant(CONTEXT context, RowConstantEval evalNode, Stack stack) { + protected EvalNode visitRowConstant(CONTEXT context, RowConstantEval evalNode, Stack stack) { return evalNode; } - public EvalNode visitField(CONTEXT context, Stack stack, FieldEval evalNode) { + protected EvalNode visitField(CONTEXT context, Stack stack, FieldEval evalNode) { return evalNode; } @@ -130,7 +130,7 @@ public EvalNode visitField(CONTEXT context, Stack stack, FieldEval eva // SQL standard predicates /////////////////////////////////////////////////////////////////////////////////////////////// - public EvalNode visitBetween(CONTEXT context, BetweenPredicateEval evalNode, Stack stack) { + protected EvalNode visitBetween(CONTEXT context, BetweenPredicateEval evalNode, Stack stack) { stack.push(evalNode); visit(context, evalNode.getPredicand(), stack); visit(context, evalNode.getBegin(), stack); @@ -138,7 +138,7 @@ public EvalNode visitBetween(CONTEXT context, BetweenPredicateEval evalNode, Sta return evalNode; } - public EvalNode visitCaseWhen(CONTEXT context, CaseWhenEval evalNode, Stack stack) { + protected EvalNode visitCaseWhen(CONTEXT context, CaseWhenEval evalNode, Stack stack) { stack.push(evalNode); for (CaseWhenEval.IfThenEval ifThenEval : evalNode.getIfThenEvals()) { visitIfThen(context, ifThenEval, stack); @@ -150,7 +150,7 @@ public EvalNode visitCaseWhen(CONTEXT context, CaseWhenEval evalNode, Stack stack) { + protected EvalNode visitIfThen(CONTEXT context, CaseWhenEval.IfThenEval evalNode, Stack stack) { stack.push(evalNode); visit(context, evalNode.getCondition(), stack); visit(context, evalNode.getResult(), stack); @@ -158,7 +158,7 @@ public EvalNode visitIfThen(CONTEXT context, CaseWhenEval.IfThenEval evalNode, S return evalNode; } - public EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack stack) { + protected EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack stack) { return visitBinaryEval(context, stack, evalNode); } @@ -166,7 +166,7 @@ public EvalNode visitInPredicate(CONTEXT context, InEval evalNode, Stack stack) { + protected EvalNode visitFuncCall(CONTEXT context, FunctionEval evalNode, Stack stack) { return visitDefaultFunctionEval(context, stack, evalNode); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java index 7c6833fb98..e7ae112893 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.eval; import com.google.common.base.Objects; +import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; @@ -37,6 +38,16 @@ public UnaryEval(EvalType type, EvalNode child) { this.child = child; } + @Override + public int childNum() { + return 1; + } + + public EvalNode getChild(int idx) { + Preconditions.checkArgument(idx == 0, "UnaryEval always has one child."); + return child; + } + public void setChild(EvalNode child) { this.child = child; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java new file mode 100644 index 0000000000..0fa6dbb090 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java @@ -0,0 +1,218 @@ +/* + * Lisensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.exception.NoSuchFunctionException; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.datum.*; +import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.function.AggFunction; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.exception.InternalException; + +import java.util.*; + +/** + * It deserializes a serialized eval tree consisting of a number of EvalNodes. + * + * {@link org.apache.tajo.engine.plan.EvalTreeProtoSerializer} serializes an eval tree in a postfix traverse order. + * So, this class firstly sorts all serialized eval nodes in ascending order of their sequence IDs. Then, + * it sequentially restores each serialized node to EvalNode instance. + * + * @see org.apache.tajo.engine.plan.EvalTreeProtoSerializer + */ +public class EvalTreeProtoDeserializer { + + public static EvalNode deserialize(PlanProto.EvalTree tree) { + Map evalNodeMap = Maps.newHashMap(); + + // sort serialized eval nodes in an ascending order of their IDs. + List nodeList = Lists.newArrayList(tree.getNodesList()); + Collections.sort(nodeList, new Comparator() { + @Override + public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { + return o1.getId() - o2.getId(); + } + }); + + EvalNode current = null; + + // The sorted order is the same of a postfix traverse order. + // So, it sequentially transforms each serialized node into a EvalNode instance in a postfix order of + // the original eval tree. + + Iterator it = nodeList.iterator(); + while (it.hasNext()) { + PlanProto.EvalNode protoNode = it.next(); + + EvalType type = EvalType.valueOf(protoNode.getType().name()); + + if (EvalType.isUnaryOperator(type)) { + PlanProto.UnaryEval unaryProto = protoNode.getUnary(); + EvalNode child = evalNodeMap.get(unaryProto.getChildId()); + + switch (type) { + case NOT: + current = new NotEval(child); + break; + case IS_NULL: + current = new IsNullEval(unaryProto.getNegative(), child); + break; + case CAST: + current = new CastEval(child, unaryProto.getCastingType()); + break; + case SIGNED: + current = new SignedEval(unaryProto.getNegative(), child); + break; + default: + throw new RuntimeException("Unknown EvalType: " + type.name()); + } + + } else if (EvalType.isBinaryOperator(type)) { + PlanProto.BinaryEval binProto = protoNode.getBinary(); + EvalNode lhs = evalNodeMap.get(binProto.getLhsId()); + EvalNode rhs = evalNodeMap.get(binProto.getRhsId()); + + switch (type) { + case IN: + current = new InEval(lhs, (RowConstantEval) rhs, binProto.getNegative()); + break; + default: + current = new BinaryEval(type, lhs, rhs); + } + + } else if (type == EvalType.CONST) { + PlanProto.ConstEval constProto = protoNode.getConst(); + current = new ConstEval(deserialize(constProto.getValue())); + + } else if (type == EvalType.ROW_CONSTANT) { + PlanProto.RowConstEval rowConstProto = protoNode.getRowConst(); + Datum[] values = new Datum[rowConstProto.getValuesCount()]; + for (int i = 0; i < rowConstProto.getValuesCount(); i++) { + values[i] = deserialize(rowConstProto.getValues(i)); + } + current = new RowConstantEval(values); + + } else if (type == EvalType.FIELD) { + CatalogProtos.ColumnProto columnProto = protoNode.getField(); + current = new FieldEval(new Column(columnProto)); + + } else if (type == EvalType.BETWEEN) { + PlanProto.BetweenEval betweenProto = protoNode.getBetween(); + current = new BetweenPredicateEval(betweenProto.getNegative(), betweenProto.getSymmetric(), + evalNodeMap.get(betweenProto.getPredicand()), + evalNodeMap.get(betweenProto.getBegin()), + evalNodeMap.get(betweenProto.getEnd())); + + } else if (type == EvalType.CASE) { + PlanProto.CaseWhenEval caseWhenProto = protoNode.getCasewhen(); + CaseWhenEval caseWhenEval = new CaseWhenEval(); + for (int i = 0; i < caseWhenProto.getIfCondsCount(); i++) { + caseWhenEval.addIfCond((CaseWhenEval.IfThenEval) evalNodeMap.get(caseWhenProto.getIfConds(i))); + } + if (caseWhenProto.hasElse()) { + caseWhenEval.setElseResult(evalNodeMap.get(caseWhenProto.getElse())); + } + current = caseWhenEval; + + } else if (type == EvalType.IF_THEN) { + PlanProto.IfCondEval ifCondProto = protoNode.getIfCond(); + current = new CaseWhenEval.IfThenEval(evalNodeMap.get(ifCondProto.getCondition()), + evalNodeMap.get(ifCondProto.getThen())); + + } else if (EvalType.isFunction(type)) { + PlanProto.FunctionEval funcProto = protoNode.getFunction(); + + EvalNode [] params = new EvalNode[funcProto.getParamIdsCount()]; + for (int i = 0; i < funcProto.getParamIdsCount(); i++) { + params[i] = evalNodeMap.get(funcProto.getParamIds(i)); + } + + FunctionDesc funcDesc = null; + try { + funcDesc = new FunctionDesc(funcProto.getFuncion()); + if (type == EvalType.FUNCTION) { + GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); + current = new GeneralFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { + AggFunction instance = (AggFunction) funcDesc.newInstance(); + if (type == EvalType.AGG_FUNCTION) { + current = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + } else { + current = new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params, null); + } + } + } catch (ClassNotFoundException cnfe) { + throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + } catch (InternalException ie) { + throw new NoSuchFunctionException(funcDesc.getSignature(), funcDesc.getParamTypes()); + } + } else { + throw new RuntimeException("Unknown EvalType: " + type.name()); + } + + evalNodeMap.put(protoNode.getId(), current); + } + + return current; + } + + public static Datum deserialize(PlanProto.Datum datum) { + switch (datum.getType()) { + case BOOLEAN: + return DatumFactory.createBool(datum.getBoolean()); + case CHAR: + return DatumFactory.createChar(datum.getText()); + case INT1: + case INT2: + return DatumFactory.createInt2((short) datum.getInt4()); + case INT4: + return DatumFactory.createInt4(datum.getInt4()); + case INT8: + return DatumFactory.createInt8(datum.getInt8()); + case FLOAT4: + return DatumFactory.createFloat4(datum.getFloat4()); + case FLOAT8: + return DatumFactory.createFloat8(datum.getFloat8()); + case VARCHAR: + case TEXT: + return DatumFactory.createText(datum.getText()); + case TIMESTAMP: + return new TimestampDatum(datum.getInt8()); + case DATE: + return DatumFactory.createDate(datum.getInt4()); + case TIME: + return DatumFactory.createTime(datum.getInt8()); + case BINARY: + case BLOB: + return DatumFactory.createBlob(datum.getBlob().toByteArray()); + case INTERVAL: + return new IntervalDatum(datum.getInterval().getMonth(), datum.getInterval().getMsec()); + case NULL_TYPE: + return NullDatum.get(); + default: + throw new RuntimeException("Unknown data type: " + datum.getType().name()); + } + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java new file mode 100644 index 0000000000..62ca702248 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java @@ -0,0 +1,310 @@ +/* + * Lisensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.IntervalDatum; +import org.apache.tajo.engine.eval.*; +import org.apache.tajo.engine.plan.proto.PlanProto; + +import java.util.Map; +import java.util.Stack; + +/** + * It traverses an eval tree consisting of a number of {@link org.apache.tajo.engine.eval.EvalNode} + * in a postfix traverse order. The postfix traverse order guarantees that all child nodes of some node N + * were already visited when the node N is visited. This manner makes tree serialization possible in a simple logic. + */ +public class EvalTreeProtoSerializer + extends SimpleEvalNodeVisitor { + + private static final EvalTreeProtoSerializer instance; + + static { + instance = new EvalTreeProtoSerializer(); + } + + public static class EvalTreeProtoBuilderContext { + private int seqId = 0; + private Map idMap = Maps.newHashMap(); + private PlanProto.EvalTree.Builder evalTreeBuilder = PlanProto.EvalTree.newBuilder(); + } + + public static PlanProto.EvalTree serialize(EvalNode evalNode) { + EvalTreeProtoSerializer.EvalTreeProtoBuilderContext context = + new EvalTreeProtoSerializer.EvalTreeProtoBuilderContext(); + instance.visit(context, evalNode, new Stack()); + return context.evalTreeBuilder.build(); + } + + private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalNode) { + int selfId; + if (context.idMap.containsKey(evalNode)) { + selfId = context.idMap.get(evalNode); + } else { + selfId = context.seqId++; + context.idMap.put(evalNode, selfId); + } + + return selfId; + } + + private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { + int [] childIds = new int[evalNode.childNum()]; + for (int i = 0; i < evalNode.childNum(); i++) { + if (context.idMap.containsKey(evalNode.getChild(i))) { + childIds[i] = context.idMap.get(evalNode.getChild(i)); + } else { + childIds[i] = context.seqId++; + } + } + return childIds; + } + + private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { + PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); + nodeBuilder.setId(id); + nodeBuilder.setDataType(eval.getValueType()); + nodeBuilder.setType(PlanProto.EvalType.valueOf(eval.getType().name())); + return nodeBuilder; + } + + @Override + public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack, UnaryEval unary) { + // visiting and registering childs + super.visitUnaryEval(context, stack, unary); + int [] childIds = registerGetChildIds(context, unary); + + // building itself + PlanProto.UnaryEval.Builder unaryBuilder = PlanProto.UnaryEval.newBuilder(); + unaryBuilder.setChildId(childIds[0]); + if (unary.getType() == EvalType.IS_NULL) { + IsNullEval isNullEval = (IsNullEval) unary; + unaryBuilder.setNegative(isNullEval.isNot()); + } else if (unary.getType() == EvalType.SIGNED) { + SignedEval signedEval = (SignedEval) unary; + unaryBuilder.setNegative(signedEval.isNegative()); + } else if (unary.getType() == EvalType.CAST) { + CastEval castEval = (CastEval) unary; + unaryBuilder.setCastingType(castEval.getValueType()); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, unary); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, unary); + builder.setUnary(unaryBuilder); + context.evalTreeBuilder.addNodes(builder); + return unary; + } + + @Override + public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack stack, BinaryEval binary) { + // visiting and registering childs + super.visitBinaryEval(context, stack, binary); + int [] childIds = registerGetChildIds(context, binary); + + // building itself + PlanProto.BinaryEval.Builder binaryBuilder = PlanProto.BinaryEval.newBuilder(); + binaryBuilder.setLhsId(childIds[0]); + binaryBuilder.setRhsId(childIds[1]); + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, binary); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, binary); + builder.setBinary(binaryBuilder); + context.evalTreeBuilder.addNodes(builder); + return binary; + } + + @Override + public EvalNode visitConst(EvalTreeProtoBuilderContext context, ConstEval constant, Stack stack) { + int selfId = registerAndGetId(context, constant); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constant); + builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constant.getValue()))); + context.evalTreeBuilder.addNodes(builder); + return constant; + } + + @Override + public EvalNode visitRowConstant(EvalTreeProtoBuilderContext context, RowConstantEval rowConst, + Stack stack) { + + PlanProto.RowConstEval.Builder rowConstBuilder = PlanProto.RowConstEval.newBuilder(); + for (Datum d : rowConst.getValues()) { + rowConstBuilder.addValues(serialize(d)); + } + + int selfId = registerAndGetId(context, rowConst); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, rowConst); + builder.setRowConst(rowConstBuilder); + context.evalTreeBuilder.addNodes(builder); + return rowConst; + } + + public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { + int selfId = registerAndGetId(context, field); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, field); + builder.setField(field.getColumnRef().getProto()); + context.evalTreeBuilder.addNodes(builder); + return field; + } + + public EvalNode visitBetween(EvalTreeProtoBuilderContext context, BetweenPredicateEval between, + Stack stack) { + // visiting and registering childs + super.visitBetween(context, between, stack); + int [] childIds = registerGetChildIds(context, between); + Preconditions.checkState(childIds.length == 3, "Between must have three childs, but there are " + childIds.length + + " child nodes"); + + // building itself + PlanProto.BetweenEval.Builder betweenBuilder = PlanProto.BetweenEval.newBuilder(); + betweenBuilder.setNegative(between.isNot()); + betweenBuilder.setSymmetric(between.isSymmetric()); + betweenBuilder.setPredicand(childIds[0]); + betweenBuilder.setBegin(childIds[1]); + betweenBuilder.setEnd(childIds[2]); + + int selfId = registerAndGetId(context, between); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, between); + builder.setBetween(betweenBuilder); + context.evalTreeBuilder.addNodes(builder); + return between; + } + + public EvalNode visitCaseWhen(EvalTreeProtoBuilderContext context, CaseWhenEval caseWhen, Stack stack) { + // visiting and registering childs + super.visitCaseWhen(context, caseWhen, stack); + int [] childIds = registerGetChildIds(context, caseWhen); + Preconditions.checkState(childIds.length > 0, "Case When must have at least one child, but there is no child"); + + // building itself + PlanProto.CaseWhenEval.Builder caseWhenBuilder = PlanProto.CaseWhenEval.newBuilder(); + int ifCondsNum = childIds.length - (caseWhen.hasElse() ? 1 : 0); + for (int i = 0; i < ifCondsNum; i++) { + caseWhenBuilder.addIfConds(childIds[i]); + } + if (caseWhen.hasElse()) { + caseWhenBuilder.setElse(childIds[childIds.length - 1]); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, caseWhen); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, caseWhen); + builder.setCasewhen(caseWhenBuilder); + context.evalTreeBuilder.addNodes(builder); + + return caseWhen; + } + + public EvalNode visitIfThen(EvalTreeProtoBuilderContext context, CaseWhenEval.IfThenEval ifCond, + Stack stack) { + // visiting and registering childs + super.visitIfThen(context, ifCond, stack); + int [] childIds = registerGetChildIds(context, ifCond); + + // building itself + PlanProto.IfCondEval.Builder ifCondBuilder = PlanProto.IfCondEval.newBuilder(); + ifCondBuilder.setCondition(childIds[0]); + ifCondBuilder.setThen(childIds[1]); + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, ifCond); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, ifCond); + builder.setIfCond(ifCondBuilder); + context.evalTreeBuilder.addNodes(builder); + + return ifCond; + } + + public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval function, Stack stack) { + // visiting and registering childs + super.visitFuncCall(context, function, stack); + int [] childIds = registerGetChildIds(context, function); + + // building itself + PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); + funcBuilder.setFuncion(function.getFuncDesc().getProto()); + for (int i = 0; i < childIds.length; i++) { + funcBuilder.addParamIds(childIds[i]); + } + + // registering itself and building EvalNode + int selfId = registerAndGetId(context, function); + PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, function); + builder.setFunction(funcBuilder); + context.evalTreeBuilder.addNodes(builder); + + return function; + } + + public static PlanProto.Datum serialize(Datum datum) { + PlanProto.Datum.Builder builder = PlanProto.Datum.newBuilder(); + + builder.setType(datum.type()); + + switch (datum.type()) { + case NULL_TYPE: + break; + case BOOLEAN: + builder.setBoolean(datum.asBool()); + break; + case INT1: + case INT2: + case INT4: + case DATE: + builder.setInt4(datum.asInt4()); + break; + case INT8: + case TIMESTAMP: + case TIME: + builder.setInt8(datum.asInt8()); + break; + case FLOAT4: + builder.setFloat4(datum.asFloat4()); + break; + case FLOAT8: + builder.setFloat8(datum.asFloat8()); + break; + case CHAR: + case VARCHAR: + case TEXT: + builder.setText(datum.asChars()); + break; + case BINARY: + case BLOB: + builder.setBlob(ByteString.copyFrom(datum.asByteArray())); + break; + case INTERVAL: + IntervalDatum interval = (IntervalDatum) datum; + PlanProto.Interval.Builder intervalBuilder = PlanProto.Interval.newBuilder(); + intervalBuilder.setMonth(interval.getMonths()); + intervalBuilder.setMsec(interval.getMilliSeconds()); + builder.setInterval(intervalBuilder); + break; + default: + throw new RuntimeException("Unknown data type: " + datum.type().name()); + } + + return builder.build(); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 87e54de332..6a3af98b09 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -352,7 +352,7 @@ public EvalNode visitCaseWhen(Context ctx, Stack stack, CaseWhenPredicate for (CaseWhenPredicate.WhenExpr when : caseWhen.getWhens()) { condition = visit(ctx, stack, when.getCondition()); result = visit(ctx, stack, when.getResult()); - caseWhenEval.addWhen(condition, result); + caseWhenEval.addIfCond(condition, result); } if (caseWhen.hasElseResult()) { diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto new file mode 100644 index 0000000000..fa8cbb8c44 --- /dev/null +++ b/tajo-core/src/main/proto/Plan.proto @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tajo.plan; +option cc_generic_services = false; +option java_package = "org.apache.tajo.engine.plan.proto"; +option java_outer_classname = "PlanProto"; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +import "PrimitiveProtos.proto"; +import "CatalogProtos.proto"; +import "DataTypes.proto"; + +enum NodeType { + BST_INDEX_SCAN = 0; + EXCEPT = 1; + EXPRS = 2; + DISTINCT_GROUP_BY = 3; + GROUP_BY = 4; + HAVING = 5; + JOIN = 6; + INSERT = 7; + INTERSECT = 8; + LIMIT = 9; + PARTITIONS_SCAN = 10; + PROJECTION = 11; + ROOT = 12; + SCAN = 13; + SELECTION = 14; + SORT = 15; + STORE = 16; + TABLE_SUBQUERY = 17; + UNION = 18; + WINDOW_AGG = 19; + + CREATE_DATABASE = 20; + DROP_DATABASE = 21; + CREATE_TABLE = 22; + DROP_TABLE = 23; + ALTER_TABLESPACE = 24; + ALTER_TABLE = 25; + TRUNCATE_TABLE = 26; +} + +message LogicalPlan { + required KeyValueSetProto adjacentList = 1; +} + +message LogicalNode { + required int32 pid = 1; + required NodeType type = 2; + required SchemaProto in_schema = 3; + required SchemaProto out_schema = 4; + required NodeSpec spec = 5; +} + +message NodeSpec { + optional ScanNode scan = 1; +} + +message ScanNode { + required TableDescProto table = 1; + optional string alias = 2; + required SchemaProto schema = 3; +} + + +enum EvalType { + NOT = 0; + AND = 1; + OR = 2; + EQUAL = 3; + IS_NULL = 4; + NOT_EQUAL = 5; + LTH = 6; + LEQ = 7; + GTH = 8; + GEQ = 9; + PLUS = 10; + MINUS = 11; + MODULAR = 12; + MULTIPLY = 13; + DIVIDE = 14; + + // Binary Bitwise expressions + BIT_AND = 15; + BIT_OR = 16; + BIT_XOR = 17; + + // Function + WINDOW_FUNCTION = 18; + AGG_FUNCTION = 19; + FUNCTION = 20; + + // String operator or pattern matching predicates + LIKE = 21; + SIMILAR_TO = 22; + REGEX = 23; + CONCATENATE = 24; + + // Other predicates + BETWEEN = 25; + CASE = 26; + IF_THEN = 27; + IN = 28; + + // Value or Reference + SIGNED = 29; + CAST = 30; + ROW_CONSTANT = 31; + FIELD = 32; + CONST = 33; +} + +message EvalTree { + repeated EvalNode nodes = 1; +} + +message EvalNode { + required int32 id = 1; + required EvalType type = 2; + required DataType data_type = 3; + + optional UnaryEval unary = 4; // NOT + optional BinaryEval binary = 5; + optional ConstEval const = 6; + optional ColumnProto field = 7; // field eval + optional FunctionEval function = 8; + optional RowConstEval rowConst = 9; + optional BetweenEval between = 10; + optional CaseWhenEval casewhen = 11; + optional IfCondEval ifCond = 12; +} + +message UnaryEval { + required int32 child_id = 1; + optional DataType castingType = 2; + optional bool negative = 3; +} + +message BinaryEval { + required int32 lhs_id = 1; + required int32 rhs_id = 2; + optional bool negative = 3 [default = false]; +} + +message BetweenEval { + required int32 predicand = 1; + required int32 begin = 2; + required int32 end = 3; + optional bool negative = 4 [default = false]; + optional bool symmetric = 5 [default = false]; +} + +message CaseWhenEval { + repeated int32 ifConds = 1; + optional int32 else = 2; +} + +message IfCondEval { + required int32 condition = 1; + required int32 then = 2; +} + +message ConstEval { + required Datum value = 1; +} + +message RowConstEval { + repeated Datum values = 1; +} + +message FunctionEval { + required FunctionDescProto funcion = 1; + repeated int32 paramIds = 2; +} + +message Datum { + required Type type = 1; + optional bool boolean = 2; + optional int32 int4 = 3; // int4, date + optional int64 int8 = 4; // int8, timestamp, and time + optional float float4 = 5; + optional double float8 = 6; + optional string text = 7; + optional bytes blob = 8; + optional Interval interval = 12; +} + +message Interval { + optional int32 month = 1 [default = 0]; + optional int64 msec = 2; +} \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 7f9436a482..3bee4f3d7e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -31,6 +31,9 @@ import org.apache.tajo.datum.*; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.plan.EvalTreeProtoDeserializer; +import org.apache.tajo.engine.plan.EvalTreeProtoSerializer; +import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.SchemaUtil; @@ -53,6 +56,7 @@ import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; public class ExprTestBase { private static TajoTestingCluster util; @@ -135,9 +139,21 @@ private static Target[] getRawTargets(String query, boolean condition) throws Pl if (targets == null) { throw new PlanningException("Wrong query statement or query plan: " + parsedResults.get(0).getHistoryStatement()); } + + // Trying regression test for cloning, (de)serialization for json and protocol buffer + for (Target t : targets) { + try { + assertEquals(t.getEvalTree(), t.getEvalTree().clone()); + } catch (CloneNotSupportedException e) { + fail(e.getMessage()); + } + } for (Target t : targets) { assertJsonSerDer(t.getEvalTree()); } + for (Target t : targets) { + assertEvalTreeProtoSerDer(t.getEvalTree()); + } return targets; } @@ -226,4 +242,9 @@ public void testEval(Schema schema, String tableName, String csvTuple, String qu } } } + + public static void assertEvalTreeProtoSerDer(EvalNode evalNode) { + PlanProto.EvalTree converted = EvalTreeProtoSerializer.serialize(evalNode); + assertEquals(evalNode, EvalTreeProtoDeserializer.deserialize(converted)); + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index da43025f50..beeb1c67e4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -31,7 +31,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.*; import static org.junit.Assert.*; -public class TestEvalTree extends ExprTestBase{ +public class TestEvalTree extends ExprTestBase { @Test public void testTupleEval() throws CloneNotSupportedException { ConstEval e1 = new ConstEval(DatumFactory.createInt4(1)); @@ -89,6 +89,15 @@ public DataType getValueType() { return CatalogUtil.newSimpleDataType(BOOLEAN); } + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getChild(int idx) { + return null; + } } public static class MockFalseExpr extends EvalNode { @@ -126,6 +135,16 @@ public void postOrder(EvalNodeVisitor visitor) { public DataType getValueType() { return CatalogUtil.newSimpleDataType(BOOLEAN); } + + @Override + public int childNum() { + return 0; + } + + @Override + public EvalNode getChild(int idx) { + return null; + } } @Test From 56df752d142d0cf0d2bf1d968ba1f99fc2a89428 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 17 Aug 2014 15:54:21 +0900 Subject: [PATCH 13/31] Refactor some methods and variables. --- .../engine/plan/EvalTreeProtoSerializer.java | 78 ++++++++----------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java index 62ca702248..a129fb2df6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java @@ -46,26 +46,14 @@ public class EvalTreeProtoSerializer public static class EvalTreeProtoBuilderContext { private int seqId = 0; private Map idMap = Maps.newHashMap(); - private PlanProto.EvalTree.Builder evalTreeBuilder = PlanProto.EvalTree.newBuilder(); + private PlanProto.EvalTree.Builder treeBuilder = PlanProto.EvalTree.newBuilder(); } public static PlanProto.EvalTree serialize(EvalNode evalNode) { EvalTreeProtoSerializer.EvalTreeProtoBuilderContext context = new EvalTreeProtoSerializer.EvalTreeProtoBuilderContext(); instance.visit(context, evalNode, new Stack()); - return context.evalTreeBuilder.build(); - } - - private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalNode) { - int selfId; - if (context.idMap.containsKey(evalNode)) { - selfId = context.idMap.get(evalNode); - } else { - selfId = context.seqId++; - context.idMap.put(evalNode, selfId); - } - - return selfId; + return context.treeBuilder.build(); } private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { @@ -80,11 +68,19 @@ private int registerAndGetId(EvalTreeProtoBuilderContext context, EvalNode evalN return childIds; } - private PlanProto.EvalNode.Builder createEvalBuilder(int id, EvalNode eval) { + private PlanProto.EvalNode.Builder createEvalBuilder(EvalTreeProtoBuilderContext context, EvalNode node) { + int sid; // serialization sequence id + if (context.idMap.containsKey(node)) { + sid = context.idMap.get(node); + } else { + sid = context.seqId++; + context.idMap.put(node, sid); + } + PlanProto.EvalNode.Builder nodeBuilder = PlanProto.EvalNode.newBuilder(); - nodeBuilder.setId(id); - nodeBuilder.setDataType(eval.getValueType()); - nodeBuilder.setType(PlanProto.EvalType.valueOf(eval.getType().name())); + nodeBuilder.setId(sid); + nodeBuilder.setDataType(node.getValueType()); + nodeBuilder.setType(PlanProto.EvalType.valueOf(node.getType().name())); return nodeBuilder; } @@ -109,10 +105,9 @@ public EvalNode visitUnaryEval(EvalTreeProtoBuilderContext context, Stack stack) { - int selfId = registerAndGetId(context, constant); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, constant); + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, constant); builder.setConst(PlanProto.ConstEval.newBuilder().setValue(serialize(constant.getValue()))); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return constant; } @@ -153,18 +146,16 @@ public EvalNode visitRowConstant(EvalTreeProtoBuilderContext context, RowConstan rowConstBuilder.addValues(serialize(d)); } - int selfId = registerAndGetId(context, rowConst); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, rowConst); + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, rowConst); builder.setRowConst(rowConstBuilder); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return rowConst; } public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { - int selfId = registerAndGetId(context, field); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, field); + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, field); builder.setField(field.getColumnRef().getProto()); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return field; } @@ -184,10 +175,10 @@ public EvalNode visitBetween(EvalTreeProtoBuilderContext context, BetweenPredica betweenBuilder.setBegin(childIds[1]); betweenBuilder.setEnd(childIds[2]); - int selfId = registerAndGetId(context, between); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, between); + // registering itself and building EvalNode + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, between); builder.setBetween(betweenBuilder); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return between; } @@ -208,10 +199,9 @@ public EvalNode visitCaseWhen(EvalTreeProtoBuilderContext context, CaseWhenEval } // registering itself and building EvalNode - int selfId = registerAndGetId(context, caseWhen); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, caseWhen); + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, caseWhen); builder.setCasewhen(caseWhenBuilder); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return caseWhen; } @@ -228,10 +218,9 @@ public EvalNode visitIfThen(EvalTreeProtoBuilderContext context, CaseWhenEval.If ifCondBuilder.setThen(childIds[1]); // registering itself and building EvalNode - int selfId = registerAndGetId(context, ifCond); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, ifCond); + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, ifCond); builder.setIfCond(ifCondBuilder); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return ifCond; } @@ -249,10 +238,9 @@ public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval } // registering itself and building EvalNode - int selfId = registerAndGetId(context, function); - PlanProto.EvalNode.Builder builder = createEvalBuilder(selfId, function); + PlanProto.EvalNode.Builder builder = createEvalBuilder(context, function); builder.setFunction(funcBuilder); - context.evalTreeBuilder.addNodes(builder); + context.treeBuilder.addNodes(builder); return function; } From b5a1ed853e8342c89872b9a312756264ec1f4630 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 17 Aug 2014 15:56:34 +0900 Subject: [PATCH 14/31] Refactor ScanNode and added (de)serializeion for ScanNode. --- .../plan/LogicalNodeTreeDeserializer.java | 100 ++++++++++++++++ .../plan/LogicalNodeTreeSerializer.java | 112 ++++++++++++++++++ .../planner/LogicalPlanPreprocessor.java | 5 +- .../tajo/engine/planner/LogicalPlanner.java | 6 +- .../engine/planner/logical/RelationNode.java | 2 +- .../tajo/engine/planner/logical/ScanNode.java | 2 +- .../planner/logical/TableSubQueryNode.java | 2 +- .../planner/nameresolver/NameResolver.java | 6 +- .../nameresolver/ResolverByLegacy.java | 2 +- .../rewrite/ProjectionPushDownRule.java | 2 +- tajo-core/src/main/proto/Plan.proto | 25 ++-- 11 files changed, 239 insertions(+), 25 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java new file mode 100644 index 0000000000..49a5998903 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java @@ -0,0 +1,100 @@ +/* + * Lisensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.derby.impl.store.access.sort.Scan; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.engine.eval.EvalType; +import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.logical.ScanNode; + +import java.util.*; + +public class LogicalNodeTreeDeserializer { + private static final LogicalNodeTreeDeserializer instance; + + static { + instance = new LogicalNodeTreeDeserializer(); + } + + public static LogicalNode deserilize(PlanProto.LogicalNodeTree tree) { + Map nodeMap = Maps.newHashMap(); + + // sort serialized logical nodes in an ascending order of their sids + List nodeList = Lists.newArrayList(tree.getNodesList()); + Collections.sort(nodeList, new Comparator() { + @Override + public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { + return o1.getSid() - o2.getSid(); + } + }); + + LogicalNode current = null; + + // The sorted order is the same of a postfix traverse order. + // So, it sequentially transforms each serialized node into a LogicalNode instance in a postfix order of + // the original logical node tree. + + Iterator it = nodeList.iterator(); + while (it.hasNext()) { + PlanProto.LogicalNode protoNode = it.next(); + + NodeType type = convertType(protoNode.getType()); + + switch (type) { + + case SCAN: + ScanNode scan = new ScanNode(protoNode.getPid()); + + PlanProto.ScanNode scanProto = protoNode.getScan(); + if (scanProto.hasAlias()) { + scan.init(new TableDesc(scanProto.getTable()), scanProto.getAlias()); + } else { + scan.init(new TableDesc(scanProto.getTable())); + } + + if (scanProto.getTargetsCount() > 0) { + Target[] targets = new Target[scanProto.getTargetsCount()]; + for (int i = 0; i < scanProto.getTargetsCount(); i++) { + targets[i] = convertTarget(scanProto.getTargets(i)); + } + } + + default: + throw new RuntimeException("Unknown NodeType: " + type.name()); + } + } + + return current; + } + + public static NodeType convertType(PlanProto.NodeType type) { + return NodeType.valueOf(type.name()); + } + + public static Target convertTarget(PlanProto.Target targetProto) { + return new Target(EvalTreeProtoDeserializer.deserialize(targetProto.getExpr()), targetProto.getAlias()); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java new file mode 100644 index 0000000000..0a571ff91b --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java @@ -0,0 +1,112 @@ +/* + * Lisensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.plan; + +import com.google.common.collect.Maps; +import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.logical.ScanNode; + +import java.util.Map; +import java.util.Stack; + +public class LogicalNodeTreeSerializer extends BasicLogicalPlanVisitor { + + private static final LogicalNodeTreeSerializer instance; + + static { + instance = new LogicalNodeTreeSerializer(); + } + + public static PlanProto.LogicalNode serialize(LogicalNode node) throws PlanningException { + instance.visit(new SerializeContext(), null, null, node, new Stack()); + + return null; + } + + public static PlanProto.NodeType convertType(NodeType type) { + return PlanProto.NodeType.valueOf(type.name()); + } + + public static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext context, LogicalNode node) { + int selfId; + if (context.idMap.containsKey(node)) { + selfId = context.idMap.get(node); + } else { + selfId = context.seqId++; + context.idMap.put(node, selfId); + } + + PlanProto.LogicalNode.Builder nodeBuilder = PlanProto.LogicalNode.newBuilder(); + nodeBuilder.setSid(selfId); + nodeBuilder.setPid(node.getPID()); + nodeBuilder.setType(convertType(node.getType())); + nodeBuilder.setInSchema(node.getInSchema().getProto()); + nodeBuilder.setOutSchema(node.getOutSchema().getProto()); + return nodeBuilder; + } + + public static class SerializeContext { + private int seqId = 0; + private Map idMap = Maps.newHashMap(); + private PlanProto.LogicalNodeTree.Builder treeBuilder = PlanProto.LogicalNodeTree.newBuilder(); + } + + @Override + public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + ScanNode scan, Stack stack) throws PlanningException { + + // building itself + PlanProto.ScanNode.Builder scanBuilder = PlanProto.ScanNode.newBuilder(); + scanBuilder.setTable(scan.getTableDesc().getProto()); + if (scan.hasAlias()) { + scanBuilder.setAlias(scan.getAlias()); + } + + if (scan.hasTargets()) { + for (Target t : scan.getTargets()) { + PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); + + targetBuilder.setExpr(EvalTreeProtoSerializer.serialize(t.getEvalTree())); + + if (t.hasAlias()) { + targetBuilder.setAlias(t.getAlias()); + } + scanBuilder.addTargets(targetBuilder); + } + } + + if (scan.hasQual()) { + scanBuilder.setQual(EvalTreeProtoSerializer.serialize(scan.getQual())); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, scan); + nodeBuilder.setScan(scanBuilder); + + context.treeBuilder.addNodes(nodeBuilder); + + return scan; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 674525c062..47d539fbc6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -29,7 +29,6 @@ import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.nameresolver.NameResolver; import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; -import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.util.TUtil; @@ -108,7 +107,7 @@ public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAster throw new NoSuchColumnException(CatalogUtil.buildFQName(qualifier, "*")); } - Schema schema = relationOp.getTableSchema(); + Schema schema = relationOp.getLogicalSchema(); Column[] resolvedColumns = new Column[schema.size()]; return schema.getColumns().toArray(resolvedColumns); } else { // if a column reference is not qualified @@ -119,7 +118,7 @@ public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAster while (iterator.hasNext()) { relationOp = iterator.next(); - schema = relationOp.getTableSchema(); + schema = relationOp.getLogicalSchema(); resolvedColumns.addAll(schema.getColumns()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 89382f48d8..901d9b4a9e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -499,7 +499,7 @@ public static void verifyProjectedFields(QueryBlock block, Projectable projectab } else if (projectable instanceof RelationNode) { RelationNode relationNode = (RelationNode) projectable; - verifyIfTargetsCanBeEvaluated(relationNode.getTableSchema(), (Projectable) relationNode); + verifyIfTargetsCanBeEvaluated(relationNode.getLogicalSchema(), (Projectable) relationNode); } else { verifyIfTargetsCanBeEvaluated(projectable.getInSchema(), projectable); @@ -1271,7 +1271,7 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e private static LinkedHashSet createFieldTargetsFromRelation(QueryBlock block, RelationNode relationNode, Set newlyEvaluatedRefNames) { LinkedHashSet targets = Sets.newLinkedHashSet(); - for (Column column : relationNode.getTableSchema().getColumns()) { + for (Column column : relationNode.getLogicalSchema().getColumns()) { String aliasName = block.namedExprsMgr.checkAndGetIfAliasedColumn(column.getQualifiedName()); if (aliasName != null) { targets.add(new Target(new FieldEval(column), aliasName)); @@ -1976,7 +1976,7 @@ public static boolean checkIfBeEvaluatedAtRelation(QueryBlock block, EvalNode ev return false; } - if (columnRefs.size() > 0 && !node.getTableSchema().containsAll(columnRefs)) { + if (columnRefs.size() > 0 && !node.getLogicalSchema().containsAll(columnRefs)) { return false; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java index 83c16cdd62..fc4090559a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java @@ -45,5 +45,5 @@ protected RelationNode(int pid, NodeType nodeType) { public abstract String getCanonicalName(); - public abstract Schema getTableSchema(); + public abstract Schema getLogicalSchema(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java index 9582deec26..f859ff0ca6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java @@ -101,7 +101,7 @@ public String getCanonicalName() { } } - public Schema getTableSchema() { + public Schema getLogicalSchema() { return logicalSchema; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java index 3c808fcd78..82af9297ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java @@ -66,7 +66,7 @@ public String getCanonicalName() { } @Override - public Schema getTableSchema() { + public Schema getLogicalSchema() { // an output schema can be determined by targets. So, an input schema of // TableSubQueryNode is only eligible for table schema. // diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java index aee5d43b6f..95a11611de 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java @@ -128,7 +128,7 @@ static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBloc CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName)); } - Schema schema = relationOp.getTableSchema(); + Schema schema = relationOp.getLogicalSchema(); Column column = schema.getColumn(canonicalName); return column; @@ -173,7 +173,7 @@ static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block, List candidates = TUtil.newList(); for (RelationNode rel : block.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); + Column found = rel.getLogicalSchema().getColumn(columnRef.getName()); if (found != null) { candidates.add(found); } @@ -201,7 +201,7 @@ static Column resolveFromAllRelsInAllBlocks(LogicalPlan plan, ColumnReferenceExp for (LogicalPlan.QueryBlock eachBlock : plan.getQueryBlocks()) { for (RelationNode rel : eachBlock.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); + Column found = rel.getLogicalSchema().getColumn(columnRef.getName()); if (found != null) { candidates.add(found); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java index 396bc1b184..d028f33897 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java @@ -74,7 +74,7 @@ private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.Q Schema currentNodeSchema = null; if (currentNode != null) { if (currentNode instanceof RelationNode) { - currentNodeSchema = ((RelationNode) currentNode).getTableSchema(); + currentNodeSchema = ((RelationNode) currentNode).getLogicalSchema(); } else { currentNodeSchema = currentNode.getInSchema(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java index ec5df04fa7..08d1b6c4f6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java @@ -1041,7 +1041,7 @@ public LogicalNode visitScan(Context context, LogicalPlan plan, LogicalPlan.Quer if (node.hasTargets()) { targets = node.getTargets(); } else { - targets = PlannerUtil.schemaToTargets(node.getTableSchema()); + targets = PlannerUtil.schemaToTargets(node.getLogicalSchema()); } LinkedHashSet projectedTargets = Sets.newLinkedHashSet(); diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index fa8cbb8c44..1107fc45c4 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -58,28 +58,31 @@ enum NodeType { TRUNCATE_TABLE = 26; } -message LogicalPlan { - required KeyValueSetProto adjacentList = 1; +message LogicalNodeTree { + repeated LogicalNode nodes = 1; } message LogicalNode { - required int32 pid = 1; - required NodeType type = 2; - required SchemaProto in_schema = 3; - required SchemaProto out_schema = 4; - required NodeSpec spec = 5; -} + required int32 sid = 1; + required int32 pid = 2; + required NodeType type = 3; + required SchemaProto in_schema = 4; + required SchemaProto out_schema = 5; -message NodeSpec { - optional ScanNode scan = 1; + optional ScanNode scan = 6; } message ScanNode { required TableDescProto table = 1; optional string alias = 2; - required SchemaProto schema = 3; + repeated Target targets = 4; + optional EvalTree qual = 5; } +message Target { + required EvalTree expr = 1; + required string alias = 2; +} enum EvalType { NOT = 0; From 5d30cef87977e5d7582857525e37512881934a56 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 17 Aug 2014 21:39:40 +0900 Subject: [PATCH 15/31] Add (de)serialization scan, filter, groupby, sort, and limit nodes. --- .../plan/LogicalNodeTreeSerializer.java | 108 ++++++++++++++++-- .../engine/planner/AlterTablespaceNode.java | 10 ++ .../planner/logical/AlterTableNode.java | 10 ++ .../engine/planner/logical/BinaryNode.java | 16 +++ .../planner/logical/CreateDatabaseNode.java | 10 ++ .../planner/logical/DropDatabaseNode.java | 10 ++ .../engine/planner/logical/DropTableNode.java | 10 ++ .../engine/planner/logical/EvalExprNode.java | 10 ++ .../engine/planner/logical/GroupbyNode.java | 45 +++++--- .../engine/planner/logical/LogicalNode.java | 4 + .../tajo/engine/planner/logical/ScanNode.java | 10 ++ .../planner/logical/TableSubQueryNode.java | 10 ++ .../planner/logical/TruncateTableNode.java | 10 ++ .../engine/planner/logical/UnaryNode.java | 14 +++ tajo-core/src/main/proto/Plan.proto | 52 +++++++++ 15 files changed, 305 insertions(+), 24 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java index 0a571ff91b..85ca3bc81f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java @@ -19,14 +19,13 @@ package org.apache.tajo.engine.plan; import com.google.common.collect.Maps; +import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; import org.apache.tajo.engine.planner.LogicalPlan; import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.engine.planner.logical.*; import java.util.Map; import java.util.Stack; @@ -46,9 +45,7 @@ public static PlanProto.LogicalNode serialize(LogicalNode node) throws PlanningE return null; } - public static PlanProto.NodeType convertType(NodeType type) { - return PlanProto.NodeType.valueOf(type.name()); - } + public static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext context, LogicalNode node) { int selfId; @@ -74,6 +71,88 @@ public static class SerializeContext { private PlanProto.LogicalNodeTree.Builder treeBuilder = PlanProto.LogicalNodeTree.newBuilder(); } + @Override + public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + LimitNode limit, Stack stack) throws PlanningException { + super.visitLimit(context, plan, block, limit, stack); + + int [] childIds = registerGetChildIds(context, limit); + + // building itself + PlanProto.LimitNode.Builder limitBuilder = PlanProto.LimitNode.newBuilder(); + limitBuilder.setChildId(childIds[0]); + limitBuilder.setFetchFirstNum(limit.getFetchFirstNum()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, limit); + nodeBuilder.setLimit(limitBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return limit; + } + + @Override + public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + SortNode sort, Stack stack) throws PlanningException { + super.visitSort(context, plan, block, sort, stack); + + int [] childIds = registerGetChildIds(context, sort); + + // building itself + PlanProto.SortNode.Builder sortBuilder = PlanProto.SortNode.newBuilder(); + sortBuilder.setChildId(childIds[0]); + for (int i = 0; i < sort.getSortKeys().length; i++) { + sortBuilder.addSortSpecs(sort.getSortKeys()[i].getProto()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, sort); + nodeBuilder.setSort(sortBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return sort; + } + + public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + GroupbyNode groupbyNode, Stack stack) throws PlanningException { + super.visitGroupBy(context, plan, block, groupbyNode, new Stack()); + + int [] childIds = registerGetChildIds(context, groupbyNode); + + PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); + groupbyBuilder.setChildId(childIds[0]); + + for (int i = 0; i < groupbyNode.getGroupingColumns().length; i++) { + groupbyBuilder.addGroupingKeys(groupbyNode.getGroupingColumns()[i].getProto()); + } + for (int i = 0; i < groupbyNode.getGroupingColumns().length; i++) { + groupbyBuilder.addAggFunctions(EvalTreeProtoSerializer.serialize(groupbyNode.getAggFunctions()[i])); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, groupbyNode); + nodeBuilder.setGroupby(groupbyBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return groupbyNode; + } + + @Override + public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + SelectionNode filter, Stack stack) throws PlanningException { + super.visitFilter(context, plan, block, filter, stack); + + int [] childIds = registerGetChildIds(context, filter); + + // building itself + PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); + filterBuilder.setChildId(childIds[0]); + filterBuilder.setQual(EvalTreeProtoSerializer.serialize(filter.getQual())); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, filter); + nodeBuilder.setFilter(filterBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return filter; + } + @Override public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scan, Stack stack) throws PlanningException { @@ -104,9 +183,24 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, scan); nodeBuilder.setScan(scanBuilder); - context.treeBuilder.addNodes(nodeBuilder); return scan; } + + public static PlanProto.NodeType convertType(NodeType type) { + return PlanProto.NodeType.valueOf(type.name()); + } + + private int [] registerGetChildIds(SerializeContext context, LogicalNode node) { + int [] childIds = new int[node.childNum()]; + for (int i = 0; i < node.childNum(); i++) { + if (context.idMap.containsKey(node.getChild(i))) { + childIds[i] = context.idMap.get(node.getChild(i)); + } else { + childIds[i] = context.seqId++; + } + } + return childIds; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java index d64a89d3ab..d1bd35bd7b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java @@ -37,6 +37,16 @@ public AlterTablespaceNode(int pid) { super(pid, NodeType.ALTER_TABLESPACE); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public String getTablespaceName() { return tablespaceName; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java index 76a47d0b3b..8c1a19cc59 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java @@ -43,6 +43,16 @@ public AlterTableNode(int pid) { super(pid, NodeType.ALTER_TABLE); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public String getTableName() { return tableName; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java index 8aaeb58405..9d1c03078d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java @@ -28,6 +28,22 @@ public abstract class BinaryNode extends LogicalNode implements Cloneable, GsonO public BinaryNode(int pid, NodeType nodeType) { super(pid, nodeType); } + + @Override + public int childNum() { + return 2; + } + + @Override + public LogicalNode getChild(int idx) { + if (idx == 0) { + return leftChild; + } else if (idx == 1) { + return rightChild; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } public T getLeftChild() { return (T) this.leftChild; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java index 9dc73e2c9b..f560f8e891 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java @@ -30,6 +30,16 @@ public CreateDatabaseNode(int pid) { super(pid, NodeType.CREATE_DATABASE); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public void init(String databaseName, boolean ifNotExists) { this.databaseName = databaseName; this.ifNotExists = ifNotExists; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java index 1578759eb1..ff27992eb8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java @@ -30,6 +30,16 @@ public DropDatabaseNode(int pid) { super(pid, NodeType.DROP_DATABASE); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public void init(String databaseName, boolean ifExists) { this.databaseName = databaseName; this.ifExists = ifExists; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java index ac68a9c06c..e254cc8e59 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java @@ -30,6 +30,16 @@ public DropTableNode(int pid) { super(pid, NodeType.DROP_TABLE); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public void init(String tableName, boolean ifExists, boolean purge) { this.tableName = tableName; this.ifExists = ifExists; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java index 6ea3f4055e..04bfa6acb4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java @@ -33,6 +33,16 @@ public EvalExprNode(int pid) { super(pid, NodeType.EXPRS); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + @Override public boolean hasTargets() { return true; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java index 75aa52c12b..9bd7b19d3f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java @@ -27,10 +27,13 @@ import org.apache.tajo.util.TUtil; public class GroupbyNode extends UnaryNode implements Projectable, Cloneable { + private static final Column [] EMPTY_GROUPING_KEYS = new Column[] {}; + private static final AggregationFunctionCallEval [] EMPTY_AGG_FUNCS = new AggregationFunctionCallEval[] {}; + /** Grouping key sets */ - @Expose private Column [] groupingColumns; + @Expose private Column [] groupingKeys = EMPTY_GROUPING_KEYS; /** Aggregation Functions */ - @Expose private AggregationFunctionCallEval [] aggrFunctions; + @Expose private AggregationFunctionCallEval [] aggrFunctions = EMPTY_AGG_FUNCS; /** * It's a list of targets. The grouping columns should be followed by aggregation functions. * aggrFunctions keep actual aggregation functions, but it only contains field references. @@ -42,16 +45,20 @@ public GroupbyNode(int pid) { super(pid, NodeType.GROUP_BY); } + public int groupingKeyNum() { + return groupingKeys.length; + } + public final boolean isEmptyGrouping() { - return groupingColumns == null || groupingColumns.length == 0; + return groupingKeys.length == 0; } - public void setGroupingColumns(Column [] groupingColumns) { - this.groupingColumns = groupingColumns; + public void setGroupingColumns(Column [] groupingKeys) { + this.groupingKeys = groupingKeys; } public final Column [] getGroupingColumns() { - return this.groupingColumns; + return this.groupingKeys; } public final boolean isDistinct() { @@ -63,7 +70,11 @@ public void setDistinct(boolean distinct) { } public boolean hasAggFunctions() { - return this.aggrFunctions != null; + return this.aggrFunctions.length > 0; + } + + public int aggregationFunctionNum() { + return this.aggrFunctions.length; } public AggregationFunctionCallEval [] getAggFunctions() { @@ -96,8 +107,8 @@ public void setChild(LogicalNode subNode) { public String toString() { StringBuilder sb = new StringBuilder("GroupBy ("); - if (groupingColumns != null || groupingColumns.length > 0) { - sb.append("grouping set=").append(TUtil.arrayToString(groupingColumns)); + if (groupingKeys != null || groupingKeys.length > 0) { + sb.append("grouping set=").append(TUtil.arrayToString(groupingKeys)); sb.append(", "); } if (hasAggFunctions()) { @@ -112,7 +123,7 @@ public boolean equals(Object obj) { if (obj instanceof GroupbyNode) { GroupbyNode other = (GroupbyNode) obj; boolean eq = super.equals(other); - eq = eq && TUtil.checkEquals(groupingColumns, other.groupingColumns); + eq = eq && TUtil.checkEquals(groupingKeys, other.groupingKeys); eq = eq && TUtil.checkEquals(aggrFunctions, other.aggrFunctions); eq = eq && TUtil.checkEquals(targets, other.targets); return eq; @@ -124,10 +135,10 @@ public boolean equals(Object obj) { @Override public Object clone() throws CloneNotSupportedException { GroupbyNode grp = (GroupbyNode) super.clone(); - if (groupingColumns != null) { - grp.groupingColumns = new Column[groupingColumns.length]; - for (int i = 0; i < groupingColumns.length; i++) { - grp.groupingColumns[i] = groupingColumns[i]; + if (groupingKeys != null) { + grp.groupingKeys = new Column[groupingKeys.length]; + for (int i = 0; i < groupingKeys.length; i++) { + grp.groupingKeys[i] = groupingKeys[i]; } } @@ -151,7 +162,7 @@ public Object clone() throws CloneNotSupportedException { public String getShortPlanString() { StringBuilder sb = new StringBuilder(); sb.append(getType().name() + "(" + getPID() + ")").append("("); - Column [] groupingColumns = this.groupingColumns; + Column [] groupingColumns = this.groupingKeys; for (int j = 0; j < groupingColumns.length; j++) { sb.append(groupingColumns[j].getSimpleName()); if(j < groupingColumns.length - 1) { @@ -196,7 +207,7 @@ public PlanString getPlanString() { StringBuilder sb = new StringBuilder(); sb.append("("); - Column [] groupingColumns = this.groupingColumns; + Column [] groupingColumns = this.groupingKeys; for (int j = 0; j < groupingColumns.length; j++) { sb.append(groupingColumns[j].getSimpleName()); if(j < groupingColumns.length - 1) { @@ -243,7 +254,7 @@ public PlanString getPlanString() { * If so, it returns TRUE. Otherwise, it returns FALSE. */ public boolean isAggregationColumn(String simpleName) { - for (int i = groupingColumns.length; i < targets.length; i++) { + for (int i = groupingKeys.length; i < targets.length; i++) { if (simpleName.equals(targets[i].getNamedColumn().getSimpleName()) || simpleName.equals(targets[i].getAlias())) { return true; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java index f62caa45f4..7841c9cdbf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java @@ -58,6 +58,10 @@ public void setType(NodeType type) { this.type = type; } + public abstract int childNum(); + + public abstract LogicalNode getChild(int idx); + public double getCost() { return this.cost; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java index f859ff0ca6..0a0f5dd0ef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java @@ -42,6 +42,16 @@ protected ScanNode(int pid, NodeType nodeType) { super(pid, nodeType); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public ScanNode(int pid) { super(pid, NodeType.SCAN); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java index 82af9297ec..af4b88f789 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java @@ -35,6 +35,16 @@ public TableSubQueryNode(int pid) { super(pid, NodeType.TABLE_SUBQUERY); } + @Override + public int childNum() { + return 1; + } + + @Override + public LogicalNode getChild(int idx) { + return subQuery; + } + public void init(String tableName, LogicalNode subQuery) { this.tableName = tableName; if (subQuery != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java index b8d9cadc8c..c990428cd1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java @@ -32,6 +32,16 @@ public TruncateTableNode(int pid) { super(pid, NodeType.TRUNCATE_TABLE); } + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + public List getTableNames() { return tableNames; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java index 0b06e9ebf3..bcb84a1c3c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java @@ -31,6 +31,20 @@ public abstract class UnaryNode extends LogicalNode implements Cloneable { public UnaryNode(int pid, NodeType type) { super(pid, type); } + + @Override + public int childNum() { + return 1; + } + + @Override + public LogicalNode getChild(int idx) { + if (idx == 0) { + return child; + } else { + throw new ArrayIndexOutOfBoundsException(idx); + } + } public void setChild(LogicalNode subNode) { this.child = subNode; diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index 1107fc45c4..a85fcbd1da 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -70,6 +70,12 @@ message LogicalNode { required SchemaProto out_schema = 5; optional ScanNode scan = 6; + optional JoinNode join = 7; + optional FilterNode filter = 8; + optional GroupbyNode groupby = 9; + optional SortNode sort = 10; + optional LimitNode limit = 11; + optional ProjectionNode projection = 12; } message ScanNode { @@ -79,11 +85,57 @@ message ScanNode { optional EvalTree qual = 5; } +message FilterNode { + required int32 childId = 1; + required EvalTree qual = 2; +} + +message JoinNode { + required int32 leftChildId = 1; + required int32 rightChildId = 2; + required JoinType joinType = 3; + optional EvalTree joinQual = 4; + repeated Target targets = 5; +} + +message LimitNode { + required int32 childId = 1; + required int64 fetchFirstNum = 2; +} + +message GroupbyNode { + required int32 childId = 1; + repeated ColumnProto groupingKeys = 2; + repeated EvalTree aggFunctions = 3; +} + +message SortNode { + required int32 childId = 1; + repeated SortSpecProto sortSpecs = 2; +} + +message ProjectionNode { + repeated Target targets = 1; +} + message Target { required EvalTree expr = 1; required string alias = 2; } +enum JoinType { + CROSS_JOIN = 0; + INNER_JOIN = 1; + LEFT_OUTER_JOIN = 2; + RIGHT_OUTER_JOIN = 3; + FULL_OUTER_JOIN = 4; + UNION_JOIN = 5; + LEFT_ANTI_JOIN = 6; + RIGHT_ANTI_JOIN = 7; + LEFT_SEMI_JOIN = 8; + RIGHT_SEMI_JOIN = 9; +} + enum EvalType { NOT = 0; AND = 1; From 5a69b8512eefd5f73f02360976a7cfb649af7a3e Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 18 Aug 2014 02:10:03 +0900 Subject: [PATCH 16/31] Add (de)serialization root, having, and exprs. --- .../plan/LogicalNodeTreeDeserializer.java | 261 ++++++++++++++++-- .../plan/LogicalNodeTreeSerializer.java | 157 +++++++++-- .../planner/BasicLogicalPlanVisitor.java | 9 +- .../engine/planner/LogicalPlanVisitor.java | 3 + .../tajo/engine/planner/LogicalPlanner.java | 4 +- .../engine/planner/PhysicalPlannerImpl.java | 3 +- .../tajo/engine/planner/PlannerUtil.java | 9 - .../engine/planner/logical/EvalExprNode.java | 2 + .../planner/logical/ProjectionNode.java | 12 +- tajo-core/src/main/proto/Plan.proto | 18 +- 10 files changed, 410 insertions(+), 68 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java index 49a5998903..453319dbee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java @@ -20,15 +20,17 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.derby.impl.store.access.sort.Scan; +import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.engine.eval.AggregationFunctionCallEval; import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalType; import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.engine.planner.logical.*; import java.util.*; @@ -39,7 +41,7 @@ public class LogicalNodeTreeDeserializer { instance = new LogicalNodeTreeDeserializer(); } - public static LogicalNode deserilize(PlanProto.LogicalNodeTree tree) { + public static LogicalNode deserialize(PlanProto.LogicalNodeTree tree) { Map nodeMap = Maps.newHashMap(); // sort serialized logical nodes in an ascending order of their sids @@ -61,40 +63,239 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { while (it.hasNext()) { PlanProto.LogicalNode protoNode = it.next(); - NodeType type = convertType(protoNode.getType()); - - switch (type) { - + switch (protoNode.getType()) { + case ROOT: + current = convertRoot(nodeMap, protoNode); + break; + case EXPRS: + current = convertEvalExpr(nodeMap, protoNode); + break; + case PROJECTION: + current = convertProjection(nodeMap, protoNode); + break; + case LIMIT: + current = convertLimit(nodeMap, protoNode); + break; + case SORT: + current = convertSort(nodeMap, protoNode); + break; + case HAVING: + current = convertHaving(nodeMap, protoNode); + break; + case GROUP_BY: + current = convertGroupby(nodeMap, protoNode); + break; + case SELECTION: + current = convertFilter(nodeMap, protoNode); + break; + case JOIN: + current = convertJoin(nodeMap, protoNode); + break; case SCAN: - ScanNode scan = new ScanNode(protoNode.getPid()); - - PlanProto.ScanNode scanProto = protoNode.getScan(); - if (scanProto.hasAlias()) { - scan.init(new TableDesc(scanProto.getTable()), scanProto.getAlias()); - } else { - scan.init(new TableDesc(scanProto.getTable())); - } - - if (scanProto.getTargetsCount() > 0) { - Target[] targets = new Target[scanProto.getTargetsCount()]; - for (int i = 0; i < scanProto.getTargetsCount(); i++) { - targets[i] = convertTarget(scanProto.getTargets(i)); - } - } + current = convertScan(protoNode); + break; default: - throw new RuntimeException("Unknown NodeType: " + type.name()); + throw new RuntimeException("Unknown NodeType: " + protoNode.getType().name()); } + + nodeMap.put(protoNode.getPid(), current); } return current; } - public static NodeType convertType(PlanProto.NodeType type) { - return NodeType.valueOf(type.name()); + public static LogicalRootNode convertRoot(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.RootNode rootProto = protoNode.getRoot(); + + LogicalRootNode root = new LogicalRootNode(protoNode.getPid()); + root.setChild(nodeMap.get(rootProto.getChildId())); + + return root; + } + + public static EvalExprNode convertEvalExpr(PlanProto.LogicalNode protoNode) { + PlanProto.EvalExprNode evalExprProto = protoNode.getExprEval(); + + EvalExprNode evalExpr = new EvalExprNode(protoNode.getPid()); + evalExpr.setInSchema(convertSchema(protoNode.getInSchema())); + evalExpr.setTargets(convertTargets(evalExprProto.getTargetsList())); + + return evalExpr; + } + + public static ProjectionNode convertProjection(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.ProjectionNode projectionProto = protoNode.getProjection(); + + ProjectionNode projectionNode = new ProjectionNode(protoNode.getPid()); + projectionNode.init(projectionProto.getDistinct(), convertTargets(projectionProto.getTargetsList())); + projectionNode.setChild(nodeMap.get(projectionProto.getChildId())); + projectionNode.setInSchema(convertSchema(protoNode.getInSchema())); + projectionNode.setTargets(convertTargets(projectionProto.getTargetsList())); + + return projectionNode; + } + + public static LimitNode convertLimit(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.LimitNode limitProto = protoNode.getLimit(); + + LimitNode limitNode = new LimitNode(protoNode.getPid()); + limitNode.setChild(nodeMap.get(limitProto.getChildId())); + limitNode.setInSchema(convertSchema(protoNode.getInSchema())); + limitNode.setOutSchema(convertSchema(protoNode.getOutSchema())); + limitNode.setFetchFirst(limitProto.getFetchFirstNum()); + + return limitNode; + } + + public static SortNode convertSort(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.SortNode sortProto = protoNode.getSort(); + + SortNode sortNode = new SortNode(protoNode.getPid()); + sortNode.setChild(nodeMap.get(sortProto.getChildId())); + sortNode.setInSchema(convertSchema(protoNode.getInSchema())); + sortNode.setOutSchema(convertSchema(protoNode.getOutSchema())); + sortNode.setSortSpecs(convertSortSpecs(sortProto.getSortSpecsList())); + + return sortNode; + } + + public static HavingNode convertHaving(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.FilterNode havingProto = protoNode.getFilter(); + + HavingNode having = new HavingNode(protoNode.getPid()); + having.setChild(nodeMap.get(havingProto.getChildId())); + having.setQual(EvalTreeProtoDeserializer.deserialize(havingProto.getQual()); + + return having; + } + + public static GroupbyNode convertGroupby(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.GroupbyNode groupbyProto = protoNode.getGroupby(); + + GroupbyNode groupby = new GroupbyNode(protoNode.getPid()); + groupby.setChild(nodeMap.get(groupbyProto.getChildId())); + groupby.setInSchema(convertSchema(protoNode.getInSchema())); + groupby.setOutSchema(convertSchema(protoNode.getOutSchema())); + if (groupbyProto.getAggFunctionsCount() > 0) { + groupby.setGroupingColumns(convertColumns(groupbyProto.getGroupingKeysList())); + } + if (groupbyProto.getAggFunctionsCount() > 0) { + groupby.setAggFunctions((AggregationFunctionCallEval[])convertEvalNodes(groupbyProto.getAggFunctionsList())); + } + return groupby; + } + + public static JoinNode convertJoin(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.JoinNode joinProto = protoNode.getJoin(); + + JoinNode join = new JoinNode(protoNode.getPid()); + join.setLeftChild(nodeMap.get(joinProto.getLeftChildId())); + join.setRightChild(nodeMap.get(joinProto.getRightChildId())); + join.setJoinType(convertJoinType(joinProto.getJoinType())); + join.setInSchema(convertSchema(protoNode.getInSchema())); + join.setOutSchema(convertSchema(protoNode.getOutSchema())); + if (joinProto.hasJoinQual()) { + join.setJoinQual(EvalTreeProtoDeserializer.deserialize(joinProto.getJoinQual())); + } + if (joinProto.getTargetsCount() > 0) { + join.setTargets(convertTargets(joinProto.getTargetsList())); + } + + return join; + } + + public static SelectionNode convertFilter(Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.FilterNode filterProto = protoNode.getFilter(); + + SelectionNode selection = new SelectionNode(protoNode.getPid()); + selection.setInSchema(convertSchema(protoNode.getInSchema())); + selection.setOutSchema(convertSchema(protoNode.getOutSchema())); + selection.setChild(nodeMap.get(filterProto.getChildId())); + selection.setQual(EvalTreeProtoDeserializer.deserialize(filterProto.getQual())); + + return selection; + } + + public static ScanNode convertScan(PlanProto.LogicalNode protoNode) { + ScanNode scan = new ScanNode(protoNode.getPid()); + + PlanProto.ScanNode scanProto = protoNode.getScan(); + if (scanProto.hasAlias()) { + scan.init(new TableDesc(scanProto.getTable()), scanProto.getAlias()); + } else { + scan.init(new TableDesc(scanProto.getTable())); + } + + if (scanProto.getTargetsCount() > 0) { + scan.setTargets(convertTargets(scanProto.getTargetsList())); + } + + return scan; } - public static Target convertTarget(PlanProto.Target targetProto) { - return new Target(EvalTreeProtoDeserializer.deserialize(targetProto.getExpr()), targetProto.getAlias()); + public static Schema convertSchema(CatalogProtos.SchemaProto proto) { + return new Schema(proto); + } + + public static T [] convertEvalNodes(List evalTrees) { + EvalNode [] evalNodes = new EvalNode[evalTrees.size()]; + for (int i = 0; i < evalNodes.length; i++) { + evalNodes[i] = EvalTreeProtoDeserializer.deserialize(evalTrees.get(i)); + } + return (T[]) evalNodes; + } + + public static Column[] convertColumns(List columnProtos) { + Column [] columns = new Column[columnProtos.size()]; + for (int i = 0; i < columns.length; i++) { + columns[i] = new Column(columnProtos.get(i)); + } + return columns; + } + + public static Target [] convertTargets(List targetsProto) { + Target [] targets = new Target[targetsProto.size()]; + for (int i = 0; i < targets.length; i++) { + PlanProto.Target targetProto = targetsProto.get(i); + targets[i] = new Target(EvalTreeProtoDeserializer.deserialize(targetProto.getExpr()), targetProto.getAlias()); + } + return targets; + } + + public static SortSpec[] convertSortSpecs(List sortSpecProtos) { + SortSpec[] sortSpecs = new SortSpec[sortSpecProtos.size()]; + int i = 0; + for (CatalogProtos.SortSpecProto proto : sortSpecProtos) { + sortSpecs[i++] = new SortSpec(proto); + } + return sortSpecs; + } + + public static JoinType convertJoinType(PlanProto.JoinType type) { + switch (type) { + case CROSS_JOIN: + return JoinType.CROSS; + case INNER_JOIN: + return JoinType.INNER; + case LEFT_OUTER_JOIN: + return JoinType.LEFT_OUTER; + case RIGHT_OUTER_JOIN: + return JoinType.RIGHT_OUTER; + case FULL_OUTER_JOIN: + return JoinType.FULL_OUTER; + case LEFT_SEMI_JOIN: + return JoinType.LEFT_SEMI; + case RIGHT_SEMI_JOIN: + return JoinType.RIGHT_SEMI; + case LEFT_ANTI_JOIN: + return JoinType.LEFT_ANTI; + case RIGHT_ANTI_JOIN: + return JoinType.RIGHT_ANTI; + case UNION_JOIN: + return JoinType.UNION; + default: + throw new RuntimeException("Unknown JoinType: " + type.name()); + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java index 85ca3bc81f..93693ba291 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java @@ -18,8 +18,9 @@ package org.apache.tajo.engine.plan; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.algebra.JoinType; import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; import org.apache.tajo.engine.planner.LogicalPlan; @@ -27,6 +28,7 @@ import org.apache.tajo.engine.planner.Target; import org.apache.tajo.engine.planner.logical.*; +import java.util.List; import java.util.Map; import java.util.Stack; @@ -39,14 +41,12 @@ public class LogicalNodeTreeSerializer extends BasicLogicalPlanVisitor()); - - return null; + public static PlanProto.LogicalNodeTree serialize(LogicalNode node) throws PlanningException { + SerializeContext context = new SerializeContext(); + instance.visit(context, null, null, node, new Stack()); + return context.treeBuilder.build(); } - - public static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext context, LogicalNode node) { int selfId; if (context.idMap.containsKey(node)) { @@ -71,6 +71,51 @@ public static class SerializeContext { private PlanProto.LogicalNodeTree.Builder treeBuilder = PlanProto.LogicalNodeTree.newBuilder(); } + public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + LogicalRootNode root, Stack stack) throws PlanningException { + super.visitRoot(context, plan, block, root, stack); + + int [] childIds = registerGetChildIds(context, root); + + PlanProto.RootNode.Builder rootBuilder = PlanProto.RootNode.newBuilder(); + rootBuilder.setChildId(childIds[0]); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, root); + nodeBuilder.setRoot(rootBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return root; + } + + public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + EvalExprNode exprEval, Stack stack) throws PlanningException { + PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder(); + exprEvalBuilder.addAllTargets(convertTargets(exprEval.getTargets())); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, exprEval); + nodeBuilder.setExprEval(exprEvalBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return exprEval; + } + + public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + ProjectionNode projection, Stack stack) throws PlanningException { + super.visitProjection(context, plan, block, projection, stack); + + int [] childIds = registerGetChildIds(context, projection); + + PlanProto.ProjectionNode.Builder projectionBuilder = PlanProto.ProjectionNode.newBuilder(); + projectionBuilder.setChildId(childIds[0]); + projectionBuilder.addAllTargets(convertTargets(projection.getTargets())); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, projection); + nodeBuilder.setProjection(projectionBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return projection; + } + @Override public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, LimitNode limit, Stack stack) throws PlanningException { @@ -78,7 +123,6 @@ public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, Logica int [] childIds = registerGetChildIds(context, limit); - // building itself PlanProto.LimitNode.Builder limitBuilder = PlanProto.LimitNode.newBuilder(); limitBuilder.setChildId(childIds[0]); limitBuilder.setFetchFirstNum(limit.getFetchFirstNum()); @@ -97,7 +141,6 @@ public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, Logical int [] childIds = registerGetChildIds(context, sort); - // building itself PlanProto.SortNode.Builder sortBuilder = PlanProto.SortNode.newBuilder(); sortBuilder.setChildId(childIds[0]); for (int i = 0; i < sort.getSortKeys().length; i++) { @@ -111,6 +154,24 @@ public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, Logical return sort; } + @Override + public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + HavingNode having, Stack stack) throws PlanningException { + super.visitHaving(context, plan, block, having, stack); + + int [] childIds = registerGetChildIds(context, having); + + PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); + filterBuilder.setChildId(childIds[0]); + filterBuilder.setQual(EvalTreeProtoSerializer.serialize(having.getQual())); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, having); + nodeBuilder.setFilter(filterBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return having; + } + public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, GroupbyNode groupbyNode, Stack stack) throws PlanningException { super.visitGroupBy(context, plan, block, groupbyNode, new Stack()); @@ -141,7 +202,6 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic int [] childIds = registerGetChildIds(context, filter); - // building itself PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); filterBuilder.setChildId(childIds[0]); filterBuilder.setQual(EvalTreeProtoSerializer.serialize(filter.getQual())); @@ -153,11 +213,35 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic return filter; } + public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode join, + Stack stack) throws PlanningException { + super.visitJoin(context, plan, block, join, stack); + + int [] childIds = registerGetChildIds(context, join); + + // building itself + PlanProto.JoinNode.Builder joinBuilder = PlanProto.JoinNode.newBuilder(); + joinBuilder.setJoinType(convertJoinType(join.getJoinType())); + joinBuilder.setLeftChildId(childIds[0]); + joinBuilder.setRightChildId(childIds[1]); + if (join.hasJoinQual()) { + joinBuilder.setJoinQual(EvalTreeProtoSerializer.serialize(join.getJoinQual())); + } + if (join.hasTargets()) { + joinBuilder.addAllTargets(convertTargets(join.getTargets())); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, join); + nodeBuilder.setJoin(joinBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return join; + } + @Override public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scan, Stack stack) throws PlanningException { - // building itself PlanProto.ScanNode.Builder scanBuilder = PlanProto.ScanNode.newBuilder(); scanBuilder.setTable(scan.getTableDesc().getProto()); if (scan.hasAlias()) { @@ -165,16 +249,7 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical } if (scan.hasTargets()) { - for (Target t : scan.getTargets()) { - PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); - - targetBuilder.setExpr(EvalTreeProtoSerializer.serialize(t.getEvalTree())); - - if (t.hasAlias()) { - targetBuilder.setAlias(t.getAlias()); - } - scanBuilder.addTargets(targetBuilder); - } + scanBuilder.addAllTargets(convertTargets(scan.getTargets())); } if (scan.hasQual()) { @@ -192,6 +267,46 @@ public static PlanProto.NodeType convertType(NodeType type) { return PlanProto.NodeType.valueOf(type.name()); } + public static PlanProto.JoinType convertJoinType(JoinType type) { + switch (type) { + case CROSS: + return PlanProto.JoinType.CROSS_JOIN; + case INNER: + return PlanProto.JoinType.INNER_JOIN; + case LEFT_OUTER: + return PlanProto.JoinType.LEFT_OUTER_JOIN; + case RIGHT_OUTER: + return PlanProto.JoinType.RIGHT_OUTER_JOIN; + case FULL_OUTER: + return PlanProto.JoinType.FULL_OUTER_JOIN; + case LEFT_SEMI: + return PlanProto.JoinType.LEFT_SEMI_JOIN; + case RIGHT_SEMI: + return PlanProto.JoinType.RIGHT_SEMI_JOIN; + case LEFT_ANTI: + return PlanProto.JoinType.LEFT_ANTI_JOIN; + case RIGHT_ANTI: + return PlanProto.JoinType.RIGHT_ANTI_JOIN; + case UNION: + return PlanProto.JoinType.UNION_JOIN; + default: + throw new RuntimeException("Unknown JoinType: " + type.name()); + } + } + + private Iterable convertTargets(Target [] targets) { + List converted = Lists.newArrayList(); + for (int i = 0; i < targets.length; i++) { + PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); + targetBuilder.setExpr(EvalTreeProtoSerializer.serialize(targets[i].getEvalTree())); + if (targets[i].hasAlias()) { + targetBuilder.setAlias(targets[i].getAlias()); + } + converted.add(targetBuilder.build()); + } + return converted; + } + private int [] registerGetChildIds(SerializeContext context, LogicalNode node) { int [] childIds = new int[node.childNum()]; for (int i = 0; i < node.childNum(); i++) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index a4e90b432b..1f6a541987 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -58,7 +58,8 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl current = visitRoot(context, plan, block, (LogicalRootNode) node, stack); break; case EXPRS: - return null; + current = visitEvalExpr(context, plan, block, (EvalExprNode) node, stack); + break; case PROJECTION: current = visitProjection(context, plan, block, (ProjectionNode) node, stack); break; @@ -147,6 +148,12 @@ public RESULT visitRoot(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBloc return result; } + @Override + public RESULT visitEvalExpr(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, EvalExprNode node, + Stack stack) throws PlanningException { + return null; + } + @Override public RESULT visitProjection(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode node, Stack stack) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java index 0a36610476..9682b37405 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java @@ -26,6 +26,9 @@ public interface LogicalPlanVisitor { RESULT visitRoot(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, LogicalRootNode node, Stack stack) throws PlanningException; + RESULT visitEvalExpr(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, EvalExprNode node, + Stack stack) throws PlanningException; + RESULT visitProjection(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode node, Stack stack) throws PlanningException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 901d9b4a9e..d5db22947a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -235,9 +235,9 @@ public LogicalNode visitProjection(PlanContext context, Stack stack, Proje // Set ProjectionNode projectionNode = context.queryBlock.getNodeFromExpr(projection); - projectionNode.setInSchema(child.getOutSchema()); - projectionNode.setTargets(targets); + projectionNode.init(projection.isDistinct(), targets); projectionNode.setChild(child); + projectionNode.setInSchema(child.getOutSchema()); if (projection.isDistinct() && block.hasNode(NodeType.GROUP_BY)) { throw new VerifyException("Cannot support grouping and distinct at the same time yet"); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 9f533e272e..f5920d6e64 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -34,6 +34,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.engine.plan.LogicalNodeTreeDeserializer; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.logical.*; @@ -868,7 +869,7 @@ private boolean checkIfSortEquivalance(TaskAttemptContext ctx, ScanNode scanNode TajoWorkerProtocol.SortedInputEnforce sortEnforcer = property.get(0).getSortedInput(); boolean condition = scanNode.getTableName().equals(sortEnforcer.getTableName()); - SortSpec [] sortSpecs = PlannerUtil.convertSortSpecs(sortEnforcer.getSortSpecsList()); + SortSpec [] sortSpecs = LogicalNodeTreeDeserializer.convertSortSpecs(sortEnforcer.getSortSpecsList()); return condition && TUtil.checkEquals(sortNode.getSortKeys(), sortSpecs); } else { return false; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java index 3390758b62..5e8c7edbd9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java @@ -731,15 +731,6 @@ public static Collection toQualifiedFieldNames(Collection fieldN return names; } - public static SortSpec[] convertSortSpecs(Collection sortSpecProtos) { - SortSpec[] sortSpecs = new SortSpec[sortSpecProtos.size()]; - int i = 0; - for (CatalogProtos.SortSpecProto proto : sortSpecProtos) { - sortSpecs[i++] = new SortSpec(proto); - } - return sortSpecs; - } - /** * Generate an explain string of a LogicalNode and its descendant nodes. * diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java index 04bfa6acb4..1a5debf260 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java @@ -23,6 +23,7 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Target; import org.apache.tajo.util.TUtil; @@ -51,6 +52,7 @@ public boolean hasTargets() { @Override public void setTargets(Target[] targets) { this.exprs = targets; + this.setOutSchema(PlannerUtil.targetToSchema(targets)); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java index e9fd803c1a..cdc929f61f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java @@ -25,16 +25,26 @@ import org.apache.tajo.util.TUtil; public class ProjectionNode extends UnaryNode implements Projectable { + + @Expose private boolean distinct = false; /** * the targets are always filled even if the query is 'select *' */ @Expose private Target [] targets; - @Expose private boolean distinct = false; public ProjectionNode(int pid) { super(pid, NodeType.PROJECTION); } + public void init(boolean distinct, Target [] targets) { + this.distinct = distinct; + this.targets = targets; + } + + public boolean isDistinct() { + return distinct; + } + public boolean hasTargets() { return this.targets != null; } diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index a85fcbd1da..97cc7537dc 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -73,9 +73,11 @@ message LogicalNode { optional JoinNode join = 7; optional FilterNode filter = 8; optional GroupbyNode groupby = 9; - optional SortNode sort = 10; - optional LimitNode limit = 11; - optional ProjectionNode projection = 12; + optional SortNode sort = 11; + optional LimitNode limit = 12; + optional ProjectionNode projection = 13; + optional EvalExprNode exprEval = 14; + optional RootNode root = 15; } message ScanNode { @@ -115,9 +117,19 @@ message SortNode { } message ProjectionNode { + required int32 childId = 1; + required bool distinct = 2; + repeated Target targets = 3; +} + +message EvalExprNode { repeated Target targets = 1; } +message RootNode { + required int32 childId = 1; +} + message Target { required EvalTree expr = 1; required string alias = 2; From eb50db0b6e73dde90945c43273e59cc1fe472bab Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 18 Aug 2014 02:10:51 +0900 Subject: [PATCH 17/31] Fixed compilation error. --- .../tajo/engine/plan/LogicalNodeTreeDeserializer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java index 453319dbee..8e327ec921 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeDeserializer.java @@ -68,7 +68,7 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { current = convertRoot(nodeMap, protoNode); break; case EXPRS: - current = convertEvalExpr(nodeMap, protoNode); + current = convertEvalExpr(protoNode); break; case PROJECTION: current = convertProjection(nodeMap, protoNode); @@ -165,7 +165,9 @@ public static HavingNode convertHaving(Map nodeMap, PlanPr HavingNode having = new HavingNode(protoNode.getPid()); having.setChild(nodeMap.get(havingProto.getChildId())); - having.setQual(EvalTreeProtoDeserializer.deserialize(havingProto.getQual()); + having.setQual(EvalTreeProtoDeserializer.deserialize(havingProto.getQual())); + having.setInSchema(convertSchema(protoNode.getInSchema())); + having.setOutSchema(convertSchema(protoNode.getOutSchema())); return having; } From b1eb0e9482bdd5e41a698f96738e1fe5af0b1a61 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 18 Aug 2014 11:10:18 +0900 Subject: [PATCH 18/31] Add (de)serialization for other nodes. --- .../tajo/engine/planner/logical/NodeType.java | 4 +- tajo-core/src/main/proto/Plan.proto | 101 ++++++++++++++++-- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java index fa1199bc69..d98c768b59 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java @@ -31,11 +31,12 @@ public enum NodeType { ROOT(LogicalRootNode.class), EXPRS(EvalExprNode.class), PROJECTION(ProjectionNode.class), + WINDOW_AGG(WindowAggNode.class), LIMIT(LimitNode.class), SORT(SortNode.class), HAVING(HavingNode.class), + DISTINCT_GROUP_BY(DistinctGroupbyNode.class), GROUP_BY(GroupbyNode.class), - WINDOW_AGG(WindowAggNode.class), SELECTION(SelectionNode.class), JOIN(JoinNode.class), UNION(UnionNode.class), @@ -47,7 +48,6 @@ public enum NodeType { BST_INDEX_SCAN(IndexScanNode.class), STORE(StoreTableNode.class), INSERT(InsertNode.class), - DISTINCT_GROUP_BY(DistinctGroupbyNode.class), CREATE_DATABASE(CreateDatabaseNode.class), DROP_DATABASE(DropDatabaseNode.class), diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index 97cc7537dc..41dad2a98f 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -75,9 +75,20 @@ message LogicalNode { optional GroupbyNode groupby = 9; optional SortNode sort = 11; optional LimitNode limit = 12; - optional ProjectionNode projection = 13; - optional EvalExprNode exprEval = 14; - optional RootNode root = 15; + optional WindowAggNode window = 13; + optional ProjectionNode projection = 14; + optional EvalExprNode exprEval = 15; + optional SetNode set = 16; + optional TableSubQueryNode tableSubQuery = 17; + optional PersistentStoreNode persistentStore = 18; + optional StoreTableNodeSpec storeTable = 19; + optional InsertNodeSpec insert = 20; + optional CreateTableNodeSpec createTable = 21; + optional RootNode root = 22; + + optional CreateDatabaseNode createDatabase = 23; + optional DropDatabaseNode dropDatabase = 24; + optional DropTableNode dropTable = 25; } message ScanNode { @@ -100,22 +111,73 @@ message JoinNode { repeated Target targets = 5; } -message LimitNode { - required int32 childId = 1; - required int64 fetchFirstNum = 2; -} - message GroupbyNode { required int32 childId = 1; repeated ColumnProto groupingKeys = 2; repeated EvalTree aggFunctions = 3; } +message DistinctGroupbyNode { + required int32 childId = 1; + repeated GroupbyNode groupbyNodes = 2; + repeated Target targets = 3; + repeated ColumnProto groupingKeys = 4; + repeated int32 resultId = 5; +} + message SortNode { required int32 childId = 1; repeated SortSpecProto sortSpecs = 2; } +message LimitNode { + required int32 childId = 1; + required int64 fetchFirstNum = 2; +} + +message WindowAggNode { + required int32 childId = 1; + repeated ColumnProto partitionKeys = 2; + repeated SortSpecProto sortSpecs = 3; + repeated EvalTree windowFunctions = 4; + required bool distinct = 5; // if distinct aggregation function is included in window function + repeated Target targets = 6; +} + +message SetNode { + required int32 leftChildId = 1; + required int32 rightChildId = 2; +} + +message TableSubQueryNode { + required int32 childId = 1; + required string tableName = 2; + repeated Target targets = 3; +} + +message PartitionTableScanSpec { + repeated string paths = 1; +} + +message PersistentStoreNode { + required int32 childId = 1; + required StoreType storageType = 2; + required KeyValueSetProto tableProperties = 3; +} + +message StoreTableNodeSpec { // required PersistentStoreSpec + repeated string tableName = 2; + optional PartitionDescProto partitionDesc = 3; +} + +message InsertNodeSpec { // required StoreTableSpec + required bool overwrite = 1; + required SchemaProto tableSchema = 2; + required SchemaProto targetSchema = 3; + required SchemaProto projectedSchema = 4; + required string path = 5; +} + message ProjectionNode { required int32 childId = 1; required bool distinct = 2; @@ -130,6 +192,29 @@ message RootNode { required int32 childId = 1; } +message CreateDatabaseNode { + required string dbName = 1; + required bool ifNotExists = 2; +} + +message DropDatabaseNode { + required string dbName = 1; + required bool ifExists = 2; +} + +message CreateTableNodeSpec { // required PersistentStoreSpec + required SchemaProto schema = 1; + required string path = 2; + required bool external = 3; + required bool ifNotExists = 4; +} + +message DropTableNode { + required string tableName = 1; + required bool ifExists = 2; + required bool purge = 3; +} + message Target { required EvalTree expr = 1; required string alias = 2; From 3fd70b59d769b1263025708a839793df1492db7a Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 18 Aug 2014 18:46:32 +0900 Subject: [PATCH 19/31] Updated NodeType, and added more (de)serialization code. --- .../org/apache/tajo/engine/eval/EvalNode.java | 10 ++- .../plan/LogicalNodeTreeSerializer.java | 75 ++++++++++++++----- .../planner/BasicLogicalPlanVisitor.java | 6 +- .../apache/tajo/engine/planner/Target.java | 10 ++- .../tajo/engine/planner/logical/NodeType.java | 4 +- tajo-core/src/main/proto/Plan.proto | 40 +++++----- 6 files changed, 101 insertions(+), 44 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java index 48ab5167ec..ef986dada2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java @@ -20,9 +20,12 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.engine.json.CoreGsonHelper; +import org.apache.tajo.engine.plan.EvalTreeProtoSerializer; +import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.json.GsonObject; import org.apache.tajo.storage.Tuple; @@ -30,7 +33,7 @@ * An annotated expression which includes actual data domains. * It is also used for evaluation. */ -public abstract class EvalNode implements Cloneable, GsonObject { +public abstract class EvalNode implements Cloneable, GsonObject, ProtoObject { @Expose protected EvalType type; @Expose protected DataType returnType = null; @@ -73,4 +76,9 @@ public Object clone() throws CloneNotSupportedException { evalNode.returnType = returnType; return evalNode; } + + @Override + public PlanProto.EvalTree getProto() { + return EvalTreeProtoSerializer.serialize(this); + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java index 93693ba291..b36c84937a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/plan/LogicalNodeTreeSerializer.java @@ -21,6 +21,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.common.ProtoObject; +import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; import org.apache.tajo.engine.planner.LogicalPlan; @@ -90,7 +93,8 @@ public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, Logical public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, EvalExprNode exprEval, Stack stack) throws PlanningException { PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder(); - exprEvalBuilder.addAllTargets(convertTargets(exprEval.getTargets())); + exprEvalBuilder.addAllTargets( + LogicalNodeTreeSerializer.toProtoObjects(exprEval.getTargets())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, exprEval); nodeBuilder.setExprEval(exprEvalBuilder); @@ -107,7 +111,8 @@ public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, L PlanProto.ProjectionNode.Builder projectionBuilder = PlanProto.ProjectionNode.newBuilder(); projectionBuilder.setChildId(childIds[0]); - projectionBuilder.addAllTargets(convertTargets(projection.getTargets())); + projectionBuilder.addAllTargets( + LogicalNodeTreeSerializer.toProtoObjects(projection.getTargets())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, projection); nodeBuilder.setProjection(projectionBuilder); @@ -134,6 +139,36 @@ public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, Logica return limit; } + public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + WindowAggNode windowAgg, Stack stack) throws PlanningException { + super.visitWindowAgg(context, plan, block, windowAgg, stack); + + int [] childIds = registerGetChildIds(context, windowAgg); + + PlanProto.WindowAggNode.Builder windowAggBuilder = PlanProto.WindowAggNode.newBuilder(); + windowAggBuilder.setChildId(childIds[0]); + if (windowAgg.hasPartitionKeys()) { + windowAggBuilder.addAllPartitionKeys( + LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getPartitionKeys())); + } + if (windowAgg.hasAggFunctions()) { + windowAggBuilder.addAllWindowFunctions( + LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getWindowFunctions())); + } + windowAggBuilder.setDistinct(windowAgg.isDistinct()); + + if (windowAgg.hasSortSpecs()) { + windowAggBuilder.addAllSortSpecs( + LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getSortSpecs())); + } + if (windowAgg.hasTargets()) { + windowAggBuilder.addAllTargets( + LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getTargets())); + } + + return windowAgg; + } + @Override public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SortNode sort, Stack stack) throws PlanningException { @@ -181,11 +216,13 @@ public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, Logi PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); groupbyBuilder.setChildId(childIds[0]); - for (int i = 0; i < groupbyNode.getGroupingColumns().length; i++) { - groupbyBuilder.addGroupingKeys(groupbyNode.getGroupingColumns()[i].getProto()); + if (groupbyNode.groupingKeyNum() > 0) { + groupbyBuilder.addAllGroupingKeys( + LogicalNodeTreeSerializer.toProtoObjects(groupbyNode.getGroupingColumns())); } - for (int i = 0; i < groupbyNode.getGroupingColumns().length; i++) { - groupbyBuilder.addAggFunctions(EvalTreeProtoSerializer.serialize(groupbyNode.getAggFunctions()[i])); + if (groupbyNode.hasAggFunctions()) { + groupbyBuilder.addAllAggFunctions( + LogicalNodeTreeSerializer.toProtoObjects(groupbyNode.getAggFunctions())); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, groupbyNode); @@ -228,7 +265,7 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical joinBuilder.setJoinQual(EvalTreeProtoSerializer.serialize(join.getJoinQual())); } if (join.hasTargets()) { - joinBuilder.addAllTargets(convertTargets(join.getTargets())); + joinBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(join.getTargets())); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, join); @@ -249,7 +286,7 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical } if (scan.hasTargets()) { - scanBuilder.addAllTargets(convertTargets(scan.getTargets())); + scanBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(scan.getTargets())); } if (scan.hasQual()) { @@ -294,15 +331,19 @@ public static PlanProto.JoinType convertJoinType(JoinType type) { } } - private Iterable convertTargets(Target [] targets) { - List converted = Lists.newArrayList(); - for (int i = 0; i < targets.length; i++) { - PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); - targetBuilder.setExpr(EvalTreeProtoSerializer.serialize(targets[i].getEvalTree())); - if (targets[i].hasAlias()) { - targetBuilder.setAlias(targets[i].getAlias()); - } - converted.add(targetBuilder.build()); + public static PlanProto.Target convertTarget(Target target) { + PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); + targetBuilder.setExpr(EvalTreeProtoSerializer.serialize(target.getEvalTree())); + if (target.hasAlias()) { + targetBuilder.setAlias(target.getAlias()); + } + return targetBuilder.build(); + } + + public static Iterable toProtoObjects(ProtoObject[] protoObjects) { + List converted = Lists.newArrayList(); + for (int i = 0; i < protoObjects.length; i++) { + converted.add((T) protoObjects[i].getProto()); } return converted; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index 1f6a541987..296c8aa389 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -66,6 +66,9 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case LIMIT: current = visitLimit(context, plan, block, (LimitNode) node, stack); break; + case WINDOW_AGG: + current = visitWindowAgg(context, plan, block, (WindowAggNode) node, stack); + break; case SORT: current = visitSort(context, plan, block, (SortNode) node, stack); break; @@ -75,9 +78,6 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case GROUP_BY: current = visitGroupBy(context, plan, block, (GroupbyNode) node, stack); break; - case WINDOW_AGG: - current = visitWindowAgg(context, plan, block, (WindowAggNode) node, stack); - break; case DISTINCT_GROUP_BY: current = visitDistinct(context, plan, block, (DistinctGroupbyNode) node, stack); break; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java index 6a16d3c0a6..7530ea2250 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java @@ -20,17 +20,20 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.eval.FieldEval; import org.apache.tajo.engine.json.CoreGsonHelper; +import org.apache.tajo.engine.plan.LogicalNodeTreeSerializer; +import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.json.GsonObject; import org.apache.tajo.util.TUtil; /** * A Target contains how to evaluate an expression and its alias name. */ -public class Target implements Cloneable, GsonObject { +public class Target implements Cloneable, GsonObject, ProtoObject { @Expose private EvalNode expr; @Expose private Column column; @Expose private String alias = null; @@ -126,4 +129,9 @@ public Object clone() throws CloneNotSupportedException { public String toJson() { return CoreGsonHelper.toJson(this, Target.class); } + + @Override + public PlanProto.Target getProto() { + return LogicalNodeTreeSerializer.convertTarget(this); + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java index d98c768b59..2be3180c88 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java @@ -31,8 +31,8 @@ public enum NodeType { ROOT(LogicalRootNode.class), EXPRS(EvalExprNode.class), PROJECTION(ProjectionNode.class), - WINDOW_AGG(WindowAggNode.class), LIMIT(LimitNode.class), + WINDOW_AGG(WindowAggNode.class), SORT(SortNode.class), HAVING(HavingNode.class), DISTINCT_GROUP_BY(DistinctGroupbyNode.class), @@ -40,8 +40,8 @@ public enum NodeType { SELECTION(SelectionNode.class), JOIN(JoinNode.class), UNION(UnionNode.class), - EXCEPT(ExceptNode.class), INTERSECT(IntersectNode.class), + EXCEPT(ExceptNode.class), TABLE_SUBQUERY(TableSubQueryNode.class), SCAN(ScanNode.class), PARTITIONS_SCAN(PartitionedTableScanNode.class), diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-core/src/main/proto/Plan.proto index 41dad2a98f..506d140121 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-core/src/main/proto/Plan.proto @@ -28,26 +28,26 @@ import "CatalogProtos.proto"; import "DataTypes.proto"; enum NodeType { - BST_INDEX_SCAN = 0; - EXCEPT = 1; - EXPRS = 2; - DISTINCT_GROUP_BY = 3; - GROUP_BY = 4; - HAVING = 5; - JOIN = 6; - INSERT = 7; - INTERSECT = 8; - LIMIT = 9; - PARTITIONS_SCAN = 10; - PROJECTION = 11; - ROOT = 12; - SCAN = 13; - SELECTION = 14; - SORT = 15; - STORE = 16; - TABLE_SUBQUERY = 17; - UNION = 18; - WINDOW_AGG = 19; + ROOT = 0; + EXPRS = 1; + PROJECTION = 2; + LIMIT = 3; + WINDOW_AGG = 4; + SORT = 5; + HAVING = 6; + GROUP_BY = 7; + DISTINCT_GROUP_BY = 8; + SELECTION = 9; + JOIN = 10; + UNION = 11; + INTERSECT = 12; + EXCEPT = 13; + TABLE_SUBQUERY = 14; + SCAN = 15; + PARTITIONS_SCAN = 16; + BST_INDEX_SCAN = 17; + STORE = 18; + INSERT = 19; CREATE_DATABASE = 20; DROP_DATABASE = 21; From 8a69a46cfc2825704f6bd689e80aac25a543d99c Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 26 Dec 2014 05:30:29 +0900 Subject: [PATCH 20/31] All unit tests are passed. --- .../builder/DistinctGroupbyBuilder.java | 2 +- .../tajo/master/exec/QueryExecutor.java | 6 +- .../tajo/engine/query/TestTruncateTable.java | 8 +- .../org/apache/tajo/plan/LogicalPlanner.java | 10 +- .../tajo/plan/logical/CreateTableNode.java | 11 +- .../apache/tajo/plan/logical/GroupbyNode.java | 6 +- .../apache/tajo/plan/logical/InsertNode.java | 15 +- .../tajo/plan/logical/SetSessionNode.java | 11 +- .../tajo/plan/logical/StoreTableNode.java | 11 +- .../serder/LogicalNodeTreeDeserializer.java | 257 ++++++++++++++++- .../serder/LogicalNodeTreeSerializer.java | 270 +++++++++++++++++- .../plan/visitor/BasicLogicalPlanVisitor.java | 24 +- tajo-plan/src/main/proto/Plan.proto | 206 ++++++++----- 13 files changed, 721 insertions(+), 116 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 671bb1920f..4c5e8b1fa6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -534,7 +534,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou if (firstStageGroupbyNode.isDistinct()) { // FirstStage: Remove aggregation, Set target with only grouping columns - firstStageGroupbyNode.setAggFunctions(null); + firstStageGroupbyNode.setAggFunctions(GroupbyNode.EMPTY_AGG_FUNCS); List firstGroupbyTargets = new ArrayList(); for (Column column : firstStageGroupbyNode.getGroupingColumns()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 3585ae7430..10701f9dcc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -144,10 +144,10 @@ public void execSetSession(Session session, LogicalPlan plan, // others } else { - if (setSessionNode.isDefaultValue()) { - session.removeVariable(varName); - } else { + if (setSessionNode.hasValue()) { session.setVariable(varName, setSessionNode.getValue()); + } else { + session.removeVariable(varName); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTruncateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTruncateTable.java index 455213b475..1be21e4486 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTruncateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTruncateTable.java @@ -25,8 +25,10 @@ import org.junit.experimental.categories.Category; import java.sql.ResultSet; +import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; @Category(IntegrationTest.class) public class TestTruncateTable extends QueryTestCaseBase { @@ -63,11 +65,6 @@ public final void testTruncateTable() throws Exception { } } - - /* - Currently TajoClient can't throw exception when plan error. - The following test cast should be uncommented after https://issues.apache.org/jira/browse/TAJO-762 - @Test public final void testTruncateExternalTable() throws Exception { try { @@ -100,5 +97,4 @@ public final void testTruncateExternalTable() throws Exception { executeString("DROP TABLE truncate_table2 PURGE"); } } - */ } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 7f2dc2786f..55ca7651aa 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -46,6 +46,9 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; +import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.catalog.SchemaUtil; @@ -144,11 +147,16 @@ public LogicalPlan createPlan(OverridableConf queryContext, Expr expr, boolean d // Add Root Node LogicalRootNode root = plan.createNode(LogicalRootNode.class); + root.setInSchema(topMostNode.getOutSchema()); root.setChild(topMostNode); root.setOutSchema(topMostNode.getOutSchema()); plan.getRootBlock().setRoot(root); + PlanProto.LogicalNodeTree serialized = LogicalNodeTreeSerializer.serialize(root); + LogicalNode deserialized = LogicalNodeTreeDeserializer.deserialize(queryContext, serialized); + assert root.deepEquals(deserialized); + return plan; } @@ -1569,7 +1577,7 @@ private void buildProjectedInsert(PlanContext context, InsertNode insertNode) { } if (child instanceof Projectable) { - Projectable projectionNode = (Projectable) insertNode.getChild(); + Projectable projectionNode = insertNode.getChild(); // Modifying projected columns by adding NULL constants // It is because that table appender does not support target columns to be written. diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java index d03da6a725..0976ab5663 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java @@ -99,11 +99,12 @@ public int hashCode() { public boolean equals(Object obj) { if (obj instanceof CreateTableNode) { CreateTableNode other = (CreateTableNode) obj; - return super.equals(other) - && this.schema.equals(other.schema) - && this.external == other.external - && TUtil.checkEquals(path, other.path) - && ifNotExists == other.ifNotExists; + boolean eq = super.equals(other); + eq &= this.schema.equals(other.schema); + eq &= this.external == other.external; + eq &= TUtil.checkEquals(path, other.path); + eq &= ifNotExists == other.ifNotExists;; + return eq; } else { return false; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java index 097ae4a700..6a8f41272e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.logical; +import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; import org.apache.tajo.plan.PlanString; @@ -28,7 +29,7 @@ public class GroupbyNode extends UnaryNode implements Projectable, Cloneable { private static final Column [] EMPTY_GROUPING_KEYS = new Column[] {}; - private static final AggregationFunctionCallEval [] EMPTY_AGG_FUNCS = new AggregationFunctionCallEval[] {}; + public static final AggregationFunctionCallEval [] EMPTY_AGG_FUNCS = new AggregationFunctionCallEval[] {}; /** Grouping key sets */ @Expose private Column [] groupingKeys = EMPTY_GROUPING_KEYS; @@ -70,7 +71,7 @@ public void setDistinct(boolean distinct) { } public boolean hasAggFunctions() { - return this.aggrFunctions.length > 0; + return aggrFunctions.length > 0; } public int aggregationFunctionNum() { @@ -82,6 +83,7 @@ public AggregationFunctionCallEval[] getAggFunctions() { } public void setAggFunctions(AggregationFunctionCallEval[] evals) { + Preconditions.checkNotNull(evals); this.aggrFunctions = evals; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java index d1d8582ae9..769cb59f04 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java @@ -95,6 +95,10 @@ public void setTargetSchema(Schema schema) { this.targetSchema = schema; } + public boolean hasProjectedSchema() { + return this.projectedSchema != null; + } + public Schema getProjectedSchema() { return this.projectedSchema; } @@ -123,11 +127,12 @@ public boolean hasStorageType() { public boolean equals(Object obj) { if (obj instanceof InsertNode) { InsertNode other = (InsertNode) obj; - return super.equals(other) - && this.overwrite == other.overwrite - && TUtil.checkEquals(this.tableSchema, other.tableSchema) - && TUtil.checkEquals(this.targetSchema, other.targetSchema) - && TUtil.checkEquals(path, other.path); + boolean eq = super.equals(other); + eq &= this.overwrite == other.overwrite; + eq &= TUtil.checkEquals(this.tableSchema, other.tableSchema); + eq &= TUtil.checkEquals(this.targetSchema, other.targetSchema); + eq &= TUtil.checkEquals(path, other.path); + return eq; } else { return false; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java index 59a8f89463..117315f104 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SetSessionNode.java @@ -30,6 +30,13 @@ public SetSessionNode(int pid) { super(pid, NodeType.SET_SESSION); } + /** + * If both name and value are given, it will set a session variable. + * If a name is only given, it will unset a session variable. + * + * @param name Session variable name + * @param value Session variable value + */ public void init(String name, String value) { this.name = name; this.value = value; @@ -39,8 +46,8 @@ public String getName() { return name; } - public boolean isDefaultValue() { - return value == null; + public boolean hasValue() { + return value != null; } public String getValue() { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java index 0623d212ec..730eb35248 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java @@ -39,6 +39,15 @@ public boolean hasTargetTable() { return tableName != null; } + /** + * Check if a table name is specified. + * + * @return FALSE if this node is used for 'INSERT INTO LOCATION'. Otherwise, it will be TRUE. + */ + public boolean hasTableName() { + return tableName != null; + } + public void setTableName(String tableName) { this.tableName = tableName; } @@ -73,7 +82,7 @@ public boolean equals(Object obj) { if (obj instanceof StoreTableNode) { StoreTableNode other = (StoreTableNode) obj; boolean eq = super.equals(other); - eq = eq && this.tableName.equals(other.tableName); + eq = eq && TUtil.checkEquals(this.tableName, other.tableName); eq = eq && TUtil.checkEquals(partitionDesc, other.partitionDesc); return eq; } else { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java index a14571d536..47cf19cf56 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java @@ -20,17 +20,26 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.sun.tools.javac.util.Name; +import org.apache.hadoop.fs.Path; import org.apache.tajo.OverridableConf; +import org.apache.tajo.algebra.DropTable; +import org.apache.tajo.algebra.Insert; import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.algebra.TruncateTable; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.AggregationFunctionCallEval; import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.FieldEval; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.util.KeyValueSet; import java.util.*; @@ -67,6 +76,9 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { case ROOT: current = convertRoot(nodeMap, protoNode); break; + case SET_SESSION: + current = convertSetSession(protoNode); + break; case EXPRS: current = convertEvalExpr(context, protoNode); break; @@ -91,15 +103,48 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { case JOIN: current = convertJoin(context, nodeMap, protoNode); break; + case TABLE_SUBQUERY: + current = convertTableSubQuery(context, nodeMap, protoNode); + break; + case UNION: + current = convertUnion(nodeMap, protoNode); + break; case SCAN: current = convertScan(context, protoNode); break; + case CREATE_TABLE: + current = convertCreateTable(nodeMap, protoNode); + break; + case INSERT: + current = convertInsert(nodeMap, protoNode); + break; + case DROP_TABLE: + current = convertDropTable(protoNode); + break; + + case CREATE_DATABASE: + current = convertCreateDatabase(protoNode); + break; + case DROP_DATABASE: + current = convertDropDatabase(protoNode); + break; + + case ALTER_TABLESPACE: + current = convertAlterTablespace(protoNode); + break; + case ALTER_TABLE: + current = convertAlterTable(protoNode); + break; + case TRUNCATE_TABLE: + current = convertTruncateTable(protoNode); + break; + default: throw new RuntimeException("Unknown NodeType: " + protoNode.getType().name()); } - nodeMap.put(protoNode.getPid(), current); + nodeMap.put(protoNode.getSid(), current); } return current; @@ -111,10 +156,25 @@ public static LogicalRootNode convertRoot(Map nodeMap, LogicalRootNode root = new LogicalRootNode(protoNode.getPid()); root.setChild(nodeMap.get(rootProto.getChildId())); + if (protoNode.hasInSchema()) { + root.setInSchema(convertSchema(protoNode.getInSchema())); + } + if (protoNode.hasOutSchema()) { + root.setOutSchema(convertSchema(protoNode.getOutSchema())); + } return root; } + public static SetSessionNode convertSetSession(PlanProto.LogicalNode protoNode) { + PlanProto.SetSessionNode setSessionProto = protoNode.getSetSession(); + + SetSessionNode setSession = new SetSessionNode(protoNode.getPid()); + setSession.init(setSessionProto.getName(), setSessionProto.hasValue() ? setSessionProto.getValue() : null); + + return setSession; + } + public static EvalExprNode convertEvalExpr(OverridableConf context, PlanProto.LogicalNode protoNode) { PlanProto.EvalExprNode evalExprProto = protoNode.getExprEval(); @@ -133,7 +193,7 @@ public static ProjectionNode convertProjection(OverridableConf context, Map 0) { - groupby.setAggFunctions((AggregationFunctionCallEval[])convertEvalNodes(context, - groupbyProto.getAggFunctionsList())); + groupby.setAggFunctions(convertAggFuncCallEvals(context, groupbyProto.getAggFunctionsList())); } return groupby; } @@ -226,6 +285,18 @@ public static SelectionNode convertFilter(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { + PlanProto.UnionNode unionProto = protoNode.getUnion(); + + UnionNode union = new UnionNode(protoNode.getPid()); + union.setInSchema(convertSchema(protoNode.getInSchema())); + union.setOutSchema(convertSchema(protoNode.getOutSchema())); + union.setLeftChild(nodeMap.get(unionProto.getLeftChildId())); + union.setRightChild(nodeMap.get(unionProto.getRightChildId())); + + return union; + } + public static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNode protoNode) { ScanNode scan = new ScanNode(protoNode.getPid()); @@ -243,6 +314,167 @@ public static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNod return scan; } + public static TableSubQueryNode convertTableSubQuery(OverridableConf context, + Map nodeMap, + PlanProto.LogicalNode protoNode) { + PlanProto.TableSubQueryNode proto = protoNode.getTableSubQuery(); + + TableSubQueryNode tableSubQuery = new TableSubQueryNode(protoNode.getPid()); + tableSubQuery.setInSchema(convertSchema(protoNode.getInSchema())); + tableSubQuery.setSubQuery(nodeMap.get(proto.getChildId())); + tableSubQuery.setTargets(convertTargets(context, proto.getTargetsList())); + + return tableSubQuery; + } + + public static CreateTableNode convertCreateTable(Map nodeMap, + PlanProto.LogicalNode protoNode) { + PlanProto.PersistentStoreNode persistentStoreProto = protoNode.getPersistentStore(); + PlanProto.StoreTableNodeSpec storeTableNodeSpec = protoNode.getStoreTable(); + PlanProto.CreateTableNodeSpec createTableNodeSpec = protoNode.getCreateTable(); + + CreateTableNode createTable = new CreateTableNode(protoNode.getPid()); + if (protoNode.hasInSchema()) { + createTable.setInSchema(convertSchema(protoNode.getInSchema())); + } + if (protoNode.hasOutSchema()) { + createTable.setOutSchema(convertSchema(protoNode.getOutSchema())); + } + createTable.setChild(nodeMap.get(persistentStoreProto.getChildId())); + createTable.setStorageType(persistentStoreProto.getStorageType()); + createTable.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); + + createTable.setTableName(storeTableNodeSpec.getTableName()); + if (storeTableNodeSpec.hasPartitionMethod()) { + createTable.setPartitionMethod(new PartitionMethodDesc(storeTableNodeSpec.getPartitionMethod())); + } + + createTable.setTableSchema(convertSchema(createTableNodeSpec.getSchema())); + createTable.setExternal(createTableNodeSpec.getExternal()); + if (createTableNodeSpec.getExternal() && createTableNodeSpec.hasPath()) { + createTable.setPath(new Path(createTableNodeSpec.getPath())); + } + createTable.setIfNotExists(createTableNodeSpec.getIfNotExists()); + + return createTable; + } + + public static InsertNode convertInsert(Map nodeMap, + PlanProto.LogicalNode protoNode) { + PlanProto.PersistentStoreNode persistentStoreProto = protoNode.getPersistentStore(); + PlanProto.StoreTableNodeSpec storeTableNodeSpec = protoNode.getStoreTable(); + PlanProto.InsertNodeSpec insertNodeSpec = protoNode.getInsert(); + + InsertNode insertNode = new InsertNode(protoNode.getPid()); + if (protoNode.hasInSchema()) { + insertNode.setInSchema(convertSchema(protoNode.getInSchema())); + } + if (protoNode.hasOutSchema()) { + insertNode.setOutSchema(convertSchema(protoNode.getOutSchema())); + } + insertNode.setChild(nodeMap.get(persistentStoreProto.getChildId())); + insertNode.setStorageType(persistentStoreProto.getStorageType()); + insertNode.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); + + if (storeTableNodeSpec.hasTableName()) { + insertNode.setTableName(storeTableNodeSpec.getTableName()); + } + if (storeTableNodeSpec.hasPartitionMethod()) { + insertNode.setPartitionMethod(new PartitionMethodDesc(storeTableNodeSpec.getPartitionMethod())); + } + + insertNode.setOverwrite(insertNodeSpec.getOverwrite()); + insertNode.setTableSchema(convertSchema(insertNodeSpec.getTableSchema())); + if (insertNodeSpec.hasTargetSchema()) { + insertNode.setTargetSchema(convertSchema(insertNodeSpec.getTargetSchema())); + } + if (insertNodeSpec.hasProjectedSchema()) { + insertNode.setProjectedSchema(convertSchema(insertNodeSpec.getProjectedSchema())); + } + if (insertNodeSpec.hasPath()) { + insertNode.setPath(new Path(insertNodeSpec.getPath())); + } + + return insertNode; + } + + public static DropTableNode convertDropTable(PlanProto.LogicalNode protoNode) { + DropTableNode dropTable = new DropTableNode(protoNode.getPid()); + + PlanProto.DropTableNode dropTableProto = protoNode.getDropTable(); + dropTable.init(dropTableProto.getTableName(), dropTableProto.getIfExists(), dropTableProto.getPurge()); + + return dropTable; + } + + public static CreateDatabaseNode convertCreateDatabase(PlanProto.LogicalNode protoNode) { + CreateDatabaseNode createDatabase = new CreateDatabaseNode(protoNode.getPid()); + + PlanProto.CreateDatabaseNode createDatabaseProto = protoNode.getCreateDatabase(); + createDatabase.init(createDatabaseProto.getDbName(), createDatabaseProto.getIfNotExists()); + + return createDatabase; + } + + public static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoNode) { + DropDatabaseNode dropDatabase = new DropDatabaseNode(protoNode.getPid()); + + PlanProto.DropDatabaseNode dropDatabaseProto = protoNode.getDropDatabase(); + dropDatabase.init(dropDatabaseProto.getDbName(), dropDatabase.isIfExists()); + + return dropDatabase; + } + + public static AlterTablespaceNode convertAlterTablespace(PlanProto.LogicalNode protoNode) { + AlterTablespaceNode alterTablespace = new AlterTablespaceNode(protoNode.getPid()); + + PlanProto.AlterTablespaceNode alterTablespaceProto = protoNode.getAlterTablespace(); + alterTablespace.setTablespaceName(alterTablespaceProto.getTableSpaceName()); + + switch (alterTablespaceProto.getSetType()) { + case LOCATION: + alterTablespace.setLocation(alterTablespaceProto.getSetLocation().getLocation()); + break; + default: + throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTablespaceProto.getSetType().name()); + } + + return alterTablespace; + } + + public static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) { + AlterTableNode alterTable = new AlterTableNode(protoNode.getPid()); + + PlanProto.AlterTableNode alterTableProto = protoNode.getAlterTable(); + alterTable.setTableName(alterTableProto.getTableName()); + + switch (alterTableProto.getSetType()) { + case RENAME_TABLE: + alterTable.setNewTableName(alterTableProto.getRenameTable().getNewName()); + break; + case ADD_COLUMN: + alterTable.setAddNewColumn(new Column(alterTableProto.getAddColumn().getAddColumn())); + break; + case RENAME_COLUMN: + alterTable.setColumnName(alterTableProto.getRenameColumn().getOldName()); + alterTable.setNewColumnName(alterTableProto.getRenameColumn().getNewName()); + break; + default: + throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTableProto.getSetType().name()); + } + + return alterTable; + } + + public static TruncateTableNode convertTruncateTable(PlanProto.LogicalNode protoNode) { + TruncateTableNode truncateTable = new TruncateTableNode(protoNode.getPid()); + + PlanProto.TruncateTableNode truncateTableProto = protoNode.getTruncateTableNode(); + truncateTable.setTableNames(truncateTableProto.getTableNamesList()); + + return truncateTable; + } + public static Schema convertSchema(CatalogProtos.SchemaProto proto) { return new Schema(proto); } @@ -256,6 +488,15 @@ public static Schema convertSchema(CatalogProtos.SchemaProto proto) { return (T[]) evalNodes; } + public static AggregationFunctionCallEval [] convertAggFuncCallEvals(OverridableConf context, + List evalTrees) { + AggregationFunctionCallEval [] aggFuncs = new AggregationFunctionCallEval[evalTrees.size()]; + for (int i = 0; i < aggFuncs.length; i++) { + aggFuncs[i] = (AggregationFunctionCallEval) EvalTreeProtoDeserializer.deserialize(context, evalTrees.get(i)); + } + return aggFuncs; + } + public static Column[] convertColumns(List columnProtos) { Column [] columns = new Column[columnProtos.size()]; for (int i = 0; i < columns.length; i++) { @@ -268,8 +509,12 @@ public static Target[] convertTargets(OverridableConf context, List()); return context.treeBuilder.build(); @@ -61,15 +67,21 @@ public static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext c nodeBuilder.setSid(selfId); nodeBuilder.setPid(node.getPID()); nodeBuilder.setType(convertType(node.getType())); - nodeBuilder.setInSchema(node.getInSchema().getProto()); - nodeBuilder.setOutSchema(node.getOutSchema().getProto()); + + // some DDL statements like DropTable or DropDatabase do not have in/out schemas + if (node.getInSchema() != null) { + nodeBuilder.setInSchema(node.getInSchema().getProto()); + } + if (node.getOutSchema() != null) { + nodeBuilder.setOutSchema(node.getOutSchema().getProto()); + } return nodeBuilder; } public static class SerializeContext { private int seqId = 0; private Map idMap = Maps.newHashMap(); - private PlanProto.LogicalNodeTree.Builder treeBuilder = PlanProto.LogicalNodeTree.newBuilder(); + private LogicalNodeTree.Builder treeBuilder = LogicalNodeTree.newBuilder(); } public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, @@ -88,6 +100,24 @@ public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, Logical return root; } + @Override + public LogicalNode visitSetSession(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + SetSessionNode node, Stack stack) throws PlanningException { + super.visitSetSession(context, plan, block, node, stack); + + PlanProto.SetSessionNode.Builder builder = PlanProto.SetSessionNode.newBuilder(); + builder.setName(node.getName()); + if (node.hasValue()) { + builder.setValue(node.getValue()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setSetSession(builder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, EvalExprNode exprEval, Stack stack) throws PlanningException { PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder(); @@ -111,6 +141,7 @@ public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, L projectionBuilder.setChildId(childIds[0]); projectionBuilder.addAllTargets( LogicalNodeTreeSerializer.toProtoObjects(projection.getTargets())); + projectionBuilder.setDistinct(projection.isDistinct()); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, projection); nodeBuilder.setProjection(projectionBuilder); @@ -273,6 +304,25 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical return join; } + @Override + public LogicalNode visitUnion(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, UnionNode node, + Stack stack) throws PlanningException { + super.visitUnion(context, plan, block, node, stack); + + int [] childIds = registerGetChildIds(context, node); + + PlanProto.UnionNode.Builder unionBuilder = PlanProto.UnionNode.newBuilder(); + unionBuilder.setAll(true); + unionBuilder.setLeftChildId(childIds[0]); + unionBuilder.setRightChildId(childIds[1]); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setUnion(unionBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + @Override public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scan, Stack stack) throws PlanningException { @@ -298,6 +348,218 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical return scan; } + public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + TableSubQueryNode node, Stack stack) throws PlanningException { + super.visitTableSubQuery(context, plan, block, node, stack); + + int [] childIds = registerGetChildIds(context, node); + + PlanProto.TableSubQueryNode.Builder builder = PlanProto.TableSubQueryNode.newBuilder(); + builder.setChildId(childIds[0]); + + builder.setTableName(node.getTableName()); + + if (node.hasTargets()) { + builder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(node.getTargets())); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setTableSubQuery(builder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + public LogicalNode visitCreateTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + CreateTableNode node, Stack stack) throws PlanningException { + super.visitCreateTable(context, plan, block, node, stack); + + int [] childIds = registerGetChildIds(context, node); + + PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds); + PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node); + + PlanProto.CreateTableNodeSpec.Builder createTableBuilder = PlanProto.CreateTableNodeSpec.newBuilder(); + createTableBuilder.setSchema(node.getTableSchema().getProto()); + createTableBuilder.setExternal(node.isExternal()); + if (node.isExternal() && node.hasPath()) { + createTableBuilder.setPath(node.getPath().toString()); + } + createTableBuilder.setIfNotExists(node.isIfNotExists()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setPersistentStore(persistentStoreBuilder); + nodeBuilder.setStoreTable(storeTableBuilder); + nodeBuilder.setCreateTable(createTableBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + @Override + public LogicalNode visitDropTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DropTableNode node, Stack stack) { + PlanProto.DropTableNode.Builder dropTableBuilder = PlanProto.DropTableNode.newBuilder(); + dropTableBuilder.setTableName(node.getTableName()); + dropTableBuilder.setIfExists(node.isIfExists()); + dropTableBuilder.setPurge(node.isPurge()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setDropTable(dropTableBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + @Override + public LogicalNode visitAlterTablespace(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + AlterTablespaceNode node, Stack stack) throws PlanningException { + PlanProto.AlterTablespaceNode.Builder alterTablespaceBuilder = PlanProto.AlterTablespaceNode.newBuilder(); + alterTablespaceBuilder.setTableSpaceName(node.getTablespaceName()); + + switch (node.getSetType()) { + case LOCATION: + alterTablespaceBuilder.setSetType(PlanProto.AlterTablespaceNode.Type.LOCATION); + alterTablespaceBuilder.setSetLocation(SetLocation.newBuilder().setLocation(node.getLocation())); + break; + + default: + throw new UnimplementedException("Unknown SET type in ALTER TABLESPACE: " + node.getSetType().name()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setAlterTablespace(alterTablespaceBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + @Override + public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + AlterTableNode node, Stack stack) { + PlanProto.AlterTableNode.Builder alterTableBuilder = PlanProto.AlterTableNode.newBuilder(); + alterTableBuilder.setTableName(node.getTableName()); + + switch (node.getAlterTableOpType()) { + case RENAME_TABLE: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.RENAME_TABLE); + alterTableBuilder.setRenameTable(RenameTable.newBuilder().setNewName(node.getNewTableName())); + break; + case ADD_COLUMN: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.ADD_COLUMN); + alterTableBuilder.setAddColumn(AddColumn.newBuilder().setAddColumn(node.getAddNewColumn().getProto())); + break; + case RENAME_COLUMN: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.RENAME_COLUMN); + alterTableBuilder.setRenameColumn(RenameColumn.newBuilder() + .setOldName(node.getColumnName()) + .setNewName(node.getNewColumnName())); + break; + default: + throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + node.getAlterTableOpType().name()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setAlterTable(alterTableBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + @Override + public LogicalNode visitTruncateTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + TruncateTableNode node, Stack stack) throws PlanningException { + PlanProto.TruncateTableNode.Builder truncateTableBuilder = PlanProto.TruncateTableNode.newBuilder(); + truncateTableBuilder.addAllTableNames(node.getTableNames()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setTruncateTableNode(truncateTableBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + InsertNode node, Stack stack) throws PlanningException { + super.visitInsert(context, plan, block, node, stack); + + int [] childIds = registerGetChildIds(context, node); + + PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds); + PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node); + + PlanProto.InsertNodeSpec.Builder insertNodeSpec = PlanProto.InsertNodeSpec.newBuilder(); + insertNodeSpec.setOverwrite(node.isOverwrite()); + insertNodeSpec.setTableSchema(node.getTableSchema().getProto()); + if (node.hasProjectedSchema()) { + insertNodeSpec.setProjectedSchema(node.getProjectedSchema().getProto()); + } + if (node.hasTargetSchema()) { + insertNodeSpec.setTargetSchema(node.getTargetSchema().getProto()); + } + if (node.hasPath()) { + insertNodeSpec.setPath(node.getPath().toString()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setPersistentStore(persistentStoreBuilder); + nodeBuilder.setStoreTable(storeTableBuilder); + nodeBuilder.setInsert(insertNodeSpec); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + private static PlanProto.PersistentStoreNode.Builder buildPersistentStoreBuilder(PersistentStoreNode node, + int [] childIds) { + PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = PlanProto.PersistentStoreNode.newBuilder(); + persistentStoreBuilder.setChildId(childIds[0]); + persistentStoreBuilder.setStorageType(node.getStorageType()); + if (node.hasOptions()) { + persistentStoreBuilder.setTableProperties(node.getOptions().getProto()); + } + return persistentStoreBuilder; + } + + private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { + PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); + if (node.hasPartition()) { + storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); + } + if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' + storeTableBuilder.setTableName(node.getTableName()); + } + return storeTableBuilder; + } + + @Override + public LogicalNode visitCreateDatabase(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + CreateDatabaseNode node, Stack stack) throws PlanningException { + PlanProto.CreateDatabaseNode.Builder createDatabaseBuilder = PlanProto.CreateDatabaseNode.newBuilder(); + createDatabaseBuilder.setDbName(node.getDatabaseName()); + createDatabaseBuilder.setIfNotExists(node.isIfNotExists()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setCreateDatabase(createDatabaseBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + @Override + public LogicalNode visitDropDatabase(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DropDatabaseNode node, Stack stack) throws PlanningException { + PlanProto.DropDatabaseNode.Builder dropDatabaseBuilder = PlanProto.DropDatabaseNode.newBuilder(); + dropDatabaseBuilder.setDbName(node.getDatabaseName()); + dropDatabaseBuilder.setIfExists(node.isIfExists()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setDropDatabase(dropDatabaseBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + public static PlanProto.NodeType convertType(NodeType type) { return PlanProto.NodeType.valueOf(type.name()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java index 888b9b1f40..4a574d381f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java @@ -251,10 +251,17 @@ public RESULT visitJoin(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBloc public RESULT visitUnion(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, UnionNode node, Stack stack) throws PlanningException { stack.push(node); - LogicalPlan.QueryBlock leftBlock = plan.getBlock(node.getLeftChild()); - RESULT result = visit(context, plan, leftBlock, leftBlock.getRoot(), stack); - LogicalPlan.QueryBlock rightBlock = plan.getBlock(node.getRightChild()); - visit(context, plan, rightBlock, rightBlock.getRoot(), stack); + RESULT result = null; + if (plan != null) { + LogicalPlan.QueryBlock leftBlock = plan.getBlock(node.getLeftChild()); + result = visit(context, plan, leftBlock, leftBlock.getRoot(), stack); + LogicalPlan.QueryBlock rightBlock = plan.getBlock(node.getRightChild()); + visit(context, plan, rightBlock, rightBlock.getRoot(), stack); + } else { + result = visit(context, plan, null, node.getLeftChild(), stack); + visit(context, plan, null, node.getRightChild(), stack); + } + stack.pop(); return result; } @@ -283,8 +290,13 @@ public RESULT visitIntersect(CONTEXT context, LogicalPlan plan, LogicalPlan.Quer public RESULT visitTableSubQuery(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack stack) throws PlanningException { stack.push(node); - LogicalPlan.QueryBlock childBlock = plan.getBlock(node.getSubQuery()); - RESULT result = visit(context, plan, childBlock, childBlock.getRoot(), stack); + RESULT result = null; + if (plan != null) { + LogicalPlan.QueryBlock childBlock = plan.getBlock(node.getSubQuery()); + result = visit(context, plan, childBlock, childBlock.getRoot(), stack); + } else { + result = visit(context, plan, null, node.getSubQuery(), stack); + } stack.pop(); return result; } diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index e5e8045b5e..ab9cdb8c59 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -26,34 +26,36 @@ import "CatalogProtos.proto"; import "DataTypes.proto"; enum NodeType { - ROOT = 0; - EXPRS = 1; - PROJECTION = 2; - LIMIT = 3; - WINDOW_AGG = 4; - SORT = 5; - HAVING = 6; - GROUP_BY = 7; - DISTINCT_GROUP_BY = 8; - SELECTION = 9; - JOIN = 10; - UNION = 11; - INTERSECT = 12; - EXCEPT = 13; - TABLE_SUBQUERY = 14; - SCAN = 15; - PARTITIONS_SCAN = 16; - BST_INDEX_SCAN = 17; - STORE = 18; - INSERT = 19; - - CREATE_DATABASE = 20; - DROP_DATABASE = 21; - CREATE_TABLE = 22; - DROP_TABLE = 23; - ALTER_TABLESPACE = 24; - ALTER_TABLE = 25; - TRUNCATE_TABLE = 26; + SET_SESSION = 0; + + ROOT = 1; + EXPRS = 2; + PROJECTION = 3; + LIMIT = 4; + WINDOW_AGG = 5; + SORT = 6; + HAVING = 7; + GROUP_BY = 8; + DISTINCT_GROUP_BY = 9; + SELECTION = 10; + JOIN = 11; + UNION = 12; + INTERSECT = 13; + EXCEPT = 14; + TABLE_SUBQUERY = 15; + SCAN = 16; + PARTITIONS_SCAN = 17; + BST_INDEX_SCAN = 18; + STORE = 19; + INSERT = 20; + + CREATE_DATABASE = 21; + DROP_DATABASE = 22; + CREATE_TABLE = 23; + DROP_TABLE = 24; + ALTER_TABLESPACE = 25; + ALTER_TABLE = 26; + TRUNCATE_TABLE = 27; } message LogicalNodeTree { @@ -64,8 +66,8 @@ message LogicalNode { required int32 sid = 1; required int32 pid = 2; required NodeType type = 3; - required SchemaProto in_schema = 4; - required SchemaProto out_schema = 5; + optional SchemaProto in_schema = 4; + optional SchemaProto out_schema = 5; optional ScanNode scan = 6; optional JoinNode join = 7; @@ -76,17 +78,22 @@ message LogicalNode { optional WindowAggNode window = 13; optional ProjectionNode projection = 14; optional EvalExprNode exprEval = 15; - optional SetNode set = 16; + optional UnionNode union = 16; optional TableSubQueryNode tableSubQuery = 17; optional PersistentStoreNode persistentStore = 18; optional StoreTableNodeSpec storeTable = 19; optional InsertNodeSpec insert = 20; optional CreateTableNodeSpec createTable = 21; optional RootNode root = 22; + optional SetSessionNode setSession = 23; - optional CreateDatabaseNode createDatabase = 23; - optional DropDatabaseNode dropDatabase = 24; - optional DropTableNode dropTable = 25; + optional CreateDatabaseNode createDatabase = 24; + optional DropDatabaseNode dropDatabase = 25; + optional DropTableNode dropTable = 26; + + optional AlterTablespaceNode alterTablespace = 27; + optional AlterTableNode alterTable = 28; + optional TruncateTableNode truncateTableNode = 29; } message ScanNode { @@ -142,9 +149,10 @@ message WindowAggNode { repeated Target targets = 6; } -message SetNode { +message UnionNode { required int32 leftChildId = 1; required int32 rightChildId = 2; + required bool all = 3; } message TableSubQueryNode { @@ -153,6 +161,43 @@ message TableSubQueryNode { repeated Target targets = 3; } +message ProjectionNode { + required int32 childId = 1; + required bool distinct = 2; + repeated Target targets = 3; +} + +message EvalExprNode { + repeated Target targets = 1; +} + +message RootNode { + required int32 childId = 1; +} + +message SetSessionNode { + required string name = 1; + optional string value = 2; +} + +message Target { + required EvalTree expr = 1; + optional string alias = 2; +} + +enum JoinType { + CROSS_JOIN = 0; + INNER_JOIN = 1; + LEFT_OUTER_JOIN = 2; + RIGHT_OUTER_JOIN = 3; + FULL_OUTER_JOIN = 4; + UNION_JOIN = 5; + LEFT_ANTI_JOIN = 6; + RIGHT_ANTI_JOIN = 7; + LEFT_SEMI_JOIN = 8; + RIGHT_SEMI_JOIN = 9; +} + message PartitionTableScanSpec { repeated string paths = 1; } @@ -164,30 +209,33 @@ message PersistentStoreNode { } message StoreTableNodeSpec { // required PersistentStoreSpec - repeated string tableName = 2; - optional PartitionDescProto partitionDesc = 3; + optional string tableName = 1; // 'INSERT INTO LOCATION' does not require 'table name'. + optional PartitionMethodProto partitionMethod = 2; } -message InsertNodeSpec { // required StoreTableSpec +message InsertNodeSpec { // required PersistentStoreSpec and StoreTableSpec required bool overwrite = 1; required SchemaProto tableSchema = 2; - required SchemaProto targetSchema = 3; - required SchemaProto projectedSchema = 4; - required string path = 5; + optional SchemaProto targetSchema = 4; + optional SchemaProto projectedSchema = 3; + optional string path = 5; } -message ProjectionNode { - required int32 childId = 1; - required bool distinct = 2; - repeated Target targets = 3; +message CreateTableNodeSpec { // required PersistentStoreSpec and StoreTableNodeSpec + required SchemaProto schema = 1; + required bool external = 2; + required bool ifNotExists = 3; + optional string path = 4; } -message EvalExprNode { - repeated Target targets = 1; +message DropTableNode { + required string tableName = 1; + required bool ifExists = 2; + required bool purge = 3; } -message RootNode { - required int32 childId = 1; +message TruncateTableNode { + repeated string tableNames = 1; } message CreateDatabaseNode { @@ -200,35 +248,45 @@ message DropDatabaseNode { required bool ifExists = 2; } -message CreateTableNodeSpec { // required PersistentStoreSpec - required SchemaProto schema = 1; - required string path = 2; - required bool external = 3; - required bool ifNotExists = 4; -} +message AlterTablespaceNode { + enum Type { + LOCATION = 0; + } -message DropTableNode { - required string tableName = 1; - required bool ifExists = 2; - required bool purge = 3; -} + message SetLocation { + required string location = 1; + } -message Target { - required EvalTree expr = 1; - required string alias = 2; + required string tableSpaceName = 1; + required Type setType = 2; + optional SetLocation setLocation = 3; } -enum JoinType { - CROSS_JOIN = 0; - INNER_JOIN = 1; - LEFT_OUTER_JOIN = 2; - RIGHT_OUTER_JOIN = 3; - FULL_OUTER_JOIN = 4; - UNION_JOIN = 5; - LEFT_ANTI_JOIN = 6; - RIGHT_ANTI_JOIN = 7; - LEFT_SEMI_JOIN = 8; - RIGHT_SEMI_JOIN = 9; +message AlterTableNode { + enum Type { + RENAME_TABLE = 0; + RENAME_COLUMN = 1; + ADD_COLUMN = 2; + } + + message RenameTable { + required string newName = 1; + } + + message RenameColumn { + required string oldName = 1; + required string newName = 2; + } + + message AddColumn { + required ColumnProto addColumn = 1; + } + + required string tableName = 1; + required Type setType = 2; + optional RenameTable renameTable = 3; + optional RenameColumn renameColumn = 4; + optional AddColumn addColumn = 5; } enum EvalType { From e55e168161628812a37b2ed82a9c71d9f34086a7 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 26 Dec 2014 17:06:47 +0900 Subject: [PATCH 21/31] Fix test failures. --- .../tajo/engine/planner/global/GlobalPlanner.java | 12 ++++++++++++ .../apache/tajo/plan/logical/DropDatabaseNode.java | 7 +++++-- .../plan/serder/LogicalNodeTreeDeserializer.java | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index c75b34814b..d82246b014 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -38,6 +38,9 @@ import org.apache.tajo.engine.planner.global.builder.DistinctGroupbyBuilder; import org.apache.tajo.exception.InternalException; import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.Target; @@ -163,6 +166,15 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { masterPlan.setTerminal(terminalBlock); LOG.info("\n" + masterPlan.toString()); + + ExecutionBlockCursor cursor = new ExecutionBlockCursor(masterPlan); + while (cursor.hasNext()) { + ExecutionBlock eb = cursor.nextBlock(); + LogicalNode node = eb.getPlan(); + PlanProto.LogicalNodeTree tree = LogicalNodeTreeSerializer.serialize(node); + LogicalNode deserialize = LogicalNodeTreeDeserializer.deserialize(masterPlan.getContext(), tree); + assert node.deepEquals(deserialize); + } } private static void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java index 05911e8678..c566bf5ccc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java @@ -24,7 +24,7 @@ public class DropDatabaseNode extends LogicalNode implements Cloneable { @Expose private String databaseName; - @Expose private boolean ifExists; + @Expose private boolean ifExists = false; public DropDatabaseNode(int pid) { super(pid, NodeType.DROP_DATABASE); @@ -65,7 +65,10 @@ public int hashCode() { public boolean equals(Object obj) { if (obj instanceof DropDatabaseNode) { DropDatabaseNode other = (DropDatabaseNode) obj; - return super.equals(other) && this.databaseName.equals(other.databaseName) && ifExists == other.ifExists; + boolean eq = super.equals(other); + eq &= this.databaseName.equals(other.databaseName); + eq &= ifExists == other.ifExists; + return eq; } else { return false; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java index 47cf19cf56..7bddbca0ba 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java @@ -420,7 +420,7 @@ public static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoNo DropDatabaseNode dropDatabase = new DropDatabaseNode(protoNode.getPid()); PlanProto.DropDatabaseNode dropDatabaseProto = protoNode.getDropDatabase(); - dropDatabase.init(dropDatabaseProto.getDbName(), dropDatabase.isIfExists()); + dropDatabase.init(dropDatabaseProto.getDbName(), dropDatabaseProto.getIfExists()); return dropDatabase; } From 5ac3f67cd89922e9263d13b092a4a2e5bf1ca199 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sat, 27 Dec 2014 18:12:53 +0900 Subject: [PATCH 22/31] Fixed manu test failures. --- .../main/java/org/apache/tajo/util/TUtil.java | 33 ++++++++ .../engine/codegen/ExecutorPreCompiler.java | 6 +- .../engine/planner/PhysicalPlannerImpl.java | 4 +- .../engine/planner/global/GlobalPlanner.java | 8 +- .../builder/DistinctGroupbyBuilder.java | 32 ++++---- .../DistinctGroupbyFirstAggregationExec.java | 2 +- .../DistinctGroupbyHashAggregationExec.java | 4 +- .../DistinctGroupbySecondAggregationExec.java | 2 +- .../DistinctGroupbySortAggregationExec.java | 4 +- .../DistinctGroupbyThirdAggregationExec.java | 2 +- .../plan/logical/DistinctGroupbyNode.java | 39 +++++----- .../apache/tajo/plan/logical/GroupbyNode.java | 9 +-- .../plan/serder/EvalTreeProtoSerializer.java | 4 + .../serder/LogicalNodeTreeDeserializer.java | 77 ++++++++++++++++--- .../serder/LogicalNodeTreeSerializer.java | 69 ++++++++++++++--- .../apache/tajo/plan/util/PlannerUtil.java | 8 +- .../plan/visitor/BasicLogicalPlanVisitor.java | 6 +- .../visitor/ExplainLogicalPlanVisitor.java | 5 +- .../tajo/plan/visitor/LogicalPlanVisitor.java | 4 +- tajo-plan/src/main/proto/Plan.proto | 13 +++- 20 files changed, 242 insertions(+), 89 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index 0ceb2b2dbe..3df448ca5e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -41,6 +41,29 @@ public static boolean checkEquals(Object s1, Object s2) { return Objects.equal(s1, s2); } + public static boolean checkEquals(Collection s1, Collection s2) { + if (s1 == null ^ s2 == null) { + return false; + } else if (s1 == null && s2 == null) { + return true; + } else { + if (s1.size() == 0 && s2.size() == 0) { + return true; + } else if (s1.size() == s2.size()) { + Iterator it1 = s1.iterator(); + Iterator it2 = s2.iterator(); + Object o1; + Object o2; + for (o1 = it1.next(), o2 = it2.next(); it1.hasNext() && it2.hasNext(); o1 = it1.next(), o2 = it2.next()) { + if (!o1.equals(o2)) { + return false; + } + } + } + return true; + } + } + /** * check two arrays as equals. It also check the equivalence of null. * It will return true even if they are all null. @@ -59,6 +82,16 @@ public static boolean checkEquals(Object [] s1, Object [] s2) { } } + public static boolean checkEquals(int [] s1, int [] s2) { + if (s1 == null ^ s2 == null) { + return false; + } else if (s1 == null && s2 == null) { + return true; + } else { + return Arrays.equals(s1, s2); + } + } + public static T[] concat(T[] first, T[] second) { T[] result = Arrays.copyOf(first, first.length + second.length); System.arraycopy(second, 0, result, first.length, second.length); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java index ddba57d6e6..79513dc734 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java @@ -149,9 +149,9 @@ public LogicalNode visitWindowAgg(CompilationContext context, LogicalPlan plan, return node; } - public LogicalNode visitDistinct(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, - DistinctGroupbyNode node, Stack stack) throws PlanningException { - super.visitDistinct(context, plan, block, node, stack); + public LogicalNode visitDistinctGroupby(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DistinctGroupbyNode node, Stack stack) throws PlanningException { + super.visitDistinctGroupby(context, plan, block, node, stack); compileProjectableNode(context, node.getInSchema(), node); return node; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 03dac47f90..90173d71ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -1090,7 +1090,7 @@ private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, if (phase == 3) { sortSpecs.add(new SortSpec(distinctNode.getTargets()[0].getNamedColumn())); } - for (GroupbyNode eachGroupbyNode: distinctNode.getGroupByNodes()) { + for (GroupbyNode eachGroupbyNode: distinctNode.getSubPlans()) { for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } @@ -1111,7 +1111,7 @@ private PhysicalExec createInMemoryDistinctGroupbyExec(TaskAttemptContext ctx, private PhysicalExec createSortAggregationDistinctGroupbyExec(TaskAttemptContext ctx, DistinctGroupbyNode distinctGroupbyNode, PhysicalExec subOp, DistinctGroupbyEnforcer enforcer) throws IOException { - List groupbyNodes = distinctGroupbyNode.getGroupByNodes(); + List groupbyNodes = distinctGroupbyNode.getSubPlans(); SortAggregateExec[] sortAggregateExec = new SortAggregateExec[groupbyNodes.size()]; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index d82246b014..4a2123ca62 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -171,9 +171,11 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { while (cursor.hasNext()) { ExecutionBlock eb = cursor.nextBlock(); LogicalNode node = eb.getPlan(); - PlanProto.LogicalNodeTree tree = LogicalNodeTreeSerializer.serialize(node); - LogicalNode deserialize = LogicalNodeTreeDeserializer.deserialize(masterPlan.getContext(), tree); - assert node.deepEquals(deserialize); + if (node != null) { + PlanProto.LogicalNodeTree tree = LogicalNodeTreeSerializer.serialize(node); + LogicalNode deserialize = LogicalNodeTreeDeserializer.deserialize(masterPlan.getContext(), tree); + assert node.deepEquals(deserialize); + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 4c5e8b1fa6..8a2552fde0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -99,7 +99,7 @@ public ExecutionBlock buildMultiLevelPlan(GlobalPlanContext context, DistinctGroupbyNode thirdStageDistinctNode = PlannerUtil.clone(plan, baseDistinctNode); // Set second, third non-distinct aggregation's eval node to field eval - GroupbyNode lastGroupbyNode = secondStageDistinctNode.getGroupByNodes().get(secondStageDistinctNode.getGroupByNodes().size() - 1); + GroupbyNode lastGroupbyNode = secondStageDistinctNode.getSubPlans().get(secondStageDistinctNode.getSubPlans().size() - 1); if (!lastGroupbyNode.isDistinct()) { int index = 0; for (AggregationFunctionCallEval aggrFunction: lastGroupbyNode.getAggFunctions()) { @@ -108,7 +108,7 @@ public ExecutionBlock buildMultiLevelPlan(GlobalPlanContext context, index++; } } - lastGroupbyNode = thirdStageDistinctNode.getGroupByNodes().get(thirdStageDistinctNode.getGroupByNodes().size() - 1); + lastGroupbyNode = thirdStageDistinctNode.getSubPlans().get(thirdStageDistinctNode.getSubPlans().size() - 1); if (!lastGroupbyNode.isDistinct()) { int index = 0; for (AggregationFunctionCallEval aggrFunction: lastGroupbyNode.getAggFunctions()) { @@ -300,11 +300,11 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou DistinctGroupbyNode baseDistinctNode = new DistinctGroupbyNode(context.getPlan().getLogicalPlan().newPID()); baseDistinctNode.setTargets(baseGroupByTargets.toArray(new Target[]{})); - baseDistinctNode.setGroupColumns(groupbyNode.getGroupingColumns()); + baseDistinctNode.setGroupingColumns(groupbyNode.getGroupingColumns()); baseDistinctNode.setInSchema(groupbyNode.getInSchema()); baseDistinctNode.setChild(groupbyNode.getChild()); - baseDistinctNode.setGroupbyNodes(childGroupbyNodes); + baseDistinctNode.setSubPlans(childGroupbyNodes); return baseDistinctNode; } @@ -468,11 +468,11 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou DistinctGroupbyNode baseDistinctNode = new DistinctGroupbyNode(context.getPlan().getLogicalPlan().newPID()); baseDistinctNode.setTargets(groupbyNode.getTargets()); - baseDistinctNode.setGroupColumns(groupbyNode.getGroupingColumns()); + baseDistinctNode.setGroupingColumns(groupbyNode.getGroupingColumns()); baseDistinctNode.setInSchema(groupbyNode.getInSchema()); baseDistinctNode.setChild(groupbyNode.getChild()); - baseDistinctNode.setGroupbyNodes(childGroupbyNodes); + baseDistinctNode.setSubPlans(childGroupbyNodes); return baseDistinctNode; } @@ -529,12 +529,12 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // - Change SecondStage's aggregation expr and target column name. For example: // exprs: (sum(default.lineitem.l_quantity (FLOAT8))) ==> exprs: (sum(?sum_3 (FLOAT8))) int grpIdx = 0; - for (GroupbyNode firstStageGroupbyNode: firstStageDistinctNode.getGroupByNodes()) { - GroupbyNode secondStageGroupbyNode = secondStageDistinctNode.getGroupByNodes().get(grpIdx); + for (GroupbyNode firstStageGroupbyNode: firstStageDistinctNode.getSubPlans()) { + GroupbyNode secondStageGroupbyNode = secondStageDistinctNode.getSubPlans().get(grpIdx); if (firstStageGroupbyNode.isDistinct()) { // FirstStage: Remove aggregation, Set target with only grouping columns - firstStageGroupbyNode.setAggFunctions(GroupbyNode.EMPTY_AGG_FUNCS); + firstStageGroupbyNode.setAggFunctions(PlannerUtil.EMPTY_AGG_FUNCS); List firstGroupbyTargets = new ArrayList(); for (Column column : firstStageGroupbyNode.getGroupingColumns()) { @@ -614,7 +614,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // In the case of distinct query without group by clause // other aggregation function is added to last distinct group by node. - List secondStageGroupbyNodes = secondStageDistinctNode.getGroupByNodes(); + List secondStageGroupbyNodes = secondStageDistinctNode.getSubPlans(); GroupbyNode lastSecondStageGroupbyNode = secondStageGroupbyNodes.get(secondStageGroupbyNodes.size() - 1); if (!lastSecondStageGroupbyNode.isDistinct() && lastSecondStageGroupbyNode.isEmptyGrouping()) { GroupbyNode otherGroupbyNode = lastSecondStageGroupbyNode; @@ -644,7 +644,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou List firstStageColumnIds = new ArrayList(); columnIdIndex = 0; List firstTargets = new ArrayList(); - for (GroupbyNode firstStageGroupbyNode: firstStageDistinctNode.getGroupByNodes()) { + for (GroupbyNode firstStageGroupbyNode: firstStageDistinctNode.getSubPlans()) { if (firstStageGroupbyNode.isDistinct()) { for (Column column : firstStageGroupbyNode.getGroupingColumns()) { Target firstTarget = new Target(new FieldEval(column)); @@ -674,7 +674,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou Schema secondStageInSchema = new Schema(); //TODO merged tuple schema int index = 0; - for(GroupbyNode eachNode: secondStageDistinctNode.getGroupByNodes()) { + for(GroupbyNode eachNode: secondStageDistinctNode.getSubPlans()) { eachNode.setInSchema(firstStageDistinctNode.getOutSchema()); for (Column column: eachNode.getOutSchema().getColumns()) { if (secondStageInSchema.getColumn(column) == null) { @@ -695,13 +695,13 @@ private void setDistinctAggregationEnforcer( List sortSpecArrays = new ArrayList(); int index = 0; - for (GroupbyNode groupbyNode: firstStageDistinctNode.getGroupByNodes()) { + for (GroupbyNode groupbyNode: firstStageDistinctNode.getSubPlans()) { List sortSpecs = new ArrayList(); for (Column column: groupbyNode.getGroupingColumns()) { sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); } sortSpecArrays.add( SortSpecArray.newBuilder() - .setPid(secondStageDistinctNode.getGroupByNodes().get(index).getPID()) + .setPid(secondStageDistinctNode.getSubPlans().get(index).getPID()) .addAllSortSpecs(sortSpecs).build()); } secondStageBlock.getEnforcer().enforceDistinctAggregation(secondStageDistinctNode.getPID(), @@ -723,13 +723,13 @@ private void setMultiStageAggregationEnforcer( List sortSpecArrays = new ArrayList(); int index = 0; - for (GroupbyNode groupbyNode: firstStageDistinctNode.getGroupByNodes()) { + for (GroupbyNode groupbyNode: firstStageDistinctNode.getSubPlans()) { List sortSpecs = new ArrayList(); for (Column column: groupbyNode.getGroupingColumns()) { sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); } sortSpecArrays.add( SortSpecArray.newBuilder() - .setPid(thirdStageDistinctNode.getGroupByNodes().get(index).getPID()) + .setPid(thirdStageDistinctNode.getSubPlans().get(index).getPID()) .addAllSortSpecs(sortSpecs).build()); } thirdStageBlock.getEnforcer().enforceDistinctAggregation(thirdStageDistinctNode.getPID(), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java index bd24fa352c..aca4879334 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java @@ -130,7 +130,7 @@ public void init() throws IOException { } resultTupleLength = groupingKeyIndexes.length + 1; //1 is Sequence Datum which indicates sequence of DistinctNode. - List groupbyNodes = plan.getGroupByNodes(); + List groupbyNodes = plan.getSubPlans(); List distinctAggrList = new ArrayList(); int distinctSeq = 0; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index eac5c70b68..37d61a917d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -76,7 +76,7 @@ public DistinctGroupbyHashAggregationExec(TaskAttemptContext context, DistinctGr distinctGroupingKeyIds[idx++] = intVal.intValue(); } - List groupbyNodes = plan.getGroupByNodes(); + List groupbyNodes = plan.getSubPlans(); groupbyNodeNum = groupbyNodes.size(); this.hashAggregators = new HashAggregator[groupbyNodeNum]; @@ -88,7 +88,7 @@ public DistinctGroupbyHashAggregationExec(TaskAttemptContext context, DistinctGr outputColumnNum = plan.getOutSchema().size(); int allGroupbyOutColNum = 0; - for (GroupbyNode eachGroupby: plan.getGroupByNodes()) { + for (GroupbyNode eachGroupby: plan.getSubPlans()) { allGroupbyOutColNum += eachGroupby.getOutSchema().size(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java index 383ccd3b4c..cce9a24726 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java @@ -100,7 +100,7 @@ public void init() throws IOException { numGroupingColumns = plan.getGroupingColumns().length; - List groupbyNodes = plan.getGroupByNodes(); + List groupbyNodes = plan.getSubPlans(); // Finding distinct group by column index. Set groupingKeyIndexSet = new HashSet(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java index 06b241c734..6641633ba5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java @@ -47,13 +47,13 @@ public DistinctGroupbySortAggregationExec(final TaskAttemptContext context, Dist super(context, plan.getInSchema(), plan.getOutSchema()); this.plan = plan; this.aggregateExecs = aggregateExecs; - this.groupbyNodeNum = plan.getGroupByNodes().size(); + this.groupbyNodeNum = plan.getSubPlans().size(); currentTuples = new Tuple[groupbyNodeNum]; outColumnNum = outSchema.size(); int allGroupbyOutColNum = 0; - for (GroupbyNode eachGroupby: plan.getGroupByNodes()) { + for (GroupbyNode eachGroupby: plan.getSubPlans()) { allGroupbyOutColNum += eachGroupby.getOutSchema().size(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java index ff6fc4a02f..a76b91da55 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java @@ -66,7 +66,7 @@ public void init() throws IOException { numGroupingColumns = plan.getGroupingColumns().length; resultTupleLength = numGroupingColumns; - List groupbyNodes = plan.getGroupByNodes(); + List groupbyNodes = plan.getSubPlans(); List aggregatorList = new ArrayList(); int inTupleIndex = 1 + numGroupingColumns; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java index e31e488785..a40ad59a40 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java @@ -34,19 +34,19 @@ public class DistinctGroupbyNode extends UnaryNode implements Projectable, Clone private GroupbyNode groupbyPlan; @Expose - private List groupByNodes; + private List subGroupbyPlan; @Expose private Target[] targets; @Expose - private Column[] groupingColumns; + private Column[] groupingColumns = PlannerUtil.EMPTY_COLUMNS; @Expose - private int[] resultColumnIds; + private int[] resultColumnIds = new int[]{}; /** Aggregation Functions */ - @Expose private AggregationFunctionCallEval[] aggrFunctions; + @Expose private AggregationFunctionCallEval[] aggrFunctions = PlannerUtil.EMPTY_AGG_FUNCS; public DistinctGroupbyNode(int pid) { super(pid, NodeType.DISTINCT_GROUP_BY); @@ -54,7 +54,7 @@ public DistinctGroupbyNode(int pid) { @Override public boolean hasTargets() { - return targets != null && targets.length > 0; + return targets.length > 0; } @Override @@ -72,19 +72,19 @@ public Target[] getTargets() { } } - public void setGroupbyNodes(List groupByNodes) { - this.groupByNodes = groupByNodes; + public void setSubPlans(List groupByNodes) { + this.subGroupbyPlan = groupByNodes; } - public List getGroupByNodes() { - return groupByNodes; + public List getSubPlans() { + return subGroupbyPlan; } public final Column[] getGroupingColumns() { return groupingColumns; } - public final void setGroupColumns(Column[] groupingColumns) { + public final void setGroupingColumns(Column[] groupingColumns) { this.groupingColumns = groupingColumns; } @@ -119,12 +119,12 @@ public Object clone() throws CloneNotSupportedException { } } - if (groupByNodes != null) { - cloneNode.groupByNodes = new ArrayList(); - for (GroupbyNode eachNode: groupByNodes) { + if (subGroupbyPlan != null) { + cloneNode.subGroupbyPlan = new ArrayList(); + for (GroupbyNode eachNode: subGroupbyPlan) { GroupbyNode groupbyNode = (GroupbyNode)eachNode.clone(); groupbyNode.setPID(-1); - cloneNode.groupByNodes.add(groupbyNode); + cloneNode.subGroupbyPlan.add(groupbyNode); } } @@ -151,7 +151,7 @@ public String toString() { sb.append("grouping set=").append(TUtil.arrayToString(groupingColumns)); sb.append(", "); } - for (GroupbyNode eachNode: groupByNodes) { + for (GroupbyNode eachNode: subGroupbyPlan) { sb.append(", groupbyNode=").append(eachNode.toString()); } sb.append(")"); @@ -164,8 +164,9 @@ public boolean equals(Object obj) { DistinctGroupbyNode other = (DistinctGroupbyNode) obj; boolean eq = super.equals(other); eq = eq && TUtil.checkEquals(groupingColumns, other.groupingColumns); - eq = eq && TUtil.checkEquals(groupByNodes, other.groupByNodes); + eq = eq && TUtil.checkEquals(subGroupbyPlan, other.subGroupbyPlan); eq = eq && TUtil.checkEquals(targets, other.targets); + eq = eq && TUtil.checkEquals(resultColumnIds, other.resultColumnIds); return eq; } else { return false; @@ -194,7 +195,7 @@ public PlanString getPlanString() { sb.append("("); String prefix = ""; - for (GroupbyNode eachNode: groupByNodes) { + for (GroupbyNode eachNode: subGroupbyPlan) { if (eachNode.hasAggFunctions()) { AggregationFunctionCallEval[] aggrFunctions = eachNode.getAggFunctions(); for (int j = 0; j < aggrFunctions.length; j++) { @@ -218,7 +219,7 @@ public PlanString getPlanString() { planStr.addDetail("out schema:").appendDetail(getOutSchema().toString()); planStr.addDetail("in schema:").appendDetail(getInSchema().toString()); - for (GroupbyNode eachNode: groupByNodes) { + for (GroupbyNode eachNode: subGroupbyPlan) { planStr.addDetail("\t").appendDetail("distinct: " + eachNode.isDistinct()) .appendDetail(", " + eachNode.getShortPlanString()); } @@ -236,7 +237,7 @@ public Column[] getFirstStageShuffleKeyColumns() { } } } - for (GroupbyNode eachGroupbyNode: groupByNodes) { + for (GroupbyNode eachGroupbyNode: subGroupbyPlan) { if (eachGroupbyNode.getGroupingColumns() != null && eachGroupbyNode.getGroupingColumns().length > 0) { for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { if (!shuffleKeyColumns.contains(eachColumn)) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java index 6a8f41272e..fd22d2bf27 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java @@ -28,13 +28,10 @@ import org.apache.tajo.util.TUtil; public class GroupbyNode extends UnaryNode implements Projectable, Cloneable { - private static final Column [] EMPTY_GROUPING_KEYS = new Column[] {}; - public static final AggregationFunctionCallEval [] EMPTY_AGG_FUNCS = new AggregationFunctionCallEval[] {}; - - /** Grouping key sets */ - @Expose private Column [] groupingKeys = EMPTY_GROUPING_KEYS; + /** Grouping key sets */ + @Expose private Column [] groupingKeys = PlannerUtil.EMPTY_COLUMNS; /** Aggregation Functions */ - @Expose private AggregationFunctionCallEval [] aggrFunctions = EMPTY_AGG_FUNCS; + @Expose private AggregationFunctionCallEval [] aggrFunctions = PlannerUtil.EMPTY_AGG_FUNCS; /** * It's a list of targets. The grouping columns should be followed by aggregation functions. * aggrFunctions keep actual aggregation functions, but it only contains field references. diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java index 92a245f98e..f03a2caf36 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java @@ -134,6 +134,10 @@ public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, Stack 0) { + + if (groupbyProto.getGroupingKeysCount() > 0) { groupby.setGroupingColumns(convertColumns(groupbyProto.getGroupingKeysList())); } if (groupbyProto.getAggFunctionsCount() > 0) { groupby.setAggFunctions(convertAggFuncCallEvals(context, groupbyProto.getAggFunctionsList())); } + if (groupbyProto.getTargetsCount() > 0) { + groupby.setTargets(convertTargets(context, groupbyProto.getTargetsList())); + } + + groupby.setInSchema(convertSchema(protoNode.getInSchema())); + groupby.setOutSchema(convertSchema(protoNode.getOutSchema())); + return groupby; } + public static DistinctGroupbyNode convertDistinctGroupby(OverridableConf context, Map nodeMap, + PlanProto.LogicalNode protoNode) { + PlanProto.DistinctGroupbyNode distinctGroupbyProto = protoNode.getDistinctGroupby(); + + DistinctGroupbyNode distinctGroupby = new DistinctGroupbyNode(protoNode.getPid()); + distinctGroupby.setChild(nodeMap.get(distinctGroupbyProto.getChildId())); + + if (distinctGroupbyProto.hasGroupbyNode()) { + distinctGroupby.setGroupbyPlan(convertGroupby(context, nodeMap, distinctGroupbyProto.getGroupbyNode())); + } + + if (distinctGroupbyProto.getSubPlansCount() > 0) { + List subPlans = TUtil.newList(); + for (int i = 0; i < distinctGroupbyProto.getSubPlansCount(); i++) { + subPlans.add(convertGroupby(context, nodeMap, distinctGroupbyProto.getSubPlans(i))); + } + distinctGroupby.setSubPlans(subPlans); + } + + if (distinctGroupbyProto.getGroupingKeysCount() > 0) { + distinctGroupby.setGroupingColumns(convertColumns(distinctGroupbyProto.getGroupingKeysList())); + } + if (distinctGroupbyProto.getAggFunctionsCount() > 0) { + distinctGroupby.setAggFunctions(convertAggFuncCallEvals(context, distinctGroupbyProto.getAggFunctionsList())); + } + if (distinctGroupbyProto.getTargetsCount() > 0) { + distinctGroupby.setTargets(convertTargets(context, distinctGroupbyProto.getTargetsList())); + } + int [] resultColumnIds = new int[distinctGroupbyProto.getResultIdCount()]; + for (int i = 0; i < distinctGroupbyProto.getResultIdCount(); i++) { + resultColumnIds[i] = distinctGroupbyProto.getResultId(i); + } + distinctGroupby.setResultColumnIds(resultColumnIds); + + // TODO - in distinct groupby, output and target are not matched to each other. It does not follow the convention. + distinctGroupby.setInSchema(convertSchema(protoNode.getInSchema())); + distinctGroupby.setOutSchema(convertSchema(protoNode.getOutSchema())); + + return distinctGroupby; + } + public static JoinNode convertJoin(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.JoinNode joinProto = protoNode.getJoin(); @@ -307,10 +355,17 @@ public static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNod scan.init(new TableDesc(scanProto.getTable())); } - if (scanProto.getTargetsCount() > 0) { + if (scanProto.getExistTargets()) { scan.setTargets(convertTargets(context, scanProto.getTargetsList())); } + if (scanProto.hasQual()) { + scan.setQual(EvalTreeProtoDeserializer.deserialize(context, scanProto.getQual())); + } + + scan.setInSchema(convertSchema(protoNode.getInSchema())); + scan.setOutSchema(convertSchema(protoNode.getOutSchema())); + return scan; } @@ -320,9 +375,11 @@ public static TableSubQueryNode convertTableSubQuery(OverridableConf context, PlanProto.TableSubQueryNode proto = protoNode.getTableSubQuery(); TableSubQueryNode tableSubQuery = new TableSubQueryNode(protoNode.getPid()); + tableSubQuery.init(proto.getTableName(), nodeMap.get(proto.getChildId())); tableSubQuery.setInSchema(convertSchema(protoNode.getInSchema())); - tableSubQuery.setSubQuery(nodeMap.get(proto.getChildId())); - tableSubQuery.setTargets(convertTargets(context, proto.getTargetsList())); + if (proto.getTargetsCount() > 0) { + tableSubQuery.setTargets(convertTargets(context, proto.getTargetsList())); + } return tableSubQuery; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java index a5c79ed2df..ceb6934624 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java @@ -34,6 +34,7 @@ import org.apache.tajo.plan.serder.PlanProto.AlterTablespaceNode.SetLocation; import org.apache.tajo.plan.serder.PlanProto.LogicalNodeTree; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.util.TUtil; import java.util.List; import java.util.Map; @@ -237,28 +238,75 @@ public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, Logic } public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, - GroupbyNode groupbyNode, Stack stack) throws PlanningException { - super.visitGroupBy(context, plan, block, groupbyNode, new Stack()); + GroupbyNode node, Stack stack) throws PlanningException { + super.visitGroupBy(context, plan, block, node, new Stack()); - int [] childIds = registerGetChildIds(context, groupbyNode); + PlanProto.LogicalNode.Builder nodeBuilder = convertGroupby(context, node); + context.treeBuilder.addNodes(nodeBuilder); + return node; + } + + private PlanProto.LogicalNode.Builder convertGroupby(SerializeContext context, GroupbyNode node) + throws PlanningException { + int [] childIds = registerGetChildIds(context, node); PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); groupbyBuilder.setChildId(childIds[0]); - if (groupbyNode.groupingKeyNum() > 0) { + if (node.groupingKeyNum() > 0) { groupbyBuilder.addAllGroupingKeys( - LogicalNodeTreeSerializer.toProtoObjects(groupbyNode.getGroupingColumns())); + LogicalNodeTreeSerializer.toProtoObjects(node.getGroupingColumns())); } - if (groupbyNode.hasAggFunctions()) { + if (node.hasAggFunctions()) { groupbyBuilder.addAllAggFunctions( - LogicalNodeTreeSerializer.toProtoObjects(groupbyNode.getAggFunctions())); + LogicalNodeTreeSerializer.toProtoObjects(node.getAggFunctions())); + } + if (node.hasTargets()) { + groupbyBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(node.getTargets())); } - PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, groupbyNode); + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setGroupby(groupbyBuilder); + + return nodeBuilder; + } + + public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DistinctGroupbyNode node, Stack stack) throws PlanningException { + super.visitDistinctGroupby(context, plan, block, node, new Stack()); + + int [] childIds = registerGetChildIds(context, node); + + PlanProto.DistinctGroupbyNode.Builder distGroupbyBuilder = PlanProto.DistinctGroupbyNode.newBuilder(); + distGroupbyBuilder.setChildId(childIds[0]); + if (node.getGroupbyPlan() != null) { + distGroupbyBuilder.setGroupbyNode(convertGroupby(context, node.getGroupbyPlan())); + } + + for (GroupbyNode subPlan : node.getSubPlans()) { + distGroupbyBuilder.addSubPlans(convertGroupby(context, subPlan)); + } + + if (node.getGroupingColumns().length > 0) { + distGroupbyBuilder.addAllGroupingKeys( + LogicalNodeTreeSerializer.toProtoObjects(node.getGroupingColumns())); + } + if (node.getAggFunctions().length > 0) { + distGroupbyBuilder.addAllAggFunctions( + LogicalNodeTreeSerializer.toProtoObjects(node.getAggFunctions())); + } + if (node.hasTargets()) { + distGroupbyBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(node.getTargets())); + } + for (int cid : node.getResultColumnIds()) { + distGroupbyBuilder.addResultId(cid); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setDistinctGroupby(distGroupbyBuilder); context.treeBuilder.addNodes(nodeBuilder); - return groupbyNode; + return node; } @Override @@ -334,7 +382,10 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical } if (scan.hasTargets()) { + scanBuilder.setExistTargets(true); scanBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(scan.getTargets())); + } else { + scanBuilder.setExistTargets(false); } if (scan.hasQual()) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index d9f077ca9c..36b9fa6243 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -26,10 +26,8 @@ import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; @@ -48,6 +46,10 @@ public class PlannerUtil { + public static final Column [] EMPTY_COLUMNS = new Column[] {}; + public static final Target [] EMPTY_TARGETS = new Target[] {}; + public static final AggregationFunctionCallEval [] EMPTY_AGG_FUNCS = new AggregationFunctionCallEval[] {}; + public static boolean checkIfSetSession(LogicalNode node) { LogicalNode baseNode = node; if (node instanceof LogicalRootNode) { @@ -698,7 +700,7 @@ public static T clone(LogicalPlan plan, LogicalNode node copy.setPID(plan.newPID()); if (node instanceof DistinctGroupbyNode) { DistinctGroupbyNode dNode = (DistinctGroupbyNode)copy; - for (GroupbyNode eachNode: dNode.getGroupByNodes()) { + for (GroupbyNode eachNode: dNode.getSubPlans()) { eachNode.setPID(plan.newPID()); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java index 4a574d381f..23c834d5c7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java @@ -84,7 +84,7 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl current = visitWindowAgg(context, plan, block, (WindowAggNode) node, stack); break; case DISTINCT_GROUP_BY: - current = visitDistinct(context, plan, block, (DistinctGroupbyNode) node, stack); + current = visitDistinctGroupby(context, plan, block, (DistinctGroupbyNode) node, stack); break; case SELECTION: current = visitFilter(context, plan, block, (SelectionNode) node, stack); @@ -220,8 +220,8 @@ public RESULT visitWindowAgg(CONTEXT context, LogicalPlan plan, LogicalPlan.Quer return result; } - public RESULT visitDistinct(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, - Stack stack) throws PlanningException { + public RESULT visitDistinctGroupby(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DistinctGroupbyNode node, Stack stack) throws PlanningException { stack.push(node); RESULT result = visit(context, plan, block, node.getChild(), stack); stack.pop(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java index 7065295db4..52db8eb24f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java @@ -117,8 +117,9 @@ public LogicalNode visitWindowAgg(Context context, LogicalPlan plan, LogicalPlan return visitUnaryNode(context, plan, block, node, stack); } - public LogicalNode visitDistinct(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, - Stack stack) throws PlanningException { + public LogicalNode visitDistinctGroupby(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DistinctGroupbyNode node, + Stack stack) throws PlanningException { return visitUnaryNode(context, plan, block, node, stack); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java index 8c94a563ee..5be2eec6fa 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java @@ -51,8 +51,8 @@ RESULT visitGroupBy(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl Stack stack) throws PlanningException; RESULT visitWindowAgg(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, WindowAggNode node, Stack stack) throws PlanningException; - RESULT visitDistinct(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, - Stack stack) throws PlanningException; + RESULT visitDistinctGroupby(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, + Stack stack) throws PlanningException; RESULT visitFilter(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, SelectionNode node, Stack stack) throws PlanningException; diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index ab9cdb8c59..e5e18917c8 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -73,6 +73,7 @@ message LogicalNode { optional JoinNode join = 7; optional FilterNode filter = 8; optional GroupbyNode groupby = 9; + optional DistinctGroupbyNode distinctGroupby = 10; optional SortNode sort = 11; optional LimitNode limit = 12; optional WindowAggNode window = 13; @@ -99,6 +100,7 @@ message LogicalNode { message ScanNode { required TableDescProto table = 1; optional string alias = 2; + required bool existTargets = 3; repeated Target targets = 4; optional EvalTree qual = 5; } @@ -120,14 +122,17 @@ message GroupbyNode { required int32 childId = 1; repeated ColumnProto groupingKeys = 2; repeated EvalTree aggFunctions = 3; + repeated Target targets = 4; } message DistinctGroupbyNode { required int32 childId = 1; - repeated GroupbyNode groupbyNodes = 2; - repeated Target targets = 3; - repeated ColumnProto groupingKeys = 4; - repeated int32 resultId = 5; + optional LogicalNode groupbyNode = 2; + repeated LogicalNode subPlans = 3; + repeated Target targets = 4; + repeated ColumnProto groupingKeys = 5; + repeated int32 resultId = 6; + repeated EvalTree aggFunctions = 7; } message SortNode { From ef71d30f92ea83e9311d2e3d289733ab1cd5f880 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sat, 27 Dec 2014 22:45:01 +0900 Subject: [PATCH 23/31] Fix test failures. --- .../org/apache/tajo/catalog/TableDesc.java | 2 +- .../serder/LogicalNodeTreeDeserializer.java | 68 ++++++++++++++++++- .../serder/LogicalNodeTreeSerializer.java | 50 ++++++++++++-- tajo-plan/src/main/proto/Plan.proto | 58 +++++++++------- 4 files changed, 143 insertions(+), 35 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index ce167e1cfc..ec679f9258 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -168,7 +168,7 @@ public boolean equals(Object object) { boolean eq = tableName.equals(other.tableName); eq = eq && schema.equals(other.schema); eq = eq && meta.equals(other.meta); - eq = eq && uri.equals(other.uri); + eq = eq && TUtil.checkEquals(uri, other.uri); eq = eq && TUtil.checkEquals(partitionMethodDesc, other.partitionMethodDesc); eq = eq && TUtil.checkEquals(external, other.external); return eq && TUtil.checkEquals(stats, other.stats); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java index 80f72d09b9..ed2e0ca7c7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java @@ -34,6 +34,8 @@ import org.apache.tajo.plan.expr.AggregationFunctionCallEval; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.expr.WindowFunctionEval; +import org.apache.tajo.plan.function.WindowAggFunc; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.util.KeyValueSet; @@ -89,6 +91,9 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { case SORT: current = convertSort(nodeMap, protoNode); break; + case WINDOW_AGG: + current = convertWindowAgg(context, nodeMap, protoNode); + break; case HAVING: current = convertHaving(context, nodeMap, protoNode); break; @@ -110,6 +115,9 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { case UNION: current = convertUnion(nodeMap, protoNode); break; + case PARTITIONS_SCAN: + current = convertPartitionScan(context, protoNode); + break; case SCAN: current = convertScan(context, protoNode); break; @@ -236,6 +244,37 @@ public static HavingNode convertHaving(OverridableConf context, Map nodeMap, + PlanProto.LogicalNode protoNode) { + PlanProto.WindowAggNode windowAggProto = protoNode.getWindowAgg(); + + WindowAggNode windowAgg = new WindowAggNode(protoNode.getPid()); + windowAgg.setChild(nodeMap.get(windowAggProto.getChildId())); + + if (windowAggProto.getPartitionKeysCount() > 0) { + windowAgg.setPartitionKeys(convertColumns(windowAggProto.getPartitionKeysList())); + } + + if (windowAggProto.getWindowFunctionsCount() > 0) { + windowAgg.setWindowFunctions(convertWindowFunccEvals(context, windowAggProto.getWindowFunctionsList())); + } + + windowAgg.setDistinct(windowAggProto.getDistinct()); + + if (windowAggProto.getSortSpecsCount() > 0) { + windowAgg.setSortSpecs(convertSortSpecs(windowAggProto.getSortSpecsList())); + } + + if (windowAggProto.getTargetsCount() > 0) { + windowAgg.setTargets(convertTargets(context, windowAggProto.getTargetsList())); + } + + windowAgg.setInSchema(convertSchema(protoNode.getInSchema())); + windowAgg.setOutSchema(convertSchema(protoNode.getOutSchema())); + + return windowAgg; + } + public static GroupbyNode convertGroupby(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.GroupbyNode groupbyProto = protoNode.getGroupby(); @@ -313,7 +352,7 @@ public static JoinNode convertJoin(OverridableConf context, Map 0) { + if (joinProto.getExistsTargets()) { join.setTargets(convertTargets(context, joinProto.getTargetsList())); } @@ -347,7 +386,12 @@ public static UnionNode convertUnion(Map nodeMap, PlanProt public static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNode protoNode) { ScanNode scan = new ScanNode(protoNode.getPid()); + fillScanNode(context, protoNode, scan); + return scan; + } + + public static void fillScanNode(OverridableConf context, PlanProto.LogicalNode protoNode, ScanNode scan) { PlanProto.ScanNode scanProto = protoNode.getScan(); if (scanProto.hasAlias()) { scan.init(new TableDesc(scanProto.getTable()), scanProto.getAlias()); @@ -365,8 +409,19 @@ public static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNod scan.setInSchema(convertSchema(protoNode.getInSchema())); scan.setOutSchema(convertSchema(protoNode.getOutSchema())); + } - return scan; + public static PartitionedTableScanNode convertPartitionScan(OverridableConf context, PlanProto.LogicalNode protoNode) { + PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getPid()); + fillScanNode(context, protoNode, partitionedScan); + + PlanProto.PartitionScanSpec partitionScanProto = protoNode.getPartitionScan(); + Path [] paths = new Path[partitionScanProto.getPathsCount()]; + for (int i = 0; i < partitionScanProto.getPathsCount(); i++) { + paths[i] = new Path(partitionScanProto.getPaths(i)); + } + partitionedScan.setInputPaths(paths); + return partitionedScan; } public static TableSubQueryNode convertTableSubQuery(OverridableConf context, @@ -554,6 +609,15 @@ public static Schema convertSchema(CatalogProtos.SchemaProto proto) { return aggFuncs; } + public static WindowFunctionEval[] convertWindowFunccEvals(OverridableConf context, + List evalTrees) { + WindowFunctionEval[] winFuncEvals = new WindowFunctionEval[evalTrees.size()]; + for (int i = 0; i < winFuncEvals.length; i++) { + winFuncEvals[i] = (WindowFunctionEval) EvalTreeProtoDeserializer.deserialize(context, evalTrees.get(i)); + } + return winFuncEvals; + } + public static Column[] convertColumns(List columnProtos) { Column [] columns = new Column[columnProtos.size()]; for (int i = 0; i < columns.length; i++) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java index ceb6934624..928ba7b2f5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java @@ -20,6 +20,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.hadoop.fs.Path; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; @@ -177,10 +178,12 @@ public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, Lo PlanProto.WindowAggNode.Builder windowAggBuilder = PlanProto.WindowAggNode.newBuilder(); windowAggBuilder.setChildId(childIds[0]); + if (windowAgg.hasPartitionKeys()) { windowAggBuilder.addAllPartitionKeys( LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getPartitionKeys())); } + if (windowAgg.hasAggFunctions()) { windowAggBuilder.addAllWindowFunctions( LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getWindowFunctions())); @@ -196,6 +199,10 @@ public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, Lo LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getTargets())); } + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, windowAgg); + nodeBuilder.setWindowAgg(windowAggBuilder); + context.treeBuilder.addNodes(nodeBuilder); + return windowAgg; } @@ -241,12 +248,12 @@ public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, Logi GroupbyNode node, Stack stack) throws PlanningException { super.visitGroupBy(context, plan, block, node, new Stack()); - PlanProto.LogicalNode.Builder nodeBuilder = convertGroupby(context, node); + PlanProto.LogicalNode.Builder nodeBuilder = buildGroupby(context, node); context.treeBuilder.addNodes(nodeBuilder); return node; } - private PlanProto.LogicalNode.Builder convertGroupby(SerializeContext context, GroupbyNode node) + private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, GroupbyNode node) throws PlanningException { int [] childIds = registerGetChildIds(context, node); @@ -280,11 +287,11 @@ public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan pl PlanProto.DistinctGroupbyNode.Builder distGroupbyBuilder = PlanProto.DistinctGroupbyNode.newBuilder(); distGroupbyBuilder.setChildId(childIds[0]); if (node.getGroupbyPlan() != null) { - distGroupbyBuilder.setGroupbyNode(convertGroupby(context, node.getGroupbyPlan())); + distGroupbyBuilder.setGroupbyNode(buildGroupby(context, node.getGroupbyPlan())); } for (GroupbyNode subPlan : node.getSubPlans()) { - distGroupbyBuilder.addSubPlans(convertGroupby(context, subPlan)); + distGroupbyBuilder.addSubPlans(buildGroupby(context, subPlan)); } if (node.getGroupingColumns().length > 0) { @@ -341,8 +348,12 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical if (join.hasJoinQual()) { joinBuilder.setJoinQual(EvalTreeProtoSerializer.serialize(join.getJoinQual())); } + if (join.hasTargets()) { + joinBuilder.setExistsTargets(true); joinBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(join.getTargets())); + } else { + joinBuilder.setExistsTargets(false); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, join); @@ -375,6 +386,16 @@ public LogicalNode visitUnion(SerializeContext context, LogicalPlan plan, Logica public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scan, Stack stack) throws PlanningException { + PlanProto.ScanNode.Builder scanBuilder = buildScanNode(scan); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, scan); + nodeBuilder.setScan(scanBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return scan; + } + + public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan) { PlanProto.ScanNode.Builder scanBuilder = PlanProto.ScanNode.newBuilder(); scanBuilder.setTable(scan.getTableDesc().getProto()); if (scan.hasAlias()) { @@ -391,12 +412,29 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical if (scan.hasQual()) { scanBuilder.setQual(EvalTreeProtoSerializer.serialize(scan.getQual())); } + return scanBuilder; + } - PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, scan); + @Override + public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + PartitionedTableScanNode node, Stack stack) + throws PlanningException { + + PlanProto.ScanNode.Builder scanBuilder = buildScanNode(node); + + PlanProto.PartitionScanSpec.Builder partitionScan = PlanProto.PartitionScanSpec.newBuilder(); + List pathStrs = TUtil.newList(); + for (Path p : node.getInputPaths()) { + pathStrs.add(p.toString()); + } + partitionScan.addAllPaths(pathStrs); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setScan(scanBuilder); + nodeBuilder.setPartitionScan(partitionScan); context.treeBuilder.addNodes(nodeBuilder); - return scan; + return node; } public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index e5e18917c8..30bd138adc 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -70,31 +70,32 @@ message LogicalNode { optional SchemaProto out_schema = 5; optional ScanNode scan = 6; - optional JoinNode join = 7; - optional FilterNode filter = 8; - optional GroupbyNode groupby = 9; - optional DistinctGroupbyNode distinctGroupby = 10; - optional SortNode sort = 11; - optional LimitNode limit = 12; - optional WindowAggNode window = 13; - optional ProjectionNode projection = 14; - optional EvalExprNode exprEval = 15; - optional UnionNode union = 16; - optional TableSubQueryNode tableSubQuery = 17; - optional PersistentStoreNode persistentStore = 18; - optional StoreTableNodeSpec storeTable = 19; - optional InsertNodeSpec insert = 20; - optional CreateTableNodeSpec createTable = 21; - optional RootNode root = 22; - optional SetSessionNode setSession = 23; - - optional CreateDatabaseNode createDatabase = 24; - optional DropDatabaseNode dropDatabase = 25; - optional DropTableNode dropTable = 26; - - optional AlterTablespaceNode alterTablespace = 27; - optional AlterTableNode alterTable = 28; - optional TruncateTableNode truncateTableNode = 29; + optional PartitionScanSpec partitionScan = 7; + optional JoinNode join = 8; + optional FilterNode filter = 9; + optional GroupbyNode groupby = 10; + optional DistinctGroupbyNode distinctGroupby = 11; + optional SortNode sort = 12; + optional LimitNode limit = 13; + optional WindowAggNode windowAgg = 14; + optional ProjectionNode projection = 15; + optional EvalExprNode exprEval = 16; + optional UnionNode union = 17; + optional TableSubQueryNode tableSubQuery = 18; + optional PersistentStoreNode persistentStore = 19; + optional StoreTableNodeSpec storeTable = 20; + optional InsertNodeSpec insert = 21; + optional CreateTableNodeSpec createTable = 22; + optional RootNode root = 23; + optional SetSessionNode setSession = 24; + + optional CreateDatabaseNode createDatabase = 25; + optional DropDatabaseNode dropDatabase = 26; + optional DropTableNode dropTable = 27; + + optional AlterTablespaceNode alterTablespace = 28; + optional AlterTableNode alterTable = 29; + optional TruncateTableNode truncateTableNode = 30; } message ScanNode { @@ -105,6 +106,10 @@ message ScanNode { optional EvalTree qual = 5; } +message PartitionScanSpec { + repeated string paths = 1; +} + message FilterNode { required int32 childId = 1; required EvalTree qual = 2; @@ -115,7 +120,8 @@ message JoinNode { required int32 rightChildId = 2; required JoinType joinType = 3; optional EvalTree joinQual = 4; - repeated Target targets = 5; + required bool existsTargets = 5; + repeated Target targets = 6; } message GroupbyNode { From dbfab9721dd7225a7f855617c9ac0c1660eec67c Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 28 Dec 2014 22:06:16 +0900 Subject: [PATCH 24/31] Injected test code. --- pom.xml | 1 - .../java/org/apache/tajo/conf/TajoConf.java | 18 + .../org/apache/tajo/util/ReflectionUtil.java | 56 ++- .../engine/planner/global/GlobalPlanner.java | 51 ++- .../BaseGlobalPlanRewriteRuleProvider.java | 39 ++ .../rewriter/GlobalPlanRewriteEngine.java | 85 +++++ .../rewriter/GlobalPlanRewriteRule.java | 49 +++ .../GlobalPlanRewriteRuleProvider.java | 31 +- .../rewriter/GlobalPlanTestRuleProvider.java | 44 +++ .../rules/GlobalPlanEqualityTester.java | 63 ++++ .../utils/test/ErrorInjectionRewriter.java | 10 +- .../org/apache/tajo/master/GlobalEngine.java | 1 + .../master/querymaster/QueryMasterTask.java | 9 +- .../org/apache/tajo/TajoTestingCluster.java | 15 +- .../tajo/engine/query/TestSelectQuery.java | 27 +- .../tajo/engine/query/TestWindowQuery.java | 6 +- tajo-jitvec-tests/pom.xml | 351 ------------------ .../apache/tajo/jitvec/JitVecTestServer.java | 183 --------- .../apache/tajo/plan/LogicalOptimizer.java | 45 +-- .../org/apache/tajo/plan/LogicalPlanner.java | 4 - ...java => BaseLogicalPlanRewriteEngine.java} | 35 +- .../BaseLogicalPlanRewriteRuleProvider.java | 59 +++ ...ine.java => LogicalPlanRewriteEngine.java} | 5 +- ...eRule.java => LogicalPlanRewriteRule.java} | 7 +- .../LogicalPlanRewriteRuleProvider.java | 44 +++ .../rewrite/LogicalPlanTestRuleProvider.java | 44 +++ .../rewrite/rules/FilterPushDownRule.java | 9 +- .../rules/LogicalPlanEqualityTester.java | 55 +++ .../rules/PartitionedTableRewriter.java | 44 +-- .../rewrite/rules/ProjectionPushDownRule.java | 9 +- .../apache/tajo/storage/StorageManager.java | 4 +- .../hbase/AddSortForInsertRewriter.java | 10 +- .../storage/hbase/HBaseStorageManager.java | 6 +- 33 files changed, 716 insertions(+), 703 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/BaseGlobalPlanRewriteRuleProvider.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRule.java rename tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto => tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRuleProvider.java (57%) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanTestRuleProvider.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java delete mode 100644 tajo-jitvec-tests/pom.xml delete mode 100644 tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java rename tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/{BasicQueryRewriteEngine.java => BaseLogicalPlanRewriteEngine.java} (63%) create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java rename tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/{QueryRewriteEngine.java => LogicalPlanRewriteEngine.java} (86%) rename tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/{RewriteRule.java => LogicalPlanRewriteRule.java} (88%) create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleProvider.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanTestRuleProvider.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java diff --git a/pom.xml b/pom.xml index 449e8cffbb..62e03f79dd 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,6 @@ tajo-pullserver tajo-dist tajo-thirdparty/asm - tajo-jitvec-tests diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index d0c6460a6f..3dba9e0745 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -249,6 +249,10 @@ public static enum ConfVars implements ConfigKey { TASK_DEFAULT_SIZE("tajo.task.size-mb", 128), // Query and Optimization ------------------------------------------------- + LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS("tajo.plan.logical.rewriter.provider", + "org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteRuleProvider"), + GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS("tajo.plan.global.rewriter.provider", + "org.apache.tajo.engine.planner.global.rewriter.BaseGlobalPlanRewriteRuleProvider"), EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1), EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8), @@ -561,6 +565,20 @@ public void setBoolVar(ConfVars var, boolean val) { setBoolVar(this, var, val); } + public void setClassVar(ConfVars var, Class clazz) { + setVar(var, clazz.getCanonicalName()); + } + + public Class getClassVar(ConfVars var) { + String valueString = getVar(var); + + try { + return getClassByName(valueString); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + public static String getVar(Configuration conf, ConfVars var) { return conf.get(var.varname, var.defaultVal); } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java index eccc61f7d3..5d34a9217d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java @@ -18,22 +18,66 @@ package org.apache.tajo.util; +import org.apache.tajo.conf.TajoConf; + import java.lang.reflect.Constructor; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; public class ReflectionUtil { - private static final Class[] EMPTY_ARRAY = new Class[]{}; + private static final Class[] EMPTY_PARAM = new Class[]{}; + private static final Object [] EMPTY_OBJECT = new Object[] {}; + private static final Class[] CONF_PARAM = new Class[]{TajoConf.class}; /** * Cache of constructors for each class. Pins the classes so they * can't be garbage collected until ReflectionUtils can be collected. */ - private static final Map, Constructor> CONSTRUCTOR_CACHE = - new ConcurrentHashMap, Constructor>(); + private static final Map, Constructor> EMPTY_CONSTRUCTOR_CACHE = new ConcurrentHashMap, Constructor>(); + private static final Map, Constructor> CONF_CONSTRUCTOR_CACHE = new ConcurrentHashMap, Constructor>(); + + /** + * Initialize an instance by a given class + * + * @param clazz Class to be initialized + * @return initialized object + */ + public static T newInstance(Class clazz) { + try { + Constructor constructor; + if (EMPTY_CONSTRUCTOR_CACHE.containsKey(clazz)) { + constructor = EMPTY_CONSTRUCTOR_CACHE.get(clazz); + } else { + constructor = clazz.getConstructor(EMPTY_PARAM); + EMPTY_CONSTRUCTOR_CACHE.put(clazz, constructor); + } - public static Object newInstance(Class clazz) - throws InstantiationException, IllegalAccessException { - return clazz.newInstance(); + return (T) constructor.newInstance(EMPTY_OBJECT); + } catch (Throwable t) { + throw new RuntimeException(t); + } } + + /** + * Initialize an instance by a given class with TajoConf parameter + * + * @param clazz Class to be initialized + * @param conf TajoConf instance + * @return initialized object + */ + public static T newInstance(Class clazz, TajoConf conf) { + try { + Constructor constructor; + if (CONF_CONSTRUCTOR_CACHE.containsKey(clazz)) { + constructor = CONF_CONSTRUCTOR_CACHE.get(clazz); + } else { + constructor = clazz.getConstructor(CONF_PARAM); + CONF_CONSTRUCTOR_CACHE.put(clazz, constructor); + } + + return (T) constructor.newInstance(new Object[]{conf}); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 4a2123ca62..53c5340049 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.global; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -36,20 +37,23 @@ import org.apache.tajo.engine.planner.BroadcastJoinMarkCandidateVisitor; import org.apache.tajo.engine.planner.BroadcastJoinPlanVisitor; import org.apache.tajo.engine.planner.global.builder.DistinctGroupbyBuilder; +import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanRewriteEngine; +import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanRewriteRuleProvider; import org.apache.tajo.exception.InternalException; import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; -import org.apache.tajo.plan.serder.PlanProto; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; +import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.PlanProto; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.ReflectionUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TajoWorker; @@ -57,6 +61,7 @@ import java.util.*; import static org.apache.tajo.conf.TajoConf.ConfVars; +import static org.apache.tajo.conf.TajoConf.ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS; import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.*; /** @@ -67,34 +72,29 @@ public class GlobalPlanner { private final TajoConf conf; private final CatalogProtos.StoreType storeType; - private CatalogService catalog; - private TajoWorker.WorkerContext workerContext; + private final CatalogService catalog; + private final GlobalPlanRewriteEngine rewriteEngine; + @VisibleForTesting public GlobalPlanner(final TajoConf conf, final CatalogService catalog) throws IOException { this.conf = conf; this.catalog = catalog; this.storeType = CatalogProtos.StoreType.valueOf(conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase()); Preconditions.checkArgument(storeType != null); + + Class clazz = + (Class) conf.getClassVar(GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); + GlobalPlanRewriteRuleProvider provider = ReflectionUtil.newInstance(clazz, conf); + rewriteEngine = new GlobalPlanRewriteEngine(); + rewriteEngine.addRewriteRule(provider.getRules()); } public GlobalPlanner(final TajoConf conf, final TajoWorker.WorkerContext workerContext) throws IOException { - this.conf = conf; - this.workerContext = workerContext; - this.storeType = CatalogProtos.StoreType.valueOf(conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase()); - Preconditions.checkArgument(storeType != null); + this(conf, workerContext.getCatalog()); } - /** - * TODO: this is hack. it must be refactored at TAJO-602. - */ public CatalogService getCatalog() { - if (workerContext.getCatalog() != null) { - return workerContext.getCatalog(); - } else if (catalog != null) { - return catalog; - } else { - throw new IllegalStateException("No Catalog Instance"); - } + return catalog; } public CatalogProtos.StoreType getStoreType() { @@ -167,16 +167,7 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { masterPlan.setTerminal(terminalBlock); LOG.info("\n" + masterPlan.toString()); - ExecutionBlockCursor cursor = new ExecutionBlockCursor(masterPlan); - while (cursor.hasNext()) { - ExecutionBlock eb = cursor.nextBlock(); - LogicalNode node = eb.getPlan(); - if (node != null) { - PlanProto.LogicalNodeTree tree = LogicalNodeTreeSerializer.serialize(node); - LogicalNode deserialize = LogicalNodeTreeDeserializer.deserialize(masterPlan.getContext(), tree); - assert node.deepEquals(deserialize); - } - } + masterPlan = rewriteEngine.rewrite(masterPlan); } private static void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/BaseGlobalPlanRewriteRuleProvider.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/BaseGlobalPlanRewriteRuleProvider.java new file mode 100644 index 0000000000..96ee2c6d11 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/BaseGlobalPlanRewriteRuleProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.global.rewriter; + +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.util.TUtil; + +import java.util.Collection; +import java.util.List; + +@SuppressWarnings("unused") +public class BaseGlobalPlanRewriteRuleProvider extends GlobalPlanRewriteRuleProvider { + private static final List> EMPTY_RULES = TUtil.newList(); + + public BaseGlobalPlanRewriteRuleProvider(TajoConf conf) { + super(conf); + } + + @Override + public Collection> getRules() { + return EMPTY_RULES; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java new file mode 100644 index 0000000000..6ea5de3d9e --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.global.rewriter; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.engine.planner.global.MasterPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.util.ReflectionUtil; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class GlobalPlanRewriteEngine { + /** class logger */ + private static final Log LOG = LogFactory.getLog(GlobalPlanRewriteEngine.class); + + /** a map for query rewrite rules */ + private final Map rewriteRules = new LinkedHashMap(); + + /** + * Add a query rewrite rule to this engine. + * + * @param rules Rule classes + */ + public void addRewriteRule(Iterable> rules) { + for (Class clazz : rules) { + try { + GlobalPlanRewriteRule rule = ReflectionUtil.newInstance(clazz); + addRewriteRule(rule); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + } + + /** + * Add a query rewrite rule to this engine. + * + * @param rule The rule to be added to this engine. + */ + public void addRewriteRule(GlobalPlanRewriteRule rule) { + if (!rewriteRules.containsKey(rule.getName())) { + rewriteRules.put(rule.getName(), rule); + } + } + + /** + * Rewrite a global plan with all query rewrite rules added to this engine. + * + * @param plan The plan to be rewritten with all query rewrite rule. + * @return The rewritten plan. + */ + public MasterPlan rewrite(MasterPlan plan) throws PlanningException { + GlobalPlanRewriteRule rule; + for (Map.Entry rewriteRule : rewriteRules.entrySet()) { + rule = rewriteRule.getValue(); + if (rule.isEligible(plan)) { + plan = rule.rewrite(plan); + if (LOG.isDebugEnabled()) { + LOG.debug("The rule \"" + rule.getName() + " \" rewrites the query."); + } + } + } + + return plan; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRule.java new file mode 100644 index 0000000000..4a37207a85 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRule.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.global.rewriter; + +import org.apache.tajo.engine.planner.global.MasterPlan; + +/** + * A rewrite rule for global plans + */ +public interface GlobalPlanRewriteRule { + + /** + * Return rule name + * @return Rule name + */ + public abstract String getName(); + + /** + * Check if this rule should be applied. + * + * @param plan Global Plan + * @return + */ + public abstract boolean isEligible(MasterPlan plan); + + /** + * Rewrite a global plan + * + * @param plan Global Plan + * @return + */ + public abstract MasterPlan rewrite(MasterPlan plan); +} diff --git a/tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRuleProvider.java similarity index 57% rename from tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto rename to tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRuleProvider.java index 643064f130..638b5f3c92 100644 --- a/tajo-jitvec-tests/src/main/proto/JitVecTestProtocol.proto +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteRuleProvider.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,25 +16,18 @@ * limitations under the License. */ -// JitVecEngine -> JetVecTestServer +package org.apache.tajo.engine.planner.global.rewriter; -package tajo; -option cc_generic_services = true; +import org.apache.tajo.conf.TajoConf; -option java_package = "org.apache.tajo.jitvec"; -option java_outer_classname = "JitVecTestServerProtocol"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; +import java.util.Collection; -message RequestPlan { - required string sql = 1; -} +public abstract class GlobalPlanRewriteRuleProvider { + protected final TajoConf conf; -message PlanResponse { - optional string serializedPlan = 1; - optional string errorMessage = 2; -} + public GlobalPlanRewriteRuleProvider(TajoConf conf) { + this.conf = conf; + } -service JitVecTestServerProtocolService { - rpc requestPlan(RequestPlan) returns (PlanResponse); -} \ No newline at end of file + public abstract Collection> getRules(); +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanTestRuleProvider.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanTestRuleProvider.java new file mode 100644 index 0000000000..dc91577214 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanTestRuleProvider.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.global.rewriter; + +import com.google.common.collect.Lists; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.engine.planner.global.rewriter.rules.GlobalPlanEqualityTester; + +import java.util.Collection; +import java.util.List; + +/** + * It is used only for test. + */ +@SuppressWarnings("unused") +public class GlobalPlanTestRuleProvider extends BaseGlobalPlanRewriteRuleProvider { + + public GlobalPlanTestRuleProvider(TajoConf conf) { + super(conf); + } + + @Override + public Collection> getRules() { + List> injectedRules = Lists.newArrayList(super.getRules()); + injectedRules.add(GlobalPlanEqualityTester.class); + return injectedRules; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java new file mode 100644 index 0000000000..7124c13548 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.global.rewriter.rules; + +import org.apache.tajo.engine.planner.global.ExecutionBlock; +import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; +import org.apache.tajo.engine.planner.global.MasterPlan; +import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanRewriteRule; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.PlanProto; + +/** + * It verifies the equality between the input and output of LogicalNodeTree(De)Serializer in global planning. + */ +public class GlobalPlanEqualityTester implements GlobalPlanRewriteRule { + + @Override + public String getName() { + return "GlobalPlanEqualityTester"; + } + + @Override + public boolean isEligible(MasterPlan plan) { + return true; + } + + @Override + public MasterPlan rewrite(MasterPlan plan) { + try { + ExecutionBlockCursor cursor = new ExecutionBlockCursor(plan); + while (cursor.hasNext()) { + ExecutionBlock eb = cursor.nextBlock(); + LogicalNode node = eb.getPlan(); + if (node != null) { + PlanProto.LogicalNodeTree tree = LogicalNodeTreeSerializer.serialize(node); + LogicalNode deserialize = LogicalNodeTreeDeserializer.deserialize(plan.getContext(), tree); + assert node.deepEquals(deserialize); + } + } + return plan; + } catch (Throwable t) { + throw new RuntimeException(t); + } + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java index 97872766c0..29dc845a26 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java @@ -18,23 +18,25 @@ package org.apache.tajo.engine.utils.test; +import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; -public class ErrorInjectionRewriter implements RewriteRule { +@SuppressWarnings("unused") +public class ErrorInjectionRewriter implements LogicalPlanRewriteRule { @Override public String getName() { return "ErrorInjectionRewriter"; } @Override - public boolean isEligible(LogicalPlan plan) { + public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { return true; } @Override - public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { throw new NullPointerException(); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index d7e76708e5..51964f0fb2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -94,6 +94,7 @@ public void start() { annotatedPlanVerifier = new LogicalPlanVerifier(context.getConf(), context.getCatalog()); } catch (Throwable t) { LOG.error(t.getMessage(), t); + throw new RuntimeException(t); } super.start(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index e3d3d79052..720d60a49e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -37,13 +37,11 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.master.exec.prehook.DistributedQueryHookManager; -import org.apache.tajo.master.exec.prehook.InsertIntoHook; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalRootNode; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.plan.logical.LogicalNode; @@ -53,7 +51,6 @@ import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.ipc.TajoMasterProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol; -import org.apache.tajo.master.GlobalEngine; import org.apache.tajo.master.TajoAsyncDispatcher; import org.apache.tajo.master.TajoContainerProxy; import org.apache.tajo.master.event.*; @@ -380,10 +377,10 @@ public synchronized void startQuery() { if (tableDesc == null) { throw new VerifyException("Can't get table meta data from catalog: " + tableName); } - List storageSpecifiedRewriteRules = sm.getRewriteRules( + List storageSpecifiedRewriteRules = sm.getRewriteRules( getQueryTaskContext().getQueryContext(), tableDesc); if (storageSpecifiedRewriteRules != null) { - for (RewriteRule eachRule: storageSpecifiedRewriteRules) { + for (LogicalPlanRewriteRule eachRule: storageSpecifiedRewriteRules) { optimizer.addRuleAfterToJoinOpt(eachRule); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 7dc10898ae..5ff637cc2e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -42,12 +42,14 @@ import org.apache.tajo.client.TajoClientUtil; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.querymaster.Query; import org.apache.tajo.master.querymaster.QueryMasterTask; import org.apache.tajo.master.querymaster.Stage; import org.apache.tajo.master.querymaster.StageState; import org.apache.tajo.master.rm.TajoWorkerResourceManager; +import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.NetUtils; @@ -57,10 +59,7 @@ import java.net.InetSocketAddress; import java.net.URL; import java.sql.ResultSet; -import java.util.ArrayList; -import java.util.List; -import java.util.TimeZone; -import java.util.UUID; +import java.util.*; public class TajoTestingCluster { private static Log LOG = LogFactory.getLog(TajoTestingCluster.class); @@ -119,10 +118,18 @@ void setTestingFlagProperties() { } void initPropertiesAndConfigs() { + + // Set time zone TimeZone testDefaultTZ = TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE); conf.setSystemTimezone(testDefaultTZ); TimeZone.setDefault(testDefaultTZ); + // Injection of equality testing code of logical plan (de)serialization + conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class); + conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class); + + + // default resource manager if (System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname) != null) { String testResourceManager = System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname); Preconditions.checkState(testResourceManager.equals(TajoWorkerResourceManager.class.getCanonicalName())); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 2d9b1f8302..0df4001089 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -26,18 +26,18 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.engine.utils.test.ErrorInjectionRewriter; import org.apache.tajo.jdbc.TajoResultSet; +import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteRuleProvider; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; -import java.util.HashMap; -import java.util.Map; -import java.util.TimeZone; +import java.util.*; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @@ -416,11 +416,23 @@ public final void testSumFloatOverflow() throws Exception { cleanupQuery(res); } + public static class RulesForErrorInjection extends BaseLogicalPlanRewriteRuleProvider { + public RulesForErrorInjection(TajoConf conf) { + super(conf); + } + + @Override + public Collection> getPostRules() { + List> addedRules = Lists.newArrayList(super.getPostRules()); + return addedRules; + } + } + @Test public final void testQueryMasterTaskInitError() throws Exception { // In this testcase we can check that a TajoClient receives QueryMasterTask's init error message. - testingCluster.setAllWorkersConfValue("tajo.plan.rewriter.classes", - ErrorInjectionRewriter.class.getCanonicalName()); + testingCluster.setAllWorkersConfValue(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS.name(), + RulesForErrorInjection.class.getCanonicalName()); try { // If client can't receive error status, thread runs forever. @@ -450,7 +462,8 @@ public void run() { // If query runs more than 10 secs, test is fail. assertFalse(t.isAlive()); } finally { - testingCluster.setAllWorkersConfValue("tajo.plan.rewriter.classes", ""); + // recover the rewrite rule provider to default + testingCluster.setAllWorkersConfValue(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS.name(), ""); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestWindowQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestWindowQuery.java index 2af5ce9d68..668ba70093 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestWindowQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestWindowQuery.java @@ -272,7 +272,8 @@ public final void firstValueTime() throws Exception { TajoTestingCluster.createTable("firstvaluetime", schema, tableOptions, data, 1); try { - ResultSet res = executeString("select id, first_value(time) over ( partition by id order by time ) as time_first from firstvaluetime"); + ResultSet res = executeString( + "select id, first_value(time) over ( partition by id order by time ) as time_first from firstvaluetime"); String ascExpected = "id,time_first\n" + "-------------------------------\n" + "1,12:11:12\n" + @@ -306,7 +307,8 @@ public final void lastValueTime() throws Exception { TajoTestingCluster.createTable("lastvaluetime", schema, tableOptions, data, 1); try { - ResultSet res = executeString("select id, last_value(time) over ( partition by id order by time ) as time_last from lastvaluetime"); + ResultSet res = executeString( + "select id, last_value(time) over ( partition by id order by time ) as time_last from lastvaluetime"); String ascExpected = "id,time_last\n" + "-------------------------------\n" + "1,12:11:12\n" + diff --git a/tajo-jitvec-tests/pom.xml b/tajo-jitvec-tests/pom.xml deleted file mode 100644 index f586dfa976..0000000000 --- a/tajo-jitvec-tests/pom.xml +++ /dev/null @@ -1,351 +0,0 @@ - - - - - - tajo-project - org.apache.tajo - 0.9.1-SNAPSHOT - ../tajo-project - - 4.0.0 - - tajo-jitvec-tests - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - ${project.build.sourceEncoding} - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.12.4 - - - TRUE - - -Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8 - - - - maven-deploy-plugin - - - org.apache.maven.plugins - maven-jar-plugin - 2.3.1 - - - LICENSE - - - false - - - - - package - - test-jar - - - - - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - create-protobuf-generated-sources-directory - initialize - - - - - - - run - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.2 - - - generate-sources - generate-sources - - protoc - - -Isrc/main/proto/ - --proto_path=../tajo-common/src/main/proto - --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto - --proto_path=../tajo-client/src/main/proto - --java_out=target/generated-sources/proto - src/main/proto/JitVecTestProtocol.proto - - - - exec - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 1.5 - - - add-source - generate-sources - - add-source - - - - target/generated-sources/proto - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - runtime - ${project.build.directory}/lib - false - false - true - - - - - - org.apache.maven.plugins - maven-pmd-plugin - 2.7.1 - - - - - - - org.apache.tajo - tajo-common - - - org.apache.tajo - tajo-algebra - - - org.apache.tajo - tajo-catalog-common - - - org.apache.tajo - tajo-catalog-client - - - org.apache.tajo - tajo-catalog-server - - - org.apache.tajo - tajo-storage-common - - - org.apache.tajo - tajo-client - - - org.apache.tajo - tajo-plan - - - org.apache.tajo - tajo-jdbc - - - org.apache.tajo - tajo-rpc - - - org.apache.tajo - tajo-core - jar - - - org.apache.tajo - tajo-core - compile - test-jar - - - - org.apache.hadoop - hadoop-common - compile - - - org.apache.hadoop - hadoop-minicluster - compile - - - commons-el - commons-el - - - tomcat - jasper-runtime - - - tomcat - jasper-compiler - - - org.mortbay.jetty - jsp-2.1-jetty - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - - - - - org.apache.hadoop - hadoop-hdfs - compile - - - commons-el - commons-el - - - tomcat - jasper-runtime - - - tomcat - jasper-compiler - - - org.mortbay.jetty - jsp-2.1-jetty - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - - - - - org.apache.hadoop - hadoop-yarn-api - compile - - - org.apache.hadoop - hadoop-yarn-common - compile - - - org.apache.hadoop - hadoop-yarn-server-common - compile - - - - com.google.protobuf - protobuf-java - - - org.antlr - antlr4 - - - junit - junit - test - - - org.mockito - mockito-core - test - - - org.dspace.dependencies - dspace-geoip - 1.2.3 - - - org.eclipse.jdt - core - 3.1.1 - - - org.mortbay.jetty - jetty - 6.1.14 - - - org.mortbay.jetty - jsp-2.1 - 6.1.14 - - - info.ganglia.gmetric4j - gmetric4j - 1.0.3 - - - - \ No newline at end of file diff --git a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java b/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java deleted file mode 100644 index ba019bcaa3..0000000000 --- a/tajo-jitvec-tests/src/main/java/org/apache/tajo/jitvec/JitVecTestServer.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.jitvec; - -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import com.sun.org.apache.commons.logging.Log; -import com.sun.org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.AbstractService; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.TpchTestBase; -import org.apache.tajo.algebra.Expr; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.session.Session; -import org.apache.tajo.plan.LogicalOptimizer; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.plan.PlanningException; -import org.apache.tajo.plan.verifier.LogicalPlanVerifier; -import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; -import org.apache.tajo.plan.verifier.VerificationState; -import org.apache.tajo.plan.verifier.VerifyException; -import org.apache.tajo.rpc.BlockingRpcServer; -import org.apache.tajo.util.NetUtils; - -import java.net.InetSocketAddress; - -public class JitVecTestServer extends AbstractService { - private static final Log LOG = LogFactory.getLog(JitVecTestServer.class); - - TajoTestingCluster testingCluster; - - private TajoConf conf; - private SQLAnalyzer analyzer; - private CatalogService catalog; - private PreLogicalPlanVerifier preVerifier; - private LogicalPlanner planner; - private LogicalOptimizer optimizer; - private LogicalPlanVerifier annotatedPlanVerifier; - - private BlockingRpcServer rpcServer; - private JitVecTestServerProtocol.JitVecTestServerProtocolService.BlockingInterface handler; - - public JitVecTestServer() { - super(JitVecTestServer.class.getSimpleName()); - testingCluster = TpchTestBase.getInstance().getTestingCluster(); - this.conf = testingCluster.getConfiguration(); - catalog = testingCluster.getMaster().getCatalog(); - - analyzer = new SQLAnalyzer(); - preVerifier = new PreLogicalPlanVerifier(catalog); - planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); - annotatedPlanVerifier = new LogicalPlanVerifier(conf, catalog); - - handler = new ProtocolHandler(); - } - - @Override - public void serviceInit(Configuration conf) throws Exception { - rpcServer = new BlockingRpcServer(JitVecTestServerProtocol.class, handler, - new InetSocketAddress("0.0.0.0", 30060), 1); - - super.serviceInit(conf); - } - - @Override - public void serviceStart() throws Exception { - rpcServer.start(); - - super.serviceStart(); - - InetSocketAddress bindAddr = rpcServer.getListenAddress(); - System.out.println("\n============================================================================="); - System.out.println("Test Server Addr: " + NetUtils.normalizeInetSocketAddress(bindAddr)); - System.out.println("HDFS Namenode Addr: " + testingCluster.getMiniDFSCluster().getFileSystem().getUri()); - System.out.println("Catalog Server Addr: " + - NetUtils.normalizeInetSocketAddress(testingCluster.getMaster().getCatalogServer().getBindAddress())); - System.out.println("=============================================================================\n"); - } - - public void serviceStop() throws Exception { - rpcServer.shutdown(); - - super.serviceStop(); - } - - public class ProtocolHandler implements JitVecTestServerProtocol.JitVecTestServerProtocolService.BlockingInterface { - @Override - public JitVecTestServerProtocol.PlanResponse requestPlan(RpcController controller, - JitVecTestServerProtocol.RequestPlan request) throws ServiceException { - - JitVecTestServerProtocol.PlanResponse.Builder builder = JitVecTestServerProtocol.PlanResponse.newBuilder(); - - Session session = new Session("00", "tajo", "default"); - QueryContext queryContext = new QueryContext(conf, session); - - try { - LOG.info("Request is received: " + request.getSql()); - Expr expr = analyzer.parse(request.getSql()); - VerificationState state = new VerificationState(); - preVerifier.verify(queryContext, state, expr); - if (!state.verified()) { - StringBuilder sb = new StringBuilder(); - for (String error : state.getErrorMessages()) { - sb.append(error).append("\n"); - } - throw new VerifyException(sb.toString()); - } - - LogicalPlan plan = planner.createPlan(queryContext, expr); - if (LOG.isDebugEnabled()) { - LOG.debug("============================================="); - LOG.debug("Non Optimized Query: \n" + plan.toString()); - LOG.debug("============================================="); - } - LOG.info("Non Optimized Query: \n" + plan.toString()); - optimizer.optimize(queryContext, plan); - LOG.info("============================================="); - LOG.info("Optimized Query: \n" + plan.toString()); - LOG.info("============================================="); - - annotatedPlanVerifier.verify(queryContext, state, plan); - - if (!state.verified()) { - StringBuilder sb = new StringBuilder(); - for (String error : state.getErrorMessages()) { - sb.append(error).append("\n"); - } - throw new VerifyException(sb.toString()); - } - - builder.setSerializedPlan(plan.getRootBlock().getRoot().toJson()); - - System.out.println("\n\n======================================================================\n\n"); - System.out.println(CoreGsonHelper.getPrettyInstance().toJson(plan.getRootBlock().getRoot())); - System.out.println("\n\n======================================================================\n\n"); - } catch (PlanningException e) { - if (e.getMessage() != null) { - LOG.error(e.getMessage()); - builder.setErrorMessage(e.getMessage()); - } else { - e.printStackTrace(); - builder.setErrorMessage("Internal Error"); - } - } - - return builder.build(); - } - } - - public static void startServer(String [] args) throws Exception { - TajoConf conf = new TajoConf(); - JitVecTestServer server = new JitVecTestServer(); - server.init(conf); - server.start(); - } - - public static void main(String [] args) throws Exception { - startServer(args); - } -} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 750e64e48c..82078f9d62 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -28,6 +28,7 @@ import org.apache.tajo.algebra.JoinType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.util.ReflectionUtil; import org.apache.tajo.util.graph.DirectedGraphCursor; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.expr.EvalNode; @@ -37,14 +38,10 @@ import org.apache.tajo.plan.joinorder.JoinOrderAlgorithm; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.*; -import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule; -import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; -import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; import java.util.Stack; @@ -58,36 +55,22 @@ public class LogicalOptimizer { private static final Log LOG = LogFactory.getLog(LogicalOptimizer.class.getName()); - private BasicQueryRewriteEngine rulesBeforeJoinOpt; - private BasicQueryRewriteEngine rulesAfterToJoinOpt; + private BaseLogicalPlanRewriteEngine rulesBeforeJoinOpt; + private BaseLogicalPlanRewriteEngine rulesAfterToJoinOpt; private JoinOrderAlgorithm joinOrderAlgorithm = new GreedyHeuristicJoinOrderAlgorithm(); - public LogicalOptimizer(TajoConf systemConf) { - rulesBeforeJoinOpt = new BasicQueryRewriteEngine(); - if (systemConf.getBoolVar(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED)) { - rulesBeforeJoinOpt.addRewriteRule(new FilterPushDownRule()); - } + public LogicalOptimizer(TajoConf conf) { - rulesAfterToJoinOpt = new BasicQueryRewriteEngine(); - rulesAfterToJoinOpt.addRewriteRule(new ProjectionPushDownRule()); - rulesAfterToJoinOpt.addRewriteRule(new PartitionedTableRewriter(systemConf)); - - // Currently, it is only used for some test cases to inject exception manually. - String userDefinedRewriterClass = systemConf.get("tajo.plan.rewriter.classes"); - if (userDefinedRewriterClass != null && !userDefinedRewriterClass.isEmpty()) { - for (String eachRewriterClass : userDefinedRewriterClass.split(",")) { - try { - RewriteRule rule = (RewriteRule) Class.forName(eachRewriterClass).newInstance(); - rulesAfterToJoinOpt.addRewriteRule(rule); - } catch (Exception e) { - LOG.error("Can't initiate a Rewriter object: " + eachRewriterClass, e); - continue; - } - } - } + Class clazz = conf.getClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); + LogicalPlanRewriteRuleProvider provider = (LogicalPlanRewriteRuleProvider) ReflectionUtil.newInstance(clazz, conf); + + rulesBeforeJoinOpt = new BaseLogicalPlanRewriteEngine(); + rulesBeforeJoinOpt.addRewriteRule(provider.getPreRules()); + rulesAfterToJoinOpt = new BaseLogicalPlanRewriteEngine(); + rulesAfterToJoinOpt.addRewriteRule(provider.getPostRules()); } - public void addRuleAfterToJoinOpt(RewriteRule rewriteRule) { + public void addRuleAfterToJoinOpt(LogicalPlanRewriteRule rewriteRule) { if (rewriteRule != null) { rulesAfterToJoinOpt.addRewriteRule(rewriteRule); } @@ -98,7 +81,7 @@ public LogicalNode optimize(LogicalPlan plan) throws PlanningException { } public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws PlanningException { - rulesBeforeJoinOpt.rewrite(plan); + rulesBeforeJoinOpt.rewrite(context, plan); DirectedGraphCursor blockCursor = new DirectedGraphCursor(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); @@ -111,7 +94,7 @@ public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws Pl } else { LOG.info("Skip Join Optimized."); } - rulesAfterToJoinOpt.rewrite(plan); + rulesAfterToJoinOpt.rewrite(context, plan); return plan.getRootBlock().getRoot(); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 55ca7651aa..7989a4ef70 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -153,10 +153,6 @@ public LogicalPlan createPlan(OverridableConf queryContext, Expr expr, boolean d root.setOutSchema(topMostNode.getOutSchema()); plan.getRootBlock().setRoot(root); - PlanProto.LogicalNodeTree serialized = LogicalNodeTreeSerializer.serialize(root); - LogicalNode deserialized = LogicalNodeTreeDeserializer.deserialize(queryContext, serialized); - assert root.deepEquals(deserialized); - return plan; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BasicQueryRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java similarity index 63% rename from tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BasicQueryRewriteEngine.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java index 491dda1be1..19c254b9a5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BasicQueryRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java @@ -20,6 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; @@ -31,19 +32,35 @@ * This is a basic query rewrite rule engine. This rewrite rule engine * rewrites a logical plan with various query rewrite rules. */ -public class BasicQueryRewriteEngine implements QueryRewriteEngine { +public class BaseLogicalPlanRewriteEngine implements LogicalPlanRewriteEngine { /** class logger */ - private Log LOG = LogFactory.getLog(BasicQueryRewriteEngine.class); + private Log LOG = LogFactory.getLog(BaseLogicalPlanRewriteEngine.class); /** a map for query rewrite rules */ - private Map rewriteRules = new LinkedHashMap(); + private Map rewriteRules = new LinkedHashMap(); + + /** + * Add a query rewrite rule to this engine. + * + * @param rules Rule classes + */ + public void addRewriteRule(Iterable> rules) { + for (Class clazz : rules) { + try { + LogicalPlanRewriteRule rule = clazz.newInstance(); + addRewriteRule(rule); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + } /** * Add a query rewrite rule to this engine. * * @param rule The rule to be added to this engine. */ - public void addRewriteRule(RewriteRule rule) { + public void addRewriteRule(LogicalPlanRewriteRule rule) { if (!rewriteRules.containsKey(rule.getName())) { rewriteRules.put(rule.getName(), rule); } @@ -55,12 +72,12 @@ public void addRewriteRule(RewriteRule rule) { * @param plan The plan to be rewritten with all query rewrite rule. * @return The rewritten plan. */ - public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { - RewriteRule rule; - for (Entry rewriteRule : rewriteRules.entrySet()) { + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + LogicalPlanRewriteRule rule; + for (Entry rewriteRule : rewriteRules.entrySet()) { rule = rewriteRule.getValue(); - if (rule.isEligible(plan)) { - plan = rule.rewrite(plan); + if (rule.isEligible(queryContext, plan)) { + plan = rule.rewrite(queryContext, plan); if (LOG.isDebugEnabled()) { LOG.debug("The rule \"" + rule.getName() + " \" rewrites the query."); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java new file mode 100644 index 0000000000..eb96149447 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule; +import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; +import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; +import org.apache.tajo.util.TUtil; + +import java.util.Collection; +import java.util.List; + +/** + * Default RewriteRuleProvider + */ +@SuppressWarnings("unused") +public class BaseLogicalPlanRewriteRuleProvider extends LogicalPlanRewriteRuleProvider { + + public BaseLogicalPlanRewriteRuleProvider(TajoConf conf) { + super(conf); + } + + @Override + public Collection> getPreRules() { + List> rules = TUtil.newList(); + + if (systemConf.getBoolVar(TajoConf.ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED)) { + rules.add(FilterPushDownRule.class); + } + + return rules; + } + + @Override + public Collection> getPostRules() { + List> rules = TUtil.newList( + ProjectionPushDownRule.class, + PartitionedTableRewriter.class + ); + return rules; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/QueryRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java similarity index 86% rename from tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/QueryRewriteEngine.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java index b7f5637d84..267d65134c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/QueryRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java @@ -18,15 +18,16 @@ package org.apache.tajo.plan.rewrite; +import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; -public interface QueryRewriteEngine { +public interface LogicalPlanRewriteEngine { /** * Rewrite a logical plan with all query rewrite rules added to this engine. * * @param plan The plan to be rewritten with all query rewrite rule. * @return The rewritten plan. */ - LogicalPlan rewrite(LogicalPlan plan) throws PlanningException; + LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/RewriteRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java similarity index 88% rename from tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/RewriteRule.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java index 0ba746027e..2f0652bd2b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/RewriteRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java @@ -18,13 +18,14 @@ package org.apache.tajo.plan.rewrite; +import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; /** * An interface for a rewrite rule. */ -public interface RewriteRule { +public interface LogicalPlanRewriteRule { /** * It returns the rewrite rule name. It will be used for debugging and @@ -42,7 +43,7 @@ public interface RewriteRule { * @param plan The plan to be checked * @return True if this rule can be applied to a given plan. Otherwise, false. */ - boolean isEligible(LogicalPlan plan); + boolean isEligible(OverridableConf queryContext, LogicalPlan plan); /** * Updates a logical plan and returns an updated logical plan rewritten by this rule. @@ -52,5 +53,5 @@ public interface RewriteRule { * @param plan Input logical plan. It will not be modified. * @return The rewritten logical plan. */ - LogicalPlan rewrite(LogicalPlan plan) throws PlanningException; + LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleProvider.java new file mode 100644 index 0000000000..934549e3d5 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleProvider.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.conf.TajoConf; + +import java.util.Collection; + +public abstract class LogicalPlanRewriteRuleProvider { + protected final TajoConf systemConf; + + public LogicalPlanRewriteRuleProvider(TajoConf systemConf) { + this.systemConf = systemConf; + } + + /** + * It returns RewriteRule classes which should be executed before join ordering. + * + * @return RewriteRule classes + */ + public abstract Collection> getPreRules(); + /** + * It returns RewriteRule classes which should be executed after join ordering. + * + * @return RewriteRule classes + */ + public abstract Collection> getPostRules(); +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanTestRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanTestRuleProvider.java new file mode 100644 index 0000000000..704e7edd54 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanTestRuleProvider.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import com.google.common.collect.Lists; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.plan.rewrite.rules.LogicalPlanEqualityTester; + +import java.util.Collection; +import java.util.List; + +/** + * It is used only for testing. + */ +@SuppressWarnings("unused") +public class LogicalPlanTestRuleProvider extends BaseLogicalPlanRewriteRuleProvider { + + public LogicalPlanTestRuleProvider(TajoConf conf) { + super(conf); + } + + @Override + public Collection> getPostRules() { + List> injectedRules = Lists.newArrayList(super.getPostRules()); + injectedRules.add(LogicalPlanEqualityTester.class); + return injectedRules; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index ed410f900b..15750a148c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -21,6 +21,7 @@ import com.google.common.collect.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.OverridableConf; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; @@ -30,7 +31,7 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.TUtil; @@ -42,7 +43,7 @@ * It is likely to significantly reduces the intermediate data. */ public class FilterPushDownRule extends BasicLogicalPlanVisitor - implements RewriteRule { + implements LogicalPlanRewriteRule { private final static Log LOG = LogFactory.getLog(FilterPushDownRule.class); private static final String NAME = "FilterPushDown"; @@ -79,7 +80,7 @@ public String getName() { } @Override - public boolean isEligible(LogicalPlan plan) { + public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { return true; @@ -89,7 +90,7 @@ public boolean isEligible(LogicalPlan plan) { } @Override - public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { /* FilterPushDown rule: processing when visits each node - If a target which is corresponding on a filter EvalNode's column is not FieldEval, do not PushDown. diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java new file mode 100644 index 0000000000..b7dfbbff52 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.tajo.OverridableConf; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.PlanProto; + +/** + * It verifies the equality between the input and output of LogicalNodeTree(De)Serializer in logical planning. + * It is used only for testing. + */ +@SuppressWarnings("unused") +public class LogicalPlanEqualityTester implements LogicalPlanRewriteRule { + + @Override + public String getName() { + return "LogicalPlanEqualityTester"; + } + + @Override + public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { + return true; + } + + @Override + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + LogicalNode root = plan.getRootBlock().getRoot(); + PlanProto.LogicalNodeTree serialized = LogicalNodeTreeSerializer.serialize(plan.getRootBlock().getRoot()); + LogicalNode deserialized = LogicalNodeTreeDeserializer.deserialize(queryContext, serialized); + assert root.deepEquals(deserialized); + return plan; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index ea58437a6d..7604c53d9a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -23,15 +23,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.*; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.expr.*; @@ -46,25 +46,19 @@ import java.util.Set; import java.util.Stack; -public class PartitionedTableRewriter implements RewriteRule { +public class PartitionedTableRewriter implements LogicalPlanRewriteRule { private static final Log LOG = LogFactory.getLog(PartitionedTableRewriter.class); private static final String NAME = "Partitioned Table Rewriter"; private final Rewriter rewriter = new Rewriter(); - private final TajoConf systemConf; - - public PartitionedTableRewriter(TajoConf conf) { - systemConf = conf; - } - @Override public String getName() { return NAME; } @Override - public boolean isEligible(LogicalPlan plan) { + public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { for (RelationNode relation : block.getRelations()) { if (relation.getType() == NodeType.SCAN) { @@ -79,9 +73,9 @@ public boolean isEligible(LogicalPlan plan) { } @Override - public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); - rewriter.visit(rootBlock, plan, rootBlock, rootBlock.getRoot(), new Stack()); + rewriter.visit(queryContext, plan, rootBlock, rootBlock.getRoot(), new Stack()); return plan; } @@ -120,10 +114,11 @@ public String toString() { * @return * @throws IOException */ - private Path [] findFilteredPaths(Schema partitionColumns, EvalNode [] conjunctiveForms, Path tablePath) + private Path [] findFilteredPaths(OverridableConf queryContext, Schema partitionColumns, EvalNode [] conjunctiveForms, + Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(systemConf); + FileSystem fs = tablePath.getFileSystem(queryContext.getConf()); PathFilter [] filters; if (conjunctiveForms == null) { @@ -223,7 +218,7 @@ public String toString() { return paths; } - private Path [] findFilteredPartitionPaths(ScanNode scanNode) throws IOException { + private Path [] findFilteredPartitionPaths(OverridableConf queryContext, ScanNode scanNode) throws IOException { TableDesc table = scanNode.getTableDesc(); PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod(); @@ -262,10 +257,10 @@ public String toString() { } if (indexablePredicateSet.size() > 0) { // There are at least one indexable predicates - return findFilteredPaths(paritionValuesSchema, + return findFilteredPaths(queryContext, paritionValuesSchema, indexablePredicateSet.toArray(new EvalNode[indexablePredicateSet.size()]), new Path(table.getPath())); } else { // otherwise, we will get all partition paths. - return findFilteredPaths(paritionValuesSchema, null, new Path(table.getPath())); + return findFilteredPaths(queryContext, paritionValuesSchema, null, new Path(table.getPath())); } } @@ -314,10 +309,11 @@ private boolean checkIfDisjunctiveButOneVariable(EvalNode evalNode) { } } - private void updateTableStat(PartitionedTableScanNode scanNode) throws PlanningException { + private void updateTableStat(OverridableConf queryContext, PartitionedTableScanNode scanNode) + throws PlanningException { if (scanNode.getInputPaths().length > 0) { try { - FileSystem fs = scanNode.getInputPaths()[0].getFileSystem(systemConf); + FileSystem fs = scanNode.getInputPaths()[0].getFileSystem(queryContext.getConf()); long totalVolume = 0; for (Path input : scanNode.getInputPaths()) { @@ -396,10 +392,10 @@ private static String getColumnPartitionPathPrefix(Schema partitionColumn) { return sb.toString(); } - private final class Rewriter extends BasicLogicalPlanVisitor { + private final class Rewriter extends BasicLogicalPlanVisitor { @Override - public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, - Stack stack) throws PlanningException { + public Object visitScan(OverridableConf queryContext, LogicalPlan plan, LogicalPlan.QueryBlock block, + ScanNode scanNode, Stack stack) throws PlanningException { TableDesc table = scanNode.getTableDesc(); if (!table.hasPartition()) { @@ -407,11 +403,11 @@ public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock } try { - Path [] filteredPaths = findFilteredPartitionPaths(scanNode); + Path [] filteredPaths = findFilteredPartitionPaths(queryContext, scanNode); plan.addHistory("PartitionTableRewriter chooses " + filteredPaths.length + " of partitions"); PartitionedTableScanNode rewrittenScanNode = plan.createNode(PartitionedTableScanNode.class); rewrittenScanNode.init(scanNode, filteredPaths); - updateTableStat(rewrittenScanNode); + updateTableStat(queryContext, rewrittenScanNode); // if it is topmost node, set it as the rootnode of this block. if (stack.empty() || block.getRoot().equals(scanNode)) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index 46bd0afb0d..abd2814ddf 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -21,6 +21,7 @@ import com.google.common.collect.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.OverridableConf; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; @@ -29,7 +30,7 @@ import org.apache.tajo.plan.LogicalPlan.QueryBlock; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; @@ -44,7 +45,7 @@ * It also enables scanners to read only necessary columns. */ public class ProjectionPushDownRule extends - BasicLogicalPlanVisitor implements RewriteRule { + BasicLogicalPlanVisitor implements LogicalPlanRewriteRule { /** Class Logger */ private final Log LOG = LogFactory.getLog(ProjectionPushDownRule.class); private static final String name = "ProjectionPushDown"; @@ -55,7 +56,7 @@ public String getName() { } @Override - public boolean isEligible(LogicalPlan plan) { + public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { LogicalNode toBeOptimized = plan.getRootBlock().getRoot(); if (PlannerUtil.checkIfDDLPlan(toBeOptimized)) { @@ -70,7 +71,7 @@ public boolean isEligible(LogicalPlan plan) { } @Override - public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); LogicalPlan.QueryBlock topmostBlock = rootBlock; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java index 609ca20376..c9f493daba 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -38,7 +38,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.TUtil; @@ -601,7 +601,7 @@ public void verifyInsertTableSchema(TableDesc tableDesc, Schema outSchema) throw * @return The list of storage specified rewrite rules * @throws java.io.IOException */ - public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { + public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { return null; } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java index 79161ccd62..e95aeecc93 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.hbase; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; @@ -30,12 +31,13 @@ import org.apache.tajo.plan.logical.SortNode; import org.apache.tajo.plan.logical.SortNode.SortPurpose; import org.apache.tajo.plan.logical.UnaryNode; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; -public class AddSortForInsertRewriter implements RewriteRule { +public class AddSortForInsertRewriter implements LogicalPlanRewriteRule { private int[] sortColumnIndexes; private Column[] sortColumns; + public AddSortForInsertRewriter(TableDesc tableDesc, Column[] sortColumns) { this.sortColumns = sortColumns; this.sortColumnIndexes = new int[sortColumns.length]; @@ -52,13 +54,13 @@ public String getName() { } @Override - public boolean isEligible(LogicalPlan plan) { + public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { StoreType storeType = PlannerUtil.getStoreType(plan); return storeType != null; } @Override - public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); UnaryNode insertNode = rootNode.getChild(); LogicalNode childNode = insertNode.getChild(); diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java index de4b4cbfe1..c606e88159 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java @@ -49,7 +49,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; @@ -1050,9 +1050,9 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc } } - public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { + public List getRewriteRules(OverridableConf queryContext, TableDesc tableDesc) throws IOException { if ("false".equalsIgnoreCase(queryContext.get(HBaseStorageConstants.INSERT_PUT_MODE, "false"))) { - List rules = new ArrayList(); + List rules = new ArrayList(); rules.add(new AddSortForInsertRewriter(tableDesc, getIndexColumns(tableDesc))); return rules; } else { From d4dfeefedb1f01826e4b278f22d65420d3094438 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 29 Dec 2014 17:15:44 +0900 Subject: [PATCH 25/31] Fixed test failures. --- .../java/org/apache/tajo/catalog/Schema.java | 4 + .../java/org/apache/tajo/util/ProtoUtil.java | 12 +++ .../apache/tajo/engine/query/TaskRequest.java | 4 +- .../tajo/engine/query/TaskRequestImpl.java | 28 +++--- .../tajo/master/DefaultTaskScheduler.java | 12 +-- .../java/org/apache/tajo/worker/Task.java | 3 +- .../src/main/proto/TajoWorkerProtocol.proto | 2 +- .../apache/tajo/engine/eval/ExprTestBase.java | 2 +- .../tajo/engine/query/TestGroupByQuery.java | 59 +++++++++---- .../apache/tajo/master/TestGlobalPlanner.java | 6 +- .../apache/tajo/plan/LogicalOptimizer.java | 7 +- .../expr/AggregationFunctionCallEval.java | 38 +++++++- .../tajo/plan/expr/WindowFunctionEval.java | 12 +++ .../apache/tajo/plan/logical/GroupbyNode.java | 1 + .../apache/tajo/plan/logical/WindowSpec.java | 57 ++++++++---- .../serder/EvalTreeProtoDeserializer.java | 87 ++++++++++++++++++- .../plan/serder/EvalTreeProtoSerializer.java | 86 +++++++++++++++++- .../serder/LogicalNodeTreeDeserializer.java | 1 + .../serder/LogicalNodeTreeSerializer.java | 58 ++++++------- tajo-plan/src/main/proto/Plan.proto | 66 ++++++++++++-- 20 files changed, 446 insertions(+), 99 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 672b8e34fe..5a74c63a44 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -199,6 +199,10 @@ private RuntimeException throwAmbiguousFieldException(Collection idList } public int getColumnId(String name) { + if (fieldsByQualifiedName.containsKey(name)) { + return fieldsByQualifiedName.get(name); + } + String [] parts = name.split("\\."); if (parts.length == 2 || parts.length == 3) { if (fieldsByQualifiedName.containsKey(name)) { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java index dbc987dd98..a5588377ef 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java @@ -18,7 +18,11 @@ package org.apache.tajo.util; +import com.google.common.collect.Lists; +import org.apache.tajo.common.ProtoObject; + import java.util.Collection; +import java.util.List; import java.util.Map; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.*; @@ -52,4 +56,12 @@ public static Map convertToMap(KeyValueSetProto proto) { public static KeyValueSetProto convertFromMap(Map map) { return new KeyValueSet(map).getProto(); } + + public static Iterable toProtoObjects(ProtoObject[] protoObjects) { + List converted = Lists.newArrayList(); + for (int i = 0; i < protoObjects.length; i++) { + converted.add((T) protoObjects[i].getProto()); + } + return converted; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java index a3e586a1c7..c7b80441bf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java @@ -28,6 +28,8 @@ import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.ipc.TajoWorkerProtocol; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.worker.FetchImpl; import java.util.List; @@ -38,7 +40,7 @@ public interface TaskRequest extends ProtoObject getFragments(); public String getOutputTableId(); public boolean isClusteredOutput(); - public String getSerializedData(); + public PlanProto.LogicalNodeTree getPlan(); public boolean isInterQuery(); public void setInterQuery(); public void addFetch(String name, FetchImpl fetch); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index cef5488c90..b4727dcff3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -25,6 +25,7 @@ import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProto; import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProtoOrBuilder; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.worker.FetchImpl; import java.util.ArrayList; @@ -39,7 +40,7 @@ public class TaskRequestImpl implements TaskRequest { private String outputTable; private boolean isUpdated; private boolean clusteredOutput; - private String serializedData; // logical node + private PlanProto.LogicalNodeTree plan; // logical node private Boolean interQuery; private List fetches; private Boolean shouldDie; @@ -59,9 +60,10 @@ public TaskRequestImpl() { public TaskRequestImpl(TaskAttemptId id, List fragments, String outputTable, boolean clusteredOutput, - String serializedData, QueryContext queryContext, DataChannel channel, Enforcer enforcer) { + PlanProto.LogicalNodeTree plan, QueryContext queryContext, DataChannel channel, + Enforcer enforcer) { this(); - this.set(id, fragments, outputTable, clusteredOutput, serializedData, queryContext, channel, enforcer); + this.set(id, fragments, outputTable, clusteredOutput, plan, queryContext, channel, enforcer); } public TaskRequestImpl(TaskRequestProto proto) { @@ -73,12 +75,12 @@ public TaskRequestImpl(TaskRequestProto proto) { public void set(TaskAttemptId id, List fragments, String outputTable, boolean clusteredOutput, - String serializedData, QueryContext queryContext, DataChannel dataChannel, Enforcer enforcer) { + PlanProto.LogicalNodeTree plan, QueryContext queryContext, DataChannel dataChannel, Enforcer enforcer) { this.id = id; this.fragments = fragments; this.outputTable = outputTable; this.clusteredOutput = clusteredOutput; - this.serializedData = serializedData; + this.plan = plan; this.isUpdated = true; this.queryContext = queryContext; this.queryContext = queryContext; @@ -150,16 +152,16 @@ public boolean isClusteredOutput() { } @Override - public String getSerializedData() { + public PlanProto.LogicalNodeTree getPlan() { TaskRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.serializedData != null) { - return this.serializedData; + if (this.plan != null) { + return this.plan; } - if (!p.hasSerializedData()) { + if (!p.hasPlan()) { return null; } - this.serializedData = p.getSerializedData(); - return this.serializedData; + this.plan = p.getPlan(); + return this.plan; } public boolean isInterQuery() { @@ -292,8 +294,8 @@ private void mergeLocalToBuilder() { if (this.isUpdated) { builder.setClusteredOutput(this.clusteredOutput); } - if (this.serializedData != null) { - builder.setSerializedData(this.serializedData); + if (this.plan != null) { + builder.setPlan(this.plan); } if (this.interQuery != null) { builder.setInterQuery(this.interQuery); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java index dd6233cf3f..7685411131 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java @@ -35,13 +35,15 @@ import org.apache.tajo.engine.query.TaskRequestImpl; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.master.cluster.WorkerConnectionInfo; +import org.apache.tajo.master.container.TajoContainerId; import org.apache.tajo.master.event.*; import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext; import org.apache.tajo.master.event.TaskSchedulerEvent.EventType; +import org.apache.tajo.master.querymaster.Stage; import org.apache.tajo.master.querymaster.Task; import org.apache.tajo.master.querymaster.TaskAttempt; -import org.apache.tajo.master.querymaster.Stage; -import org.apache.tajo.master.container.TajoContainerId; +import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.storage.DataLocation; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; @@ -125,7 +127,7 @@ public void run() { builder.setId(NULL_ATTEMPT_ID.getProto()); builder.setShouldDie(true); builder.setOutputTable(""); - builder.setSerializedData(""); + builder.setPlan(PlanProto.LogicalNodeTree.newBuilder()); builder.setClusteredOutput(false); stopTaskRunnerReq = builder.build(); } @@ -838,7 +840,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { new ArrayList(task.getAllFragments()), "", false, - task.getLogicalPlan().toJson(), + LogicalNodeTreeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer()); if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) { @@ -894,7 +896,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { Lists.newArrayList(task.getAllFragments()), "", false, - task.getLogicalPlan().toJson(), + LogicalNodeTreeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer()); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 70a3202cff..7836009906 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -41,6 +41,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.master.cluster.WorkerConnectionInfo; +import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.engine.query.QueryContext; @@ -124,7 +125,7 @@ public Task(String taskRunnerId, this.context.setEnforcer(request.getEnforcer()); this.inputStats = new TableStats(); - plan = CoreGsonHelper.fromJson(request.getSerializedData(), LogicalNode.class); + plan = LogicalNodeTreeDeserializer.deserialize(queryContext, request.getPlan()); LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN); if (scanNode != null) { for (LogicalNode node : scanNode) { diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto index 5acbcd9ec7..f4871cdccd 100644 --- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto +++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto @@ -70,7 +70,7 @@ message TaskRequestProto { repeated FragmentProto fragments = 2; required string outputTable = 3; required bool clusteredOutput = 4; - required string serializedData = 5; + required LogicalNodeTree plan = 5; optional bool interQuery = 6 [default = false]; repeated FetchProto fetches = 7; optional bool shouldDie = 8; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 851d151765..c01771fc0b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -144,7 +144,7 @@ private static Target[] getRawTargets(QueryContext context, String query, boolea assertFalse(state.getErrorMessages().get(0), true); } LogicalPlan plan = planner.createPlan(context, expr, true); - optimizer.optimize(plan); + optimizer.optimize(context, plan); annotatedPlanVerifier.verify(context, state, plan); if (state.getErrorMessages().size() > 0) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index bfd1700f89..794c14fc4c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -344,55 +344,84 @@ public final void testDistinctAggregationWithUnion1() throws Exception { } @Test - public final void testDistinctAggregationCasebyCase() throws Exception { - ResultSet res; - + public final void testDistinctAggregationCasebyCase1() throws Exception { // one groupby, distinct, aggregation - res = executeFile("testDistinctAggregation_case1.sql"); + ResultSet res = executeFile("testDistinctAggregation_case1.sql"); assertResultSet(res, "testDistinctAggregation_case1.result"); res.close(); + } + @Test + public final void testDistinctAggregationCasebyCase2() throws Exception { // one groupby, two distinct, one aggregation - res = executeFile("testDistinctAggregation_case2.sql"); + ResultSet res = executeFile("testDistinctAggregation_case2.sql"); assertResultSet(res, "testDistinctAggregation_case2.result"); res.close(); + } + @Test + public final void testDistinctAggregationCasebyCase3() throws Exception { // one groupby, two distinct, two aggregation(no alias) - res = executeFile("testDistinctAggregation_case3.sql"); + ResultSet res = executeFile("testDistinctAggregation_case3.sql"); assertResultSet(res, "testDistinctAggregation_case3.result"); res.close(); + } + @Test + public final void testDistinctAggregationCasebyCase4() throws Exception { // two groupby, two distinct, two aggregation - res = executeFile("testDistinctAggregation_case4.sql"); + ResultSet res = executeFile("testDistinctAggregation_case4.sql"); assertResultSet(res, "testDistinctAggregation_case4.result"); res.close(); + } + @Test + public final void testDistinctAggregationCasebyCase5() throws Exception { // two groupby, two distinct, two aggregation with stage - res = executeFile("testDistinctAggregation_case5.sql"); + ResultSet res = executeFile("testDistinctAggregation_case5.sql"); assertResultSet(res, "testDistinctAggregation_case5.result"); res.close(); + } - res = executeFile("testDistinctAggregation_case6.sql"); + @Test + public final void testDistinctAggregationCasebyCase6() throws Exception { + ResultSet res = executeFile("testDistinctAggregation_case6.sql"); assertResultSet(res, "testDistinctAggregation_case6.result"); res.close(); + } - res = executeFile("testDistinctAggregation_case7.sql"); + @Test + public final void testDistinctAggregationCasebyCase7() throws Exception { + ResultSet res = executeFile("testDistinctAggregation_case7.sql"); assertResultSet(res, "testDistinctAggregation_case7.result"); res.close(); + } - res = executeFile("testDistinctAggregation_case8.sql"); + @Test + public final void testDistinctAggregationCasebyCase8() throws Exception { + ResultSet res = executeFile("testDistinctAggregation_case8.sql"); assertResultSet(res, "testDistinctAggregation_case8.result"); res.close(); + } - res = executeFile("testDistinctAggregation_case9.sql"); + @Test + public final void testDistinctAggregationCasebyCase9() throws Exception { + ResultSet res = executeFile("testDistinctAggregation_case9.sql"); assertResultSet(res, "testDistinctAggregation_case9.result"); res.close(); + } - res = executeFile("testDistinctAggregation_case10.sql"); + @Test + public final void testDistinctAggregationCasebyCase10() throws Exception { + ResultSet res = executeFile("testDistinctAggregation_case10.sql"); assertResultSet(res, "testDistinctAggregation_case10.result"); res.close(); + } + + @Test + public final void testDistinctAggregationCasebyCase11() throws Exception { + ResultSet res; - // case9 KeyValueSet tableOptions = new KeyValueSet(); tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); @@ -417,7 +446,7 @@ public final void testDistinctAggregationCasebyCase() throws Exception { assertEquals(expected, resultSetToString(res)); - // multiple distinct with expression + // multiple distinct with expression res = executeString( "select count(distinct code) + count(distinct qty) from table10" ); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java index c908737945..d0f7cf44a8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java @@ -126,9 +126,9 @@ public static void tearDown() { private MasterPlan buildPlan(String sql) throws PlanningException, IOException { Expr expr = sqlAnalyzer.parse(sql); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(util.getConfiguration()), expr); - optimizer.optimize(plan); - QueryContext context = new QueryContext(util.getConfiguration()); + QueryContext context = LocalTajoTestingUtility.createDummyContext(util.getConfiguration()); + LogicalPlan plan = planner.createPlan(context, expr); + optimizer.optimize(context, plan); MasterPlan masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), context, plan); globalPlanner.build(masterPlan); return masterPlan; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 82078f9d62..18a8859d19 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -18,11 +18,13 @@ package org.apache.tajo.plan; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.tajo.ConfigKey; import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; @@ -76,8 +78,11 @@ public void addRuleAfterToJoinOpt(LogicalPlanRewriteRule rewriteRule) { } } + @VisibleForTesting public LogicalNode optimize(LogicalPlan plan) throws PlanningException { - return optimize(null, plan); + OverridableConf conf = new OverridableConf(new TajoConf(), + ConfigKey.ConfigType.SESSION, ConfigKey.ConfigType.QUERY, ConfigKey.ConfigType.SYSTEM); + return optimize(conf, plan); } public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws PlanningException { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java index 542eae854b..ca8c1103a0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java @@ -27,13 +27,14 @@ import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; +import org.apache.tajo.util.TUtil; public class AggregationFunctionCallEval extends FunctionEval implements Cloneable { - @Expose protected AggFunction instance; @Expose boolean intermediatePhase = false; @Expose boolean finalPhase = true; @Expose String alias; + protected AggFunction instance; private Tuple params; protected AggregationFunctionCallEval(EvalType type, FunctionDesc desc, AggFunction instance, EvalNode[] givenArgs) { @@ -91,6 +92,10 @@ public DataType getValueType() { } } + public boolean hasAlias() { + return this.alias != null; + } + public void setAlias(String alias) { this.alias = alias; } public String getAlias() { return this.alias; } @@ -106,6 +111,22 @@ public Object clone() throws CloneNotSupportedException { return clone; } + public boolean isIntermediatePhase() { + return intermediatePhase; + } + + public void setIntermediatePhase(boolean flag) { + this.intermediatePhase = flag; + } + + public void setFinalPhase(boolean flag) { + this.finalPhase = flag; + } + + public boolean isFinalPhase() { + return finalPhase; + } + public void setFirstPhase() { this.finalPhase = false; this.intermediatePhase = false; @@ -120,4 +141,19 @@ public void setIntermediatePhase() { this.finalPhase = false; this.intermediatePhase = true; } + + public boolean equals(Object obj) { + if (obj instanceof AggregationFunctionCallEval) { + AggregationFunctionCallEval other = (AggregationFunctionCallEval) obj; + + boolean eq = super.equals(other); + eq &= instance.equals(other.instance); + eq &= intermediatePhase == other.intermediatePhase; + eq &= finalPhase == other.finalPhase; + eq &= TUtil.checkEquals(alias, other.alias); + return eq; + } + + return false; + } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java index 84b4a45d2f..0ff5927a56 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java @@ -86,6 +86,17 @@ public DataType getValueType() { return funcDesc.getReturnType(); } + public boolean equals(Object obj) { + if (obj instanceof WindowFunctionEval) { + WindowFunctionEval other = (WindowFunctionEval) obj; + boolean eq = TUtil.checkEquals(sortSpecs, other.sortSpecs); + eq &= TUtil.checkEquals(windowFrame, other.windowFrame); + return eq; + } else { + return false; + } + } + @Override public Object clone() throws CloneNotSupportedException { WindowFunctionEval windowFunctionEval = (WindowFunctionEval) super.clone(); @@ -95,6 +106,7 @@ public Object clone() throws CloneNotSupportedException { windowFunctionEval.sortSpecs[i] = (SortSpec) sortSpecs[i].clone(); } } + windowFunctionEval.windowFrame = windowFrame.clone(); return windowFunctionEval; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java index fd22d2bf27..4a18cb48f4 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java @@ -122,6 +122,7 @@ public boolean equals(Object obj) { if (obj instanceof GroupbyNode) { GroupbyNode other = (GroupbyNode) obj; boolean eq = super.equals(other); + eq = eq && isDistinct() == other.isDistinct(); eq = eq && TUtil.checkEquals(groupingKeys, other.groupingKeys); eq = eq && TUtil.checkEquals(aggrFunctions, other.aggrFunctions); eq = eq && TUtil.checkEquals(targets, other.targets); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java index 73f4e13ca5..cdae68ffa8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java @@ -73,7 +73,7 @@ public int hashCode() { return Objects.hashCode(windowName, partitionKeys, windowFrame); } - public static class WindowFrame { + public static class WindowFrame implements Cloneable { @Expose private WindowStartBound startBound; @Expose private WindowEndBound endBound; @Expose org.apache.tajo.algebra.WindowSpec.WindowFrameUnit unit; // TODO - to be supported @@ -83,12 +83,8 @@ public WindowFrame() { this.endBound = new WindowEndBound(WindowFrameEndBoundType.UNBOUNDED_FOLLOWING); } - public WindowFrame(WindowStartBound startBound) { - this.startBound = startBound; - } - public WindowFrame(WindowStartBound startBound, WindowEndBound endBound) { - this(startBound); + this.startBound = startBound; this.endBound = endBound; } @@ -120,21 +116,29 @@ public org.apache.tajo.algebra.WindowSpec.WindowFrameUnit getFrameUnit() { public boolean equals(Object obj) { if (obj instanceof WindowFrame) { WindowFrame another = (WindowFrame) obj; - return - TUtil.checkEquals(startBound, another.startBound) && - TUtil.checkEquals(endBound, another.endBound) && - TUtil.checkEquals(unit, another.unit); + boolean eq = TUtil.checkEquals(startBound, another.startBound); + eq &= TUtil.checkEquals(endBound, another.endBound); + eq &= TUtil.checkEquals(unit, another.unit); + return eq; } else { return false; } } + public WindowFrame clone() throws CloneNotSupportedException { + WindowFrame newFrame = (WindowFrame) super.clone(); + newFrame.startBound = startBound.clone(); + newFrame.endBound = endBound.clone(); + newFrame.unit = unit; + return newFrame; + } + public int hashCode() { return Objects.hashCode(startBound, endBound, unit); } } - public static class WindowStartBound { + public static class WindowStartBound implements Cloneable { @Expose private WindowFrameStartBoundType boundType; @Expose private EvalNode number; @@ -158,7 +162,9 @@ public EvalNode getNumber() { public boolean equals(Object obj) { if (obj instanceof WindowStartBound) { WindowStartBound other = (WindowStartBound) obj; - return boundType == other.boundType && number.equals(other.number); + boolean eq = boundType == other.boundType; + eq &= TUtil.checkEquals(number, other.number); + return eq; } else { return false; } @@ -168,9 +174,19 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hashCode(boundType, number); } + + @Override + public WindowStartBound clone() throws CloneNotSupportedException { + WindowStartBound newStartBound = (WindowStartBound) super.clone(); + newStartBound.boundType = boundType; + if (number != null) { + newStartBound.number = (EvalNode) number.clone(); + } + return newStartBound; + } } - public static class WindowEndBound { + public static class WindowEndBound implements Cloneable { @Expose private WindowFrameEndBoundType boundType; @Expose private EvalNode number; @@ -192,9 +208,11 @@ public EvalNode getNumber() { @Override public boolean equals(Object obj) { - if (obj instanceof WindowStartBound) { + if (obj instanceof WindowEndBound) { WindowEndBound other = (WindowEndBound) obj; - return boundType == other.boundType && number.equals(other.number); + boolean eq = boundType == other.boundType; + eq &= TUtil.checkEquals(number, other.number); + return eq; } else { return false; } @@ -204,5 +222,14 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hashCode(boundType, number); } + + public WindowEndBound clone() throws CloneNotSupportedException { + WindowEndBound newEndBound = (WindowEndBound) super.clone(); + newEndBound.boundType = boundType; + if (number != null) { + newEndBound.number = (EvalNode) number.clone(); + } + return newEndBound; + } } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java index 89b4fc0653..368e98ef1b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java @@ -21,8 +21,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.tajo.OverridableConf; +import org.apache.tajo.algebra.WindowSpec.WindowFrameEndBoundType; +import org.apache.tajo.algebra.WindowSpec.WindowFrameStartBoundType; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.*; @@ -30,6 +33,8 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.plan.logical.WindowSpec; +import org.apache.tajo.plan.serder.PlanProto.WinFunctionEvalSpec; import java.util.*; @@ -98,6 +103,25 @@ public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { case IN: current = new InEval(lhs, (RowConstantEval) rhs, binProto.getNegative()); break; + case LIKE: { + PlanProto.PatternMatchEvalSpec patternMatchProto = protoNode.getPatternMatch(); + current = new LikePredicateEval(binProto.getNegative(), lhs, (ConstEval) rhs, + patternMatchProto.getCaseSensitive()); + break; + } + case REGEX: { + PlanProto.PatternMatchEvalSpec patternMatchProto = protoNode.getPatternMatch(); + current = new RegexPredicateEval(binProto.getNegative(), lhs, (ConstEval) rhs, + patternMatchProto.getCaseSensitive()); + break; + } + case SIMILAR_TO: { + PlanProto.PatternMatchEvalSpec patternMatchProto = protoNode.getPatternMatch(); + current = new SimilarToPredicateEval(binProto.getNegative(), lhs, (ConstEval) rhs, + patternMatchProto.getCaseSensitive()); + break; + } + default: current = new BinaryEval(type, lhs, rhs); } @@ -155,12 +179,34 @@ public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { if (type == EvalType.FUNCTION) { GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); current = new GeneralFunctionEval(context, new FunctionDesc(funcProto.getFuncion()), instance, params); + } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { AggFunction instance = (AggFunction) funcDesc.newInstance(); if (type == EvalType.AGG_FUNCTION) { - current = new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + AggregationFunctionCallEval aggFunc = + new AggregationFunctionCallEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + + PlanProto.AggFunctionEvalSpec aggFunctionProto = protoNode.getAggFunction(); + aggFunc.setIntermediatePhase(aggFunctionProto.getIntermediatePhase()); + aggFunc.setFinalPhase(aggFunctionProto.getFinalPhase()); + if (aggFunctionProto.hasAlias()) { + aggFunc.setAlias(aggFunctionProto.getAlias()); + } + current = aggFunc; + } else { - current = new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params, null); + WinFunctionEvalSpec windowFuncProto = protoNode.getWinFunction(); + + WindowFunctionEval winFunc = + new WindowFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params, + convertWindowFrame(windowFuncProto.getWindowFrame())); + + if (windowFuncProto.getSortSpecCount() > 0) { + SortSpec[] sortSpecs = LogicalNodeTreeDeserializer.convertSortSpecs(windowFuncProto.getSortSpecList()); + winFunc.setSortSpecs(sortSpecs); + } + + current = winFunc; } } } catch (ClassNotFoundException cnfe) { @@ -178,6 +224,43 @@ public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { return current; } + private static WindowSpec.WindowFrame convertWindowFrame(WinFunctionEvalSpec.WindowFrame windowFrame) { + WindowFrameStartBoundType startBoundType = convertWindowStartBound(windowFrame.getStartBound().getBoundType()); + WindowSpec.WindowStartBound startBound = new WindowSpec.WindowStartBound(startBoundType); + + WindowFrameEndBoundType endBoundType = convertWindowEndBound(windowFrame.getEndBound().getBoundType()); + WindowSpec.WindowEndBound endBound = new WindowSpec.WindowEndBound(endBoundType); + + WindowSpec.WindowFrame frame = new WindowSpec.WindowFrame(startBound, endBound); + return frame; + } + + private static WindowFrameStartBoundType convertWindowStartBound( + WinFunctionEvalSpec.WindowFrameStartBoundType type) { + if (type == WinFunctionEvalSpec.WindowFrameStartBoundType.S_UNBOUNDED_PRECEDING) { + return WindowFrameStartBoundType.UNBOUNDED_PRECEDING; + } else if (type == WinFunctionEvalSpec.WindowFrameStartBoundType.S_CURRENT_ROW) { + return WindowFrameStartBoundType.CURRENT_ROW; + } else if (type == WinFunctionEvalSpec.WindowFrameStartBoundType.S_PRECEDING) { + return WindowFrameStartBoundType.PRECEDING; + } else { + throw new IllegalStateException("Unknown Window Start Bound type: " + type.name()); + } + } + + private static WindowFrameEndBoundType convertWindowEndBound( + WinFunctionEvalSpec.WindowFrameEndBoundType type) { + if (type == WinFunctionEvalSpec.WindowFrameEndBoundType.E_UNBOUNDED_FOLLOWING) { + return WindowFrameEndBoundType.UNBOUNDED_FOLLOWING; + } else if (type == WinFunctionEvalSpec.WindowFrameEndBoundType.E_CURRENT_ROW) { + return WindowFrameEndBoundType.CURRENT_ROW; + } else if (type == WinFunctionEvalSpec.WindowFrameEndBoundType.E_FOLLOWING) { + return WindowFrameEndBoundType.FOLLOWING; + } else { + throw new IllegalStateException("Unknown Window Start Bound type: " + type.name()); + } + } + public static Datum deserialize(PlanProto.Datum datum) { switch (datum.getType()) { case BOOLEAN: diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java index f03a2caf36..eb624afa26 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java @@ -21,9 +21,16 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; +import org.apache.tajo.algebra.WindowSpec.WindowFrameEndBoundType; +import org.apache.tajo.algebra.WindowSpec.WindowFrameStartBoundType; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.IntervalDatum; import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.WindowSpec; +import org.apache.tajo.plan.serder.PlanProto.WinFunctionEvalSpec; +import org.apache.tajo.plan.serder.PlanProto.WinFunctionEvalSpec.WindowFrame; +import org.apache.tajo.util.ProtoUtil; import java.util.Map; import java.util.Stack; @@ -129,6 +136,9 @@ public EvalNode visitBinaryEval(EvalTreeProtoBuilderContext context, StacktoProtoObjects + (winFunc.getSortSpecs())); + } + + windowFuncBuilder.setWindowFrame(buildWindowFrame(winFunc.getWindowFrame())); + builder.setWinFunction(windowFuncBuilder); + } + + context.treeBuilder.addNodes(builder); return function; } + private WindowFrame buildWindowFrame(WindowSpec.WindowFrame frame) { + WindowFrame.Builder windowFrameBuilder = WindowFrame.newBuilder(); + + WindowSpec.WindowStartBound startBound = frame.getStartBound(); + WindowSpec.WindowEndBound endBound = frame.getEndBound(); + + WinFunctionEvalSpec.WindowStartBound.Builder startBoundBuilder = WinFunctionEvalSpec.WindowStartBound.newBuilder(); + startBoundBuilder.setBoundType(convertStartBoundType(startBound.getBoundType())); + + WinFunctionEvalSpec.WindowEndBound.Builder endBoundBuilder = WinFunctionEvalSpec.WindowEndBound.newBuilder(); + endBoundBuilder.setBoundType(convertEndBoundType(endBound.getBoundType())); + + windowFrameBuilder.setStartBound(startBoundBuilder); + windowFrameBuilder.setEndBound(endBoundBuilder); + + return windowFrameBuilder.build(); + } + + private WinFunctionEvalSpec.WindowFrameStartBoundType convertStartBoundType(WindowFrameStartBoundType type) { + if (type == WindowFrameStartBoundType.UNBOUNDED_PRECEDING) { + return WinFunctionEvalSpec.WindowFrameStartBoundType.S_UNBOUNDED_PRECEDING; + } else if (type == WindowFrameStartBoundType.CURRENT_ROW) { + return WinFunctionEvalSpec.WindowFrameStartBoundType.S_CURRENT_ROW; + } else if (type == WindowFrameStartBoundType.PRECEDING) { + return WinFunctionEvalSpec.WindowFrameStartBoundType.S_PRECEDING; + } else { + throw new IllegalStateException("Unknown Window Start Bound type: " + type.name()); + } + } + + private WinFunctionEvalSpec.WindowFrameEndBoundType convertEndBoundType(WindowFrameEndBoundType type) { + if (type == WindowFrameEndBoundType.UNBOUNDED_FOLLOWING) { + return WinFunctionEvalSpec.WindowFrameEndBoundType.E_UNBOUNDED_FOLLOWING; + } else if (type == WindowFrameEndBoundType.CURRENT_ROW) { + return WinFunctionEvalSpec.WindowFrameEndBoundType.E_CURRENT_ROW; + } else if (type == WindowFrameEndBoundType.FOLLOWING) { + return WinFunctionEvalSpec.WindowFrameEndBoundType.E_FOLLOWING; + } else { + throw new IllegalStateException("Unknown Window End Bound type: " + type.name()); + } + } + public static PlanProto.Datum serialize(Datum datum) { PlanProto.Datum.Builder builder = PlanProto.Datum.newBuilder(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java index ed2e0ca7c7..521179df85 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java @@ -281,6 +281,7 @@ public static GroupbyNode convertGroupby(OverridableConf context, Map 0) { groupby.setGroupingColumns(convertColumns(groupbyProto.getGroupingKeysList())); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java index 928ba7b2f5..883b2600bd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeSerializer.java @@ -18,12 +18,10 @@ package org.apache.tajo.plan.serder; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.common.ProtoObject; import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; @@ -35,6 +33,7 @@ import org.apache.tajo.plan.serder.PlanProto.AlterTablespaceNode.SetLocation; import org.apache.tajo.plan.serder.PlanProto.LogicalNodeTree; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.util.ProtoUtil; import org.apache.tajo.util.TUtil; import java.util.List; @@ -50,9 +49,13 @@ public class LogicalNodeTreeSerializer extends BasicLogicalPlanVisitor()); + try { + instance.visit(context, null, null, node, new Stack()); + } catch (PlanningException e) { + throw new RuntimeException(e); + } return context.treeBuilder.build(); } @@ -124,7 +127,7 @@ public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, Log EvalExprNode exprEval, Stack stack) throws PlanningException { PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder(); exprEvalBuilder.addAllTargets( - LogicalNodeTreeSerializer.toProtoObjects(exprEval.getTargets())); + ProtoUtil.toProtoObjects(exprEval.getTargets())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, exprEval); nodeBuilder.setExprEval(exprEvalBuilder); @@ -142,7 +145,7 @@ public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, L PlanProto.ProjectionNode.Builder projectionBuilder = PlanProto.ProjectionNode.newBuilder(); projectionBuilder.setChildId(childIds[0]); projectionBuilder.addAllTargets( - LogicalNodeTreeSerializer.toProtoObjects(projection.getTargets())); + ProtoUtil.toProtoObjects(projection.getTargets())); projectionBuilder.setDistinct(projection.isDistinct()); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, projection); @@ -181,22 +184,22 @@ public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, Lo if (windowAgg.hasPartitionKeys()) { windowAggBuilder.addAllPartitionKeys( - LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getPartitionKeys())); + ProtoUtil.toProtoObjects(windowAgg.getPartitionKeys())); } if (windowAgg.hasAggFunctions()) { windowAggBuilder.addAllWindowFunctions( - LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getWindowFunctions())); + ProtoUtil.toProtoObjects(windowAgg.getWindowFunctions())); } windowAggBuilder.setDistinct(windowAgg.isDistinct()); if (windowAgg.hasSortSpecs()) { windowAggBuilder.addAllSortSpecs( - LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getSortSpecs())); + ProtoUtil.toProtoObjects(windowAgg.getSortSpecs())); } if (windowAgg.hasTargets()) { windowAggBuilder.addAllTargets( - LogicalNodeTreeSerializer.toProtoObjects(windowAgg.getTargets())); + ProtoUtil.toProtoObjects(windowAgg.getTargets())); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, windowAgg); @@ -259,17 +262,18 @@ private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, Gro PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); groupbyBuilder.setChildId(childIds[0]); + groupbyBuilder.setDistinct(node.isDistinct()); if (node.groupingKeyNum() > 0) { groupbyBuilder.addAllGroupingKeys( - LogicalNodeTreeSerializer.toProtoObjects(node.getGroupingColumns())); + ProtoUtil.toProtoObjects(node.getGroupingColumns())); } if (node.hasAggFunctions()) { groupbyBuilder.addAllAggFunctions( - LogicalNodeTreeSerializer.toProtoObjects(node.getAggFunctions())); + ProtoUtil.toProtoObjects(node.getAggFunctions())); } if (node.hasTargets()) { - groupbyBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(node.getTargets())); + groupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); @@ -296,14 +300,14 @@ public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan pl if (node.getGroupingColumns().length > 0) { distGroupbyBuilder.addAllGroupingKeys( - LogicalNodeTreeSerializer.toProtoObjects(node.getGroupingColumns())); + ProtoUtil.toProtoObjects(node.getGroupingColumns())); } if (node.getAggFunctions().length > 0) { distGroupbyBuilder.addAllAggFunctions( - LogicalNodeTreeSerializer.toProtoObjects(node.getAggFunctions())); + ProtoUtil.toProtoObjects(node.getAggFunctions())); } if (node.hasTargets()) { - distGroupbyBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(node.getTargets())); + distGroupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); } for (int cid : node.getResultColumnIds()) { distGroupbyBuilder.addResultId(cid); @@ -351,7 +355,7 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical if (join.hasTargets()) { joinBuilder.setExistsTargets(true); - joinBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(join.getTargets())); + joinBuilder.addAllTargets(ProtoUtil.toProtoObjects(join.getTargets())); } else { joinBuilder.setExistsTargets(false); } @@ -404,7 +408,7 @@ public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan) { if (scan.hasTargets()) { scanBuilder.setExistTargets(true); - scanBuilder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(scan.getTargets())); + scanBuilder.addAllTargets(ProtoUtil.toProtoObjects(scan.getTargets())); } else { scanBuilder.setExistTargets(false); } @@ -424,10 +428,12 @@ public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPl PlanProto.PartitionScanSpec.Builder partitionScan = PlanProto.PartitionScanSpec.newBuilder(); List pathStrs = TUtil.newList(); - for (Path p : node.getInputPaths()) { - pathStrs.add(p.toString()); + if (node.getInputPaths() != null) { + for (Path p : node.getInputPaths()) { + pathStrs.add(p.toString()); + } + partitionScan.addAllPaths(pathStrs); } - partitionScan.addAllPaths(pathStrs); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setScan(scanBuilder); @@ -449,7 +455,7 @@ public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan builder.setTableName(node.getTableName()); if (node.hasTargets()) { - builder.addAllTargets(LogicalNodeTreeSerializer.toProtoObjects(node.getTargets())); + builder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); @@ -689,14 +695,6 @@ public static PlanProto.Target convertTarget(Target target) { return targetBuilder.build(); } - public static Iterable toProtoObjects(ProtoObject[] protoObjects) { - List converted = Lists.newArrayList(); - for (int i = 0; i < protoObjects.length; i++) { - converted.add((T) protoObjects[i].getProto()); - } - return converted; - } - private int [] registerGetChildIds(SerializeContext context, LogicalNode node) { int [] childIds = new int[node.childNum()]; for (int i = 0; i < node.childNum(); i++) { diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 30bd138adc..acd9443f39 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -126,9 +126,10 @@ message JoinNode { message GroupbyNode { required int32 childId = 1; - repeated ColumnProto groupingKeys = 2; - repeated EvalTree aggFunctions = 3; - repeated Target targets = 4; + required bool distinct = 2; + repeated ColumnProto groupingKeys = 3; + repeated EvalTree aggFunctions = 4; + repeated Target targets = 5; } message DistinctGroupbyNode { @@ -361,10 +362,13 @@ message EvalNode { optional ConstEval const = 6; optional ColumnProto field = 7; // field eval optional FunctionEval function = 8; - optional RowConstEval rowConst = 9; - optional BetweenEval between = 10; - optional CaseWhenEval casewhen = 11; - optional IfCondEval ifCond = 12; + optional AggFunctionEvalSpec aggFunction = 9; + optional WinFunctionEvalSpec winFunction = 10; + optional RowConstEval rowConst = 11; + optional BetweenEval between = 12; + optional CaseWhenEval casewhen = 13; + optional IfCondEval ifCond = 14; + optional PatternMatchEvalSpec patternMatch = 15; } message UnaryEval { @@ -380,6 +384,10 @@ message BinaryEval { optional bool negative = 3 [default = false]; } +message PatternMatchEvalSpec { // requires BinaryEval + optional bool caseSensitive = 1; +} + message BetweenEval { required int32 predicand = 1; required int32 begin = 2; @@ -411,6 +419,50 @@ message FunctionEval { repeated int32 paramIds = 2; } +message AggFunctionEvalSpec { // requires FunctionEval + required bool intermediatePhase = 1; + required bool finalPhase = 2; + optional string alias = 3; +} + +message WinFunctionEvalSpec { + message WindowFrame { + required WindowStartBound startBound = 1; + required WindowEndBound endBound = 2; + optional WindowFrameUnit unit = 3; + } + + enum WindowFrameStartBoundType { + S_UNBOUNDED_PRECEDING = 0; + S_CURRENT_ROW = 1; + S_PRECEDING = 2; + } + + enum WindowFrameEndBoundType { + E_UNBOUNDED_FOLLOWING = 0; + E_CURRENT_ROW = 1; + E_FOLLOWING = 2; + } + + enum WindowFrameUnit { + ROW = 0; + RANGE = 1; + } + + message WindowStartBound { + required WindowFrameStartBoundType boundType = 1; + optional EvalTree number = 2; + } + + message WindowEndBound { + required WindowFrameEndBoundType boundType = 1; + optional EvalTree number = 2; + } + + repeated SortSpecProto sortSpec = 1; + required WindowFrame windowFrame = 2; +} + message Datum { required Type type = 1; optional bool boolean = 2; From 49cded2ba029cd921aba3184abba1513b9888888 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 29 Dec 2014 19:29:31 +0900 Subject: [PATCH 26/31] Add more comments. --- .../java/org/apache/tajo/catalog/Schema.java | 2 ++ .../java/org/apache/tajo/conf/TajoConf.java | 2 ++ .../java/org/apache/tajo/util/ProtoUtil.java | 7 ++++++ .../org/apache/tajo/util/ReflectionUtil.java | 11 ++++++--- .../main/java/org/apache/tajo/util/TUtil.java | 8 +++++++ .../engine/planner/global/GlobalPlanner.java | 3 --- .../rewriter/GlobalPlanRewriteEngine.java | 1 - .../apache/tajo/engine/query/TaskRequest.java | 1 - .../java/org/apache/tajo/worker/Task.java | 1 - .../apache/tajo/engine/eval/ExprTestBase.java | 1 - .../org/apache/tajo/plan/LogicalPlanner.java | 3 --- .../plan/logical/AlterTablespaceNode.java | 3 --- .../plan/serder/EvalTreeProtoSerializer.java | 3 +++ .../serder/LogicalNodeTreeDeserializer.java | 2 -- .../apache/tajo/plan/util/PlannerUtil.java | 1 - .../tajo/plan/TestLogicalPlanConvertor.java | 23 ------------------- tajo-project/pom.xml | 6 ----- .../apache/tajo/rpc/ProtoPipelineFactory.java | 7 ++---- 18 files changed, 32 insertions(+), 53 deletions(-) delete mode 100644 tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalPlanConvertor.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 5a74c63a44..27127faa5c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -199,10 +199,12 @@ private RuntimeException throwAmbiguousFieldException(Collection idList } public int getColumnId(String name) { + // if the same column exists, immediately return that column. if (fieldsByQualifiedName.containsKey(name)) { return fieldsByQualifiedName.get(name); } + // The following is some workaround code. String [] parts = name.split("\\."); if (parts.length == 2 || parts.length == 3) { if (fieldsByQualifiedName.containsKey(name)) { diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 3dba9e0745..ab11ddddef 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -249,8 +249,10 @@ public static enum ConfVars implements ConfigKey { TASK_DEFAULT_SIZE("tajo.task.size-mb", 128), // Query and Optimization ------------------------------------------------- + // This class provides a ordered list of logical plan rewrite rule classes. LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS("tajo.plan.logical.rewriter.provider", "org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteRuleProvider"), + // This class provides a ordered list of global plan rewrite rule classes. GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS("tajo.plan.global.rewriter.provider", "org.apache.tajo.engine.planner.global.rewriter.BaseGlobalPlanRewriteRuleProvider"), EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1), diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java index a5588377ef..f9d759bed9 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java @@ -57,6 +57,13 @@ public static KeyValueSetProto convertFromMap(Map map) { return new KeyValueSet(map).getProto(); } + /** + * It converts an array of ProtoObjects into Iteratable one. + * + * @param protoObjects + * @param + * @return + */ public static Iterable toProtoObjects(ProtoObject[] protoObjects) { List converted = Lists.newArrayList(); for (int i = 0; i < protoObjects.length; i++) { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java index 5d34a9217d..e2def69c71 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java @@ -30,11 +30,16 @@ public class ReflectionUtil { private static final Class[] CONF_PARAM = new Class[]{TajoConf.class}; /** - * Cache of constructors for each class. Pins the classes so they + * Caches of constructors for each class. Pins the classes so they * can't be garbage collected until ReflectionUtils can be collected. + * + * EMPTY_CONSTRUCTOR_CACHE keeps classes which don't have any parameterized constructor, and + * CONF_CONSTRUCTOR_CACHE keeps classes which have one constructor to take TajoConf. */ - private static final Map, Constructor> EMPTY_CONSTRUCTOR_CACHE = new ConcurrentHashMap, Constructor>(); - private static final Map, Constructor> CONF_CONSTRUCTOR_CACHE = new ConcurrentHashMap, Constructor>(); + private static final Map, Constructor> EMPTY_CONSTRUCTOR_CACHE = + new ConcurrentHashMap, Constructor>(); + private static final Map, Constructor> CONF_CONSTRUCTOR_CACHE = + new ConcurrentHashMap, Constructor>(); /** * Initialize an instance by a given class diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index 3df448ca5e..a1de8602b6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -41,6 +41,14 @@ public static boolean checkEquals(Object s1, Object s2) { return Objects.equal(s1, s2); } + /** + * check two collections as equals. It also check the equivalence of null. + * It will return true even if they are all null. + * + * @param s1 the first collection to be compared. + * @param s2 the second collection to be compared + * @return true if they are equal or all null + */ public static boolean checkEquals(Collection s1, Collection s2) { if (s1 == null ^ s2 == null) { return false; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 53c5340049..6c3e3b8157 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -47,9 +47,6 @@ import org.apache.tajo.plan.function.AggFunction; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; -import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.KeyValueSet; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java index 6ea5de3d9e..c01ed0eb04 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.util.ReflectionUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java index c7b80441bf..2fa272a547 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java @@ -28,7 +28,6 @@ import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.ipc.TajoWorkerProtocol; -import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.worker.FetchImpl; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 7836009906..7e1f1f74ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -39,7 +39,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.master.cluster.WorkerConnectionInfo; import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; import org.apache.tajo.plan.util.PlannerUtil; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index c01771fc0b..9f00cfe5e1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -62,7 +62,6 @@ import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; -import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 7989a4ef70..74bc7a1824 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -46,9 +46,6 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; -import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.catalog.SchemaUtil; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java index 559b10cf0e..8b68dbec29 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java @@ -23,9 +23,6 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.plan.PlanString; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.LogicalNodeVisitor; -import org.apache.tajo.plan.logical.NodeType; public class AlterTablespaceNode extends LogicalNode implements Cloneable { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java index eb624afa26..42ce37a299 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java @@ -282,6 +282,8 @@ public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval builder.setAggFunction(aggFunctionEvalBuilder); } + + if (function instanceof WindowFunctionEval) { WindowFunctionEval winFunc = (WindowFunctionEval) function; WinFunctionEvalSpec.Builder windowFuncBuilder = WinFunctionEvalSpec.newBuilder(); @@ -295,6 +297,7 @@ public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval builder.setWinFunction(windowFuncBuilder); } + context.treeBuilder.addNodes(builder); return function; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java index 521179df85..0fc8cda1fd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java @@ -35,9 +35,7 @@ import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.expr.FieldEval; import org.apache.tajo.plan.expr.WindowFunctionEval; -import org.apache.tajo.plan.function.WindowAggFunc; import org.apache.tajo.plan.logical.*; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 36b9fa6243..d813432f7d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -47,7 +47,6 @@ public class PlannerUtil { public static final Column [] EMPTY_COLUMNS = new Column[] {}; - public static final Target [] EMPTY_TARGETS = new Target[] {}; public static final AggregationFunctionCallEval [] EMPTY_AGG_FUNCS = new AggregationFunctionCallEval[] {}; public static boolean checkIfSetSession(LogicalNode node) { diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalPlanConvertor.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalPlanConvertor.java deleted file mode 100644 index e738b39419..0000000000 --- a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalPlanConvertor.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.plan; - -public class TestLogicalPlanConvertor { - -} diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 59f25a0c7a..21df86348f 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -798,12 +798,6 @@ ${tajo.version} jar - - org.apache.tajo - tajo-jitvec-tests - ${tajo.version} - jar - org.apache.hadoop diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java index fdecfb660f..7aa03e787e 100644 --- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java +++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ProtoPipelineFactory.java @@ -23,8 +23,6 @@ import org.jboss.netty.channel.ChannelPipelineFactory; import org.jboss.netty.channel.ChannelUpstreamHandler; import org.jboss.netty.channel.Channels; -import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder; -import org.jboss.netty.handler.codec.frame.LengthFieldPrepender; import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder; import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder; import org.jboss.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; @@ -42,10 +40,9 @@ public ProtoPipelineFactory(ChannelUpstreamHandler handlerFactory, public ChannelPipeline getPipeline() throws Exception { ChannelPipeline p = Channels.pipeline(); - p.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(1048576*2, 0, 4, - 0, 4)); + p.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); p.addLast("protobufDecoder", new ProtobufDecoder(defaultInstance)); - p.addLast("frameEncoder", new LengthFieldPrepender(4)); + p.addLast("frameEncoder", new ProtobufVarint32LengthFieldPrepender()); p.addLast("protobufEncoder", new ProtobufEncoder()); p.addLast("handler", handler); return p; From 459b54a789ce7940c49adfd99307c1aac6023d91 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 29 Dec 2014 19:35:30 +0900 Subject: [PATCH 27/31] Fixed compilation error in jvm6. --- .../src/main/java/org/apache/tajo/plan/LogicalPlanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 74bc7a1824..eebee6f493 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1570,7 +1570,7 @@ private void buildProjectedInsert(PlanContext context, InsertNode insertNode) { } if (child instanceof Projectable) { - Projectable projectionNode = insertNode.getChild(); + Projectable projectionNode = (Projectable)insertNode.getChild(); // Modifying projected columns by adding NULL constants // It is because that table appender does not support target columns to be written. From b85f556e2b89db25c07802d1117c0397df969a77 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 29 Dec 2014 19:41:24 +0900 Subject: [PATCH 28/31] TAJO-269: Protocol buffer De/Serialization for LogicalNode. (hyunsik) --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index c35c24af8c..b5f3fe8129 100644 --- a/CHANGES +++ b/CHANGES @@ -27,6 +27,8 @@ Release 0.9.1 - unreleased IMPROVEMENT + TAJO-269: Protocol buffer De/Serialization for LogicalNode. (hyunsik) + TAJO-1266: Too many logs when writing a parquet relation. (DaeMyung Kang via jihoon) From ace5e4c26be998889e0602ba6cf61f9c365589db Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 30 Dec 2014 00:29:13 +0900 Subject: [PATCH 29/31] Add more comments and rename some classes. --- .../engine/planner/PhysicalPlannerImpl.java | 4 +- .../rules/GlobalPlanEqualityTester.java | 8 +- .../tajo/master/DefaultTaskScheduler.java | 6 +- .../java/org/apache/tajo/worker/Task.java | 4 +- .../apache/tajo/engine/eval/ExprTestBase.java | 8 +- .../java/org/apache/tajo/plan/Target.java | 4 +- .../org/apache/tajo/plan/expr/EvalNode.java | 8 +- .../rules/LogicalPlanEqualityTester.java | 8 +- ...ializer.java => EvalNodeDeserializer.java} | 10 +- ...erializer.java => EvalNodeSerializer.java} | 16 +-- ...izer.java => LogicalNodeDeserializer.java} | 105 +++++++++--------- ...alizer.java => LogicalNodeSerializer.java} | 39 +++++-- .../apache/tajo/plan/serder/package-info.java | 23 ++++ tajo-plan/src/main/proto/Plan.proto | 20 ++-- 14 files changed, 151 insertions(+), 112 deletions(-) rename tajo-plan/src/main/java/org/apache/tajo/plan/serder/{EvalTreeProtoDeserializer.java => EvalNodeDeserializer.java} (97%) rename tajo-plan/src/main/java/org/apache/tajo/plan/serder/{EvalTreeProtoSerializer.java => EvalNodeSerializer.java} (96%) rename tajo-plan/src/main/java/org/apache/tajo/plan/serder/{LogicalNodeTreeDeserializer.java => LogicalNodeDeserializer.java} (85%) rename tajo-plan/src/main/java/org/apache/tajo/plan/serder/{LogicalNodeTreeSerializer.java => LogicalNodeSerializer.java} (94%) create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/serder/package-info.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 90173d71ec..26b5d16ec5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -34,7 +34,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeDeserializer; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.physical.*; @@ -878,7 +878,7 @@ private boolean checkIfSortEquivalance(TaskAttemptContext ctx, ScanNode scanNode TajoWorkerProtocol.SortedInputEnforce sortEnforcer = property.get(0).getSortedInput(); boolean condition = scanNode.getTableName().equals(sortEnforcer.getTableName()); - SortSpec [] sortSpecs = LogicalNodeTreeDeserializer.convertSortSpecs(sortEnforcer.getSortSpecsList()); + SortSpec [] sortSpecs = LogicalNodeDeserializer.convertSortSpecs(sortEnforcer.getSortSpecsList()); return condition && TUtil.checkEquals(sortNode.getSortKeys(), sortSpecs); } else { return false; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java index 7124c13548..e2fd47f944 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java @@ -23,8 +23,8 @@ import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanRewriteRule; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.LogicalNodeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeSerializer; import org.apache.tajo.plan.serder.PlanProto; /** @@ -50,8 +50,8 @@ public MasterPlan rewrite(MasterPlan plan) { ExecutionBlock eb = cursor.nextBlock(); LogicalNode node = eb.getPlan(); if (node != null) { - PlanProto.LogicalNodeTree tree = LogicalNodeTreeSerializer.serialize(node); - LogicalNode deserialize = LogicalNodeTreeDeserializer.deserialize(plan.getContext(), tree); + PlanProto.LogicalNodeTree tree = LogicalNodeSerializer.serialize(node); + LogicalNode deserialize = LogicalNodeDeserializer.deserialize(plan.getContext(), tree); assert node.deepEquals(deserialize); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java index 7685411131..1cd6587c53 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java @@ -42,7 +42,7 @@ import org.apache.tajo.master.querymaster.Stage; import org.apache.tajo.master.querymaster.Task; import org.apache.tajo.master.querymaster.TaskAttempt; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.LogicalNodeSerializer; import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.storage.DataLocation; import org.apache.tajo.storage.fragment.FileFragment; @@ -840,7 +840,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { new ArrayList(task.getAllFragments()), "", false, - LogicalNodeTreeSerializer.serialize(task.getLogicalPlan()), + LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer()); if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) { @@ -896,7 +896,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { Lists.newArrayList(task.getAllFragments()), "", false, - LogicalNodeTreeSerializer.serialize(task.getLogicalPlan()), + LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer()); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 7e1f1f74ec..5f9c6ace1c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -40,7 +40,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.master.cluster.WorkerConnectionInfo; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeDeserializer; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.engine.query.QueryContext; @@ -124,7 +124,7 @@ public Task(String taskRunnerId, this.context.setEnforcer(request.getEnforcer()); this.inputStats = new TableStats(); - plan = LogicalNodeTreeDeserializer.deserialize(queryContext, request.getPlan()); + plan = LogicalNodeDeserializer.deserialize(queryContext, request.getPlan()); LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN); if (scanNode != null) { for (LogicalNode node : scanNode) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 9f00cfe5e1..4e4b710f28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -38,8 +38,8 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.serder.EvalTreeProtoDeserializer; -import org.apache.tajo.plan.serder.EvalTreeProtoSerializer; +import org.apache.tajo.plan.serder.EvalNodeDeserializer; +import org.apache.tajo.plan.serder.EvalNodeSerializer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.serder.PlanProto; @@ -320,7 +320,7 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S } public static void assertEvalTreeProtoSerDer(OverridableConf context, EvalNode evalNode) { - PlanProto.EvalTree converted = EvalTreeProtoSerializer.serialize(evalNode); - assertEquals(evalNode, EvalTreeProtoDeserializer.deserialize(context, converted)); + PlanProto.EvalNodeTree converted = EvalNodeSerializer.serialize(evalNode); + assertEquals(evalNode, EvalNodeDeserializer.deserialize(context, converted)); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java b/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java index b6c2e42593..a5c39b8d88 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java @@ -25,7 +25,7 @@ import org.apache.tajo.json.GsonObject; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.expr.FieldEval; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.LogicalNodeSerializer; import org.apache.tajo.plan.serder.PlanGsonHelper; import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.util.TUtil; @@ -132,6 +132,6 @@ public String toJson() { @Override public PlanProto.Target getProto() { - return LogicalNodeTreeSerializer.convertTarget(this); + return LogicalNodeSerializer.convertTarget(this); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java index 9f17d8a23a..dcb7285a33 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java @@ -24,7 +24,7 @@ import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.json.GsonObject; -import org.apache.tajo.plan.serder.EvalTreeProtoSerializer; +import org.apache.tajo.plan.serder.EvalNodeSerializer; import org.apache.tajo.plan.serder.PlanGsonHelper; import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.storage.Tuple; @@ -33,7 +33,7 @@ * An annotated expression which includes actual data domains. * It is also used for evaluation. */ -public abstract class EvalNode implements Cloneable, GsonObject, ProtoObject { +public abstract class EvalNode implements Cloneable, GsonObject, ProtoObject { @Expose protected EvalType type; public EvalNode() { @@ -76,7 +76,7 @@ public Object clone() throws CloneNotSupportedException { } @Override - public PlanProto.EvalTree getProto() { - return EvalTreeProtoSerializer.serialize(this); + public PlanProto.EvalNodeTree getProto() { + return EvalNodeSerializer.serialize(this); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java index b7dfbbff52..8a24add6c1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java @@ -23,8 +23,8 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; -import org.apache.tajo.plan.serder.LogicalNodeTreeDeserializer; -import org.apache.tajo.plan.serder.LogicalNodeTreeSerializer; +import org.apache.tajo.plan.serder.LogicalNodeDeserializer; +import org.apache.tajo.plan.serder.LogicalNodeSerializer; import org.apache.tajo.plan.serder.PlanProto; /** @@ -47,8 +47,8 @@ public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { @Override public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { LogicalNode root = plan.getRootBlock().getRoot(); - PlanProto.LogicalNodeTree serialized = LogicalNodeTreeSerializer.serialize(plan.getRootBlock().getRoot()); - LogicalNode deserialized = LogicalNodeTreeDeserializer.deserialize(queryContext, serialized); + PlanProto.LogicalNodeTree serialized = LogicalNodeSerializer.serialize(plan.getRootBlock().getRoot()); + LogicalNode deserialized = LogicalNodeDeserializer.deserialize(queryContext, serialized); assert root.deepEquals(deserialized); return plan; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeDeserializer.java similarity index 97% rename from tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeDeserializer.java index 368e98ef1b..322c8db82b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeDeserializer.java @@ -41,15 +41,15 @@ /** * It deserializes a serialized eval tree consisting of a number of EvalNodes. * - * {@link EvalTreeProtoSerializer} serializes an eval tree in a postfix traverse order. + * {@link EvalNodeSerializer} serializes an eval tree in a postfix traverse order. * So, this class firstly sorts all serialized eval nodes in ascending order of their sequence IDs. Then, * it sequentially restores each serialized node to EvalNode instance. * - * @see EvalTreeProtoSerializer + * @see EvalNodeSerializer */ -public class EvalTreeProtoDeserializer { +public class EvalNodeDeserializer { - public static EvalNode deserialize(OverridableConf context, PlanProto.EvalTree tree) { + public static EvalNode deserialize(OverridableConf context, PlanProto.EvalNodeTree tree) { Map evalNodeMap = Maps.newHashMap(); // sort serialized eval nodes in an ascending order of their IDs. @@ -202,7 +202,7 @@ public int compare(PlanProto.EvalNode o1, PlanProto.EvalNode o2) { convertWindowFrame(windowFuncProto.getWindowFrame())); if (windowFuncProto.getSortSpecCount() > 0) { - SortSpec[] sortSpecs = LogicalNodeTreeDeserializer.convertSortSpecs(windowFuncProto.getSortSpecList()); + SortSpec[] sortSpecs = LogicalNodeDeserializer.convertSortSpecs(windowFuncProto.getSortSpecList()); winFunc.setSortSpecs(sortSpecs); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java similarity index 96% rename from tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java index 42ce37a299..c7702c5117 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java @@ -40,24 +40,24 @@ * in a postfix traverse order. The postfix traverse order guarantees that all child nodes of some node N * were already visited when the node N is visited. This manner makes tree serialization possible in a simple logic. */ -public class EvalTreeProtoSerializer - extends SimpleEvalNodeVisitor { +public class EvalNodeSerializer + extends SimpleEvalNodeVisitor { - private static final EvalTreeProtoSerializer instance; + private static final EvalNodeSerializer instance; static { - instance = new EvalTreeProtoSerializer(); + instance = new EvalNodeSerializer(); } public static class EvalTreeProtoBuilderContext { private int seqId = 0; private Map idMap = Maps.newHashMap(); - private PlanProto.EvalTree.Builder treeBuilder = PlanProto.EvalTree.newBuilder(); + private PlanProto.EvalNodeTree.Builder treeBuilder = PlanProto.EvalNodeTree.newBuilder(); } - public static PlanProto.EvalTree serialize(EvalNode evalNode) { - EvalTreeProtoSerializer.EvalTreeProtoBuilderContext context = - new EvalTreeProtoSerializer.EvalTreeProtoBuilderContext(); + public static PlanProto.EvalNodeTree serialize(EvalNode evalNode) { + EvalNodeSerializer.EvalTreeProtoBuilderContext context = + new EvalNodeSerializer.EvalTreeProtoBuilderContext(); instance.visit(context, evalNode, new Stack()); return context.treeBuilder.build(); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java similarity index 85% rename from tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 0fc8cda1fd..98347d32ca 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeTreeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -41,13 +41,23 @@ import java.util.*; -public class LogicalNodeTreeDeserializer { - private static final LogicalNodeTreeDeserializer instance; +/** + * It deserializes a list of serialized logical nodes into a logical node tree. + */ +public class LogicalNodeDeserializer { + private static final LogicalNodeDeserializer instance; static { - instance = new LogicalNodeTreeDeserializer(); + instance = new LogicalNodeDeserializer(); } + /** + * Deserialize a list of nodes into a logical node tree. + * + * @param context QueryContext + * @param tree LogicalNodeTree which contains a list of serialized logical nodes. + * @return A logical node tree + */ public static LogicalNode deserialize(OverridableConf context, PlanProto.LogicalNodeTree tree) { Map nodeMap = Maps.newHashMap(); @@ -157,7 +167,7 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { return current; } - public static LogicalRootNode convertRoot(Map nodeMap, + private static LogicalRootNode convertRoot(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.RootNode rootProto = protoNode.getRoot(); @@ -173,7 +183,7 @@ public static LogicalRootNode convertRoot(Map nodeMap, return root; } - public static SetSessionNode convertSetSession(PlanProto.LogicalNode protoNode) { + private static SetSessionNode convertSetSession(PlanProto.LogicalNode protoNode) { PlanProto.SetSessionNode setSessionProto = protoNode.getSetSession(); SetSessionNode setSession = new SetSessionNode(protoNode.getPid()); @@ -182,7 +192,7 @@ public static SetSessionNode convertSetSession(PlanProto.LogicalNode protoNode) return setSession; } - public static EvalExprNode convertEvalExpr(OverridableConf context, PlanProto.LogicalNode protoNode) { + private static EvalExprNode convertEvalExpr(OverridableConf context, PlanProto.LogicalNode protoNode) { PlanProto.EvalExprNode evalExprProto = protoNode.getExprEval(); EvalExprNode evalExpr = new EvalExprNode(protoNode.getPid()); @@ -192,7 +202,7 @@ public static EvalExprNode convertEvalExpr(OverridableConf context, PlanProto.Lo return evalExpr; } - public static ProjectionNode convertProjection(OverridableConf context, Map nodeMap, + private static ProjectionNode convertProjection(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.ProjectionNode projectionProto = protoNode.getProjection(); @@ -205,7 +215,7 @@ public static ProjectionNode convertProjection(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { + private static LimitNode convertLimit(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.LimitNode limitProto = protoNode.getLimit(); LimitNode limitNode = new LimitNode(protoNode.getPid()); @@ -217,7 +227,7 @@ public static LimitNode convertLimit(Map nodeMap, PlanProt return limitNode; } - public static SortNode convertSort(Map nodeMap, PlanProto.LogicalNode protoNode) { + private static SortNode convertSort(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.SortNode sortProto = protoNode.getSort(); SortNode sortNode = new SortNode(protoNode.getPid()); @@ -229,20 +239,20 @@ public static SortNode convertSort(Map nodeMap, PlanProto. return sortNode; } - public static HavingNode convertHaving(OverridableConf context, Map nodeMap, + private static HavingNode convertHaving(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.FilterNode havingProto = protoNode.getFilter(); HavingNode having = new HavingNode(protoNode.getPid()); having.setChild(nodeMap.get(havingProto.getChildId())); - having.setQual(EvalTreeProtoDeserializer.deserialize(context, havingProto.getQual())); + having.setQual(EvalNodeDeserializer.deserialize(context, havingProto.getQual())); having.setInSchema(convertSchema(protoNode.getInSchema())); having.setOutSchema(convertSchema(protoNode.getOutSchema())); return having; } - public static WindowAggNode convertWindowAgg(OverridableConf context, Map nodeMap, + private static WindowAggNode convertWindowAgg(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.WindowAggNode windowAggProto = protoNode.getWindowAgg(); @@ -273,7 +283,7 @@ public static WindowAggNode convertWindowAgg(OverridableConf context, Map nodeMap, + private static GroupbyNode convertGroupby(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.GroupbyNode groupbyProto = protoNode.getGroupby(); @@ -297,7 +307,7 @@ public static GroupbyNode convertGroupby(OverridableConf context, Map nodeMap, + private static DistinctGroupbyNode convertDistinctGroupby(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.DistinctGroupbyNode distinctGroupbyProto = protoNode.getDistinctGroupby(); @@ -338,7 +348,7 @@ public static DistinctGroupbyNode convertDistinctGroupby(OverridableConf context return distinctGroupby; } - public static JoinNode convertJoin(OverridableConf context, Map nodeMap, + private static JoinNode convertJoin(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.JoinNode joinProto = protoNode.getJoin(); @@ -349,7 +359,7 @@ public static JoinNode convertJoin(OverridableConf context, Map nodeMap, + private static SelectionNode convertFilter(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.FilterNode filterProto = protoNode.getFilter(); @@ -366,12 +376,12 @@ public static SelectionNode convertFilter(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { + private static UnionNode convertUnion(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.UnionNode unionProto = protoNode.getUnion(); UnionNode union = new UnionNode(protoNode.getPid()); @@ -383,14 +393,14 @@ public static UnionNode convertUnion(Map nodeMap, PlanProt return union; } - public static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNode protoNode) { + private static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNode protoNode) { ScanNode scan = new ScanNode(protoNode.getPid()); fillScanNode(context, protoNode, scan); return scan; } - public static void fillScanNode(OverridableConf context, PlanProto.LogicalNode protoNode, ScanNode scan) { + private static void fillScanNode(OverridableConf context, PlanProto.LogicalNode protoNode, ScanNode scan) { PlanProto.ScanNode scanProto = protoNode.getScan(); if (scanProto.hasAlias()) { scan.init(new TableDesc(scanProto.getTable()), scanProto.getAlias()); @@ -403,14 +413,14 @@ public static void fillScanNode(OverridableConf context, PlanProto.LogicalNode p } if (scanProto.hasQual()) { - scan.setQual(EvalTreeProtoDeserializer.deserialize(context, scanProto.getQual())); + scan.setQual(EvalNodeDeserializer.deserialize(context, scanProto.getQual())); } scan.setInSchema(convertSchema(protoNode.getInSchema())); scan.setOutSchema(convertSchema(protoNode.getOutSchema())); } - public static PartitionedTableScanNode convertPartitionScan(OverridableConf context, PlanProto.LogicalNode protoNode) { + private static PartitionedTableScanNode convertPartitionScan(OverridableConf context, PlanProto.LogicalNode protoNode) { PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getPid()); fillScanNode(context, protoNode, partitionedScan); @@ -423,7 +433,7 @@ public static PartitionedTableScanNode convertPartitionScan(OverridableConf cont return partitionedScan; } - public static TableSubQueryNode convertTableSubQuery(OverridableConf context, + private static TableSubQueryNode convertTableSubQuery(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.TableSubQueryNode proto = protoNode.getTableSubQuery(); @@ -438,7 +448,7 @@ public static TableSubQueryNode convertTableSubQuery(OverridableConf context, return tableSubQuery; } - public static CreateTableNode convertCreateTable(Map nodeMap, + private static CreateTableNode convertCreateTable(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.PersistentStoreNode persistentStoreProto = protoNode.getPersistentStore(); PlanProto.StoreTableNodeSpec storeTableNodeSpec = protoNode.getStoreTable(); @@ -470,7 +480,7 @@ public static CreateTableNode convertCreateTable(Map nodeM return createTable; } - public static InsertNode convertInsert(Map nodeMap, + private static InsertNode convertInsert(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.PersistentStoreNode persistentStoreProto = protoNode.getPersistentStore(); PlanProto.StoreTableNodeSpec storeTableNodeSpec = protoNode.getStoreTable(); @@ -509,7 +519,7 @@ public static InsertNode convertInsert(Map nodeMap, return insertNode; } - public static DropTableNode convertDropTable(PlanProto.LogicalNode protoNode) { + private static DropTableNode convertDropTable(PlanProto.LogicalNode protoNode) { DropTableNode dropTable = new DropTableNode(protoNode.getPid()); PlanProto.DropTableNode dropTableProto = protoNode.getDropTable(); @@ -518,7 +528,7 @@ public static DropTableNode convertDropTable(PlanProto.LogicalNode protoNode) { return dropTable; } - public static CreateDatabaseNode convertCreateDatabase(PlanProto.LogicalNode protoNode) { + private static CreateDatabaseNode convertCreateDatabase(PlanProto.LogicalNode protoNode) { CreateDatabaseNode createDatabase = new CreateDatabaseNode(protoNode.getPid()); PlanProto.CreateDatabaseNode createDatabaseProto = protoNode.getCreateDatabase(); @@ -527,7 +537,7 @@ public static CreateDatabaseNode convertCreateDatabase(PlanProto.LogicalNode pro return createDatabase; } - public static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoNode) { + private static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoNode) { DropDatabaseNode dropDatabase = new DropDatabaseNode(protoNode.getPid()); PlanProto.DropDatabaseNode dropDatabaseProto = protoNode.getDropDatabase(); @@ -536,7 +546,7 @@ public static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoNo return dropDatabase; } - public static AlterTablespaceNode convertAlterTablespace(PlanProto.LogicalNode protoNode) { + private static AlterTablespaceNode convertAlterTablespace(PlanProto.LogicalNode protoNode) { AlterTablespaceNode alterTablespace = new AlterTablespaceNode(protoNode.getPid()); PlanProto.AlterTablespaceNode alterTablespaceProto = protoNode.getAlterTablespace(); @@ -553,7 +563,7 @@ public static AlterTablespaceNode convertAlterTablespace(PlanProto.LogicalNode p return alterTablespace; } - public static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) { + private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) { AlterTableNode alterTable = new AlterTableNode(protoNode.getPid()); PlanProto.AlterTableNode alterTableProto = protoNode.getAlterTable(); @@ -577,7 +587,7 @@ public static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) return alterTable; } - public static TruncateTableNode convertTruncateTable(PlanProto.LogicalNode protoNode) { + private static TruncateTableNode convertTruncateTable(PlanProto.LogicalNode protoNode) { TruncateTableNode truncateTable = new TruncateTableNode(protoNode.getPid()); PlanProto.TruncateTableNode truncateTableProto = protoNode.getTruncateTableNode(); @@ -586,37 +596,28 @@ public static TruncateTableNode convertTruncateTable(PlanProto.LogicalNode proto return truncateTable; } - public static Schema convertSchema(CatalogProtos.SchemaProto proto) { - return new Schema(proto); - } - - public static T [] convertEvalNodes(OverridableConf context, - List evalTrees) { - EvalNode [] evalNodes = new EvalNode[evalTrees.size()]; - for (int i = 0; i < evalNodes.length; i++) { - evalNodes[i] = EvalTreeProtoDeserializer.deserialize(context, evalTrees.get(i)); - } - return (T[]) evalNodes; - } - - public static AggregationFunctionCallEval [] convertAggFuncCallEvals(OverridableConf context, - List evalTrees) { + private static AggregationFunctionCallEval [] convertAggFuncCallEvals(OverridableConf context, + List evalTrees) { AggregationFunctionCallEval [] aggFuncs = new AggregationFunctionCallEval[evalTrees.size()]; for (int i = 0; i < aggFuncs.length; i++) { - aggFuncs[i] = (AggregationFunctionCallEval) EvalTreeProtoDeserializer.deserialize(context, evalTrees.get(i)); + aggFuncs[i] = (AggregationFunctionCallEval) EvalNodeDeserializer.deserialize(context, evalTrees.get(i)); } return aggFuncs; } - public static WindowFunctionEval[] convertWindowFunccEvals(OverridableConf context, - List evalTrees) { + private static WindowFunctionEval[] convertWindowFunccEvals(OverridableConf context, + List evalTrees) { WindowFunctionEval[] winFuncEvals = new WindowFunctionEval[evalTrees.size()]; for (int i = 0; i < winFuncEvals.length; i++) { - winFuncEvals[i] = (WindowFunctionEval) EvalTreeProtoDeserializer.deserialize(context, evalTrees.get(i)); + winFuncEvals[i] = (WindowFunctionEval) EvalNodeDeserializer.deserialize(context, evalTrees.get(i)); } return winFuncEvals; } + public static Schema convertSchema(CatalogProtos.SchemaProto proto) { + return new Schema(proto); + } + public static Column[] convertColumns(List columnProtos) { Column [] columns = new Column[columnProtos.size()]; for (int i = 0; i < columns.length; i++) { @@ -629,7 +630,7 @@ public static Target[] convertTargets(OverridableConf context, List { - private static final LogicalNodeTreeSerializer instance; + private static final LogicalNodeSerializer instance; static { - instance = new LogicalNodeTreeSerializer(); + instance = new LogicalNodeSerializer(); } + /** + * Serialize a logical plan into a protobuf-based serialized bytes. + * + * @param node LogicalNode to be serialized + * @return A list of serialized nodes + */ public static LogicalNodeTree serialize(LogicalNode node) { SerializeContext context = new SerializeContext(); try { @@ -59,7 +74,7 @@ public static LogicalNodeTree serialize(LogicalNode node) { return context.treeBuilder.build(); } - public static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext context, LogicalNode node) { + private static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext context, LogicalNode node) { int selfId; if (context.idMap.containsKey(node)) { selfId = context.idMap.get(node); @@ -189,7 +204,7 @@ public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, Lo if (windowAgg.hasAggFunctions()) { windowAggBuilder.addAllWindowFunctions( - ProtoUtil.toProtoObjects(windowAgg.getWindowFunctions())); + ProtoUtil.toProtoObjects(windowAgg.getWindowFunctions())); } windowAggBuilder.setDistinct(windowAgg.isDistinct()); @@ -238,7 +253,7 @@ public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, Logic PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); filterBuilder.setChildId(childIds[0]); - filterBuilder.setQual(EvalTreeProtoSerializer.serialize(having.getQual())); + filterBuilder.setQual(EvalNodeSerializer.serialize(having.getQual())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, having); nodeBuilder.setFilter(filterBuilder); @@ -270,7 +285,7 @@ private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, Gro } if (node.hasAggFunctions()) { groupbyBuilder.addAllAggFunctions( - ProtoUtil.toProtoObjects(node.getAggFunctions())); + ProtoUtil.toProtoObjects(node.getAggFunctions())); } if (node.hasTargets()) { groupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); @@ -304,7 +319,7 @@ public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan pl } if (node.getAggFunctions().length > 0) { distGroupbyBuilder.addAllAggFunctions( - ProtoUtil.toProtoObjects(node.getAggFunctions())); + ProtoUtil.toProtoObjects(node.getAggFunctions())); } if (node.hasTargets()) { distGroupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); @@ -329,7 +344,7 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); filterBuilder.setChildId(childIds[0]); - filterBuilder.setQual(EvalTreeProtoSerializer.serialize(filter.getQual())); + filterBuilder.setQual(EvalNodeSerializer.serialize(filter.getQual())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, filter); nodeBuilder.setFilter(filterBuilder); @@ -350,7 +365,7 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical joinBuilder.setLeftChildId(childIds[0]); joinBuilder.setRightChildId(childIds[1]); if (join.hasJoinQual()) { - joinBuilder.setJoinQual(EvalTreeProtoSerializer.serialize(join.getJoinQual())); + joinBuilder.setJoinQual(EvalNodeSerializer.serialize(join.getJoinQual())); } if (join.hasTargets()) { @@ -414,7 +429,7 @@ public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan) { } if (scan.hasQual()) { - scanBuilder.setQual(EvalTreeProtoSerializer.serialize(scan.getQual())); + scanBuilder.setQual(EvalNodeSerializer.serialize(scan.getQual())); } return scanBuilder; } @@ -688,7 +703,7 @@ public static PlanProto.JoinType convertJoinType(JoinType type) { public static PlanProto.Target convertTarget(Target target) { PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); - targetBuilder.setExpr(EvalTreeProtoSerializer.serialize(target.getEvalTree())); + targetBuilder.setExpr(EvalNodeSerializer.serialize(target.getEvalTree())); if (target.hasAlias()) { targetBuilder.setAlias(target.getAlias()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/package-info.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/package-info.java new file mode 100644 index 0000000000..b148fec588 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package provides (de)serialization API for logical plan and it related parts. + * They employ protocol buffer to (de)serialize logical plans. + */ +package org.apache.tajo.plan.serder; \ No newline at end of file diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index acd9443f39..7942fd9d99 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -103,7 +103,7 @@ message ScanNode { optional string alias = 2; required bool existTargets = 3; repeated Target targets = 4; - optional EvalTree qual = 5; + optional EvalNodeTree qual = 5; } message PartitionScanSpec { @@ -112,14 +112,14 @@ message PartitionScanSpec { message FilterNode { required int32 childId = 1; - required EvalTree qual = 2; + required EvalNodeTree qual = 2; } message JoinNode { required int32 leftChildId = 1; required int32 rightChildId = 2; required JoinType joinType = 3; - optional EvalTree joinQual = 4; + optional EvalNodeTree joinQual = 4; required bool existsTargets = 5; repeated Target targets = 6; } @@ -128,7 +128,7 @@ message GroupbyNode { required int32 childId = 1; required bool distinct = 2; repeated ColumnProto groupingKeys = 3; - repeated EvalTree aggFunctions = 4; + repeated EvalNodeTree aggFunctions = 4; repeated Target targets = 5; } @@ -139,7 +139,7 @@ message DistinctGroupbyNode { repeated Target targets = 4; repeated ColumnProto groupingKeys = 5; repeated int32 resultId = 6; - repeated EvalTree aggFunctions = 7; + repeated EvalNodeTree aggFunctions = 7; } message SortNode { @@ -156,7 +156,7 @@ message WindowAggNode { required int32 childId = 1; repeated ColumnProto partitionKeys = 2; repeated SortSpecProto sortSpecs = 3; - repeated EvalTree windowFunctions = 4; + repeated EvalNodeTree windowFunctions = 4; required bool distinct = 5; // if distinct aggregation function is included in window function repeated Target targets = 6; } @@ -193,7 +193,7 @@ message SetSessionNode { } message Target { - required EvalTree expr = 1; + required EvalNodeTree expr = 1; optional string alias = 2; } @@ -348,7 +348,7 @@ enum EvalType { CONST = 33; } -message EvalTree { +message EvalNodeTree { repeated EvalNode nodes = 1; } @@ -451,12 +451,12 @@ message WinFunctionEvalSpec { message WindowStartBound { required WindowFrameStartBoundType boundType = 1; - optional EvalTree number = 2; + optional EvalNodeTree number = 2; } message WindowEndBound { required WindowFrameEndBoundType boundType = 1; - optional EvalTree number = 2; + optional EvalNodeTree number = 2; } repeated SortSpecProto sortSpec = 1; From 7d4c8f3878ded73e76d22aa88831d6348e77d643 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 30 Dec 2014 18:06:53 +0900 Subject: [PATCH 30/31] Rename sid to visitSeq and pid to nodeId. --- .../engine/planner/PhysicalPlannerImpl.java | 10 +-- .../tajo/engine/planner/enforce/Enforcer.java | 12 +-- .../builder/DistinctGroupbyBuilder.java | 4 +- .../src/main/proto/TajoWorkerProtocol.proto | 12 +-- .../apache/tajo/plan/logical/LogicalNode.java | 12 +-- .../plan/serder/LogicalNodeDeserializer.java | 84 +++++++++---------- .../plan/serder/LogicalNodeSerializer.java | 34 ++++---- tajo-plan/src/main/proto/Plan.proto | 32 +++---- 8 files changed, 100 insertions(+), 100 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 26b5d16ec5..d043a27a7f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -1217,15 +1217,15 @@ public static EnforceProperty getAlgorithmEnforceProperty(Enforcer enforcer, Log List properties = enforcer.getEnforceProperties(type); EnforceProperty found = null; for (EnforceProperty property : properties) { - if (type == EnforceType.JOIN && property.getJoin().getPid() == node.getPID()) { + if (type == EnforceType.JOIN && property.getJoin().getNodeId() == node.getPID()) { found = property; - } else if (type == EnforceType.GROUP_BY && property.getGroupby().getPid() == node.getPID()) { + } else if (type == EnforceType.GROUP_BY && property.getGroupby().getNodeId() == node.getPID()) { found = property; - } else if (type == EnforceType.DISTINCT_GROUP_BY && property.getDistinct().getPid() == node.getPID()) { + } else if (type == EnforceType.DISTINCT_GROUP_BY && property.getDistinct().getNodeId() == node.getPID()) { found = property; - } else if (type == EnforceType.SORT && property.getSort().getPid() == node.getPID()) { + } else if (type == EnforceType.SORT && property.getSort().getNodeId() == node.getPID()) { found = property; - } else if (type == EnforceType.COLUMN_PARTITION && property.getColumnPartition().getPid() == node.getPID()) { + } else if (type == EnforceType.COLUMN_PARTITION && property.getColumnPartition().getNodeId() == node.getPID()) { found = property; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java index e2d77444a0..8128390933 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java @@ -98,7 +98,7 @@ public void addOutputDistinct() { public void enforceJoinAlgorithm(int pid, JoinEnforce.JoinAlgorithm algorithm) { EnforceProperty.Builder builder = newProperty(); JoinEnforce.Builder enforce = JoinEnforce.newBuilder(); - enforce.setPid(pid); + enforce.setNodeId(pid); enforce.setAlgorithm(algorithm); builder.setType(EnforceType.JOIN); @@ -109,7 +109,7 @@ public void enforceJoinAlgorithm(int pid, JoinEnforce.JoinAlgorithm algorithm) { public void enforceSortAggregation(int pid, @Nullable SortSpec[] sortSpecs) { EnforceProperty.Builder builder = newProperty(); GroupbyEnforce.Builder enforce = GroupbyEnforce.newBuilder(); - enforce.setPid(pid); + enforce.setNodeId(pid); enforce.setAlgorithm(GroupbyAlgorithm.SORT_AGGREGATION); if (sortSpecs != null) { for (SortSpec sortSpec : sortSpecs) { @@ -125,7 +125,7 @@ public void enforceSortAggregation(int pid, @Nullable SortSpec[] sortSpecs) { public void enforceHashAggregation(int pid) { EnforceProperty.Builder builder = newProperty(); GroupbyEnforce.Builder enforce = GroupbyEnforce.newBuilder(); - enforce.setPid(pid); + enforce.setNodeId(pid); enforce.setAlgorithm(GroupbyAlgorithm.HASH_AGGREGATION); builder.setType(EnforceType.GROUP_BY); @@ -146,7 +146,7 @@ public void enforceDistinctAggregation(int pid, List sortSpecArrays) { EnforceProperty.Builder builder = newProperty(); DistinctGroupbyEnforcer.Builder enforce = DistinctGroupbyEnforcer.newBuilder(); - enforce.setPid(pid); + enforce.setNodeId(pid); enforce.setIsMultipleAggregation(isMultipleAggregation); enforce.setAlgorithm(algorithm); if (sortSpecArrays != null) { @@ -164,7 +164,7 @@ public void enforceDistinctAggregation(int pid, public void enforceSortAlgorithm(int pid, SortEnforce.SortAlgorithm algorithm) { EnforceProperty.Builder builder = newProperty(); SortEnforce.Builder enforce = SortEnforce.newBuilder(); - enforce.setPid(pid); + enforce.setNodeId(pid); enforce.setAlgorithm(algorithm); builder.setType(EnforceType.SORT); @@ -203,7 +203,7 @@ public void removeBroadcast(String tableName) { public void enforceColumnPartitionAlgorithm(int pid, ColumnPartitionAlgorithm algorithm) { EnforceProperty.Builder builder = newProperty(); ColumnPartitionEnforcer.Builder enforce = ColumnPartitionEnforcer.newBuilder(); - enforce.setPid(pid); + enforce.setNodeId(pid); enforce.setAlgorithm(algorithm); builder.setType(EnforceType.COLUMN_PARTITION); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 8a2552fde0..5c6e80ecb5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -701,7 +701,7 @@ private void setDistinctAggregationEnforcer( sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); } sortSpecArrays.add( SortSpecArray.newBuilder() - .setPid(secondStageDistinctNode.getSubPlans().get(index).getPID()) + .setNodeId(secondStageDistinctNode.getSubPlans().get(index).getPID()) .addAllSortSpecs(sortSpecs).build()); } secondStageBlock.getEnforcer().enforceDistinctAggregation(secondStageDistinctNode.getPID(), @@ -729,7 +729,7 @@ private void setMultiStageAggregationEnforcer( sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); } sortSpecArrays.add( SortSpecArray.newBuilder() - .setPid(thirdStageDistinctNode.getSubPlans().get(index).getPID()) + .setNodeId(thirdStageDistinctNode.getSubPlans().get(index).getPID()) .addAllSortSpecs(sortSpecs).build()); } thirdStageBlock.getEnforcer().enforceDistinctAggregation(thirdStageDistinctNode.getPID(), diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto index f4871cdccd..b8c9575b35 100644 --- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto +++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto @@ -261,7 +261,7 @@ message JoinEnforce { MERGE_JOIN = 4; } - required int32 pid = 1; + required int32 nodeId = 1; required JoinAlgorithm algorithm = 2; } @@ -271,7 +271,7 @@ message GroupbyEnforce { SORT_AGGREGATION = 1; } - required int32 pid = 1; + required int32 nodeId = 1; required GroupbyAlgorithm algorithm = 2; repeated SortSpecProto sortSpecs = 3; } @@ -282,7 +282,7 @@ message SortEnforce { MERGE_SORT = 1; } - required int32 pid = 1; + required int32 nodeId = 1; required SortAlgorithm algorithm = 2; } @@ -296,7 +296,7 @@ message ColumnPartitionEnforcer { SORT_PARTITION = 1; } - required int32 pid = 1; + required int32 nodeId = 1; required ColumnPartitionAlgorithm algorithm = 2; } @@ -313,10 +313,10 @@ message DistinctGroupbyEnforcer { } message SortSpecArray { - required int32 pid = 1; + required int32 nodeId = 1; repeated SortSpecProto sortSpecs = 2; } - required int32 pid = 1; + required int32 nodeId = 1; required DistinctAggregationAlgorithm algorithm = 2; repeated SortSpecArray sortSpecArrays = 3; required bool isMultipleAggregation = 4 [default = false]; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java index 71cb6727b1..200977be3f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java @@ -30,24 +30,24 @@ import org.apache.tajo.util.TUtil; public abstract class LogicalNode implements Cloneable, GsonObject { - @Expose private int pid; + @Expose private int nodeId; @Expose private NodeType type; @Expose private Schema inputSchema; @Expose private Schema outputSchema; @Expose private double cost = 0; - protected LogicalNode(int pid, NodeType type) { - this.pid = pid; + protected LogicalNode(int nodeId, NodeType type) { + this.nodeId = nodeId; this.type = type; } public int getPID() { - return pid; + return nodeId; } public void setPID(int pid) { - this.pid = pid; + this.nodeId = pid; } public NodeType getType() { @@ -109,7 +109,7 @@ public boolean deepEquals(Object o) { @Override public Object clone() throws CloneNotSupportedException { LogicalNode node = (LogicalNode)super.clone(); - node.pid = pid; + node.nodeId = nodeId; node.type = type; node.inputSchema = (Schema) (inputSchema != null ? inputSchema.clone() : null); node.outputSchema = (Schema) (outputSchema != null ? outputSchema.clone() : null); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 98347d32ca..5cbed7ec27 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -66,7 +66,7 @@ public static LogicalNode deserialize(OverridableConf context, PlanProto.Logical Collections.sort(nodeList, new Comparator() { @Override public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { - return o1.getSid() - o2.getSid(); + return o1.getVisitSeq() - o2.getVisitSeq(); } }); @@ -161,7 +161,7 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { throw new RuntimeException("Unknown NodeType: " + protoNode.getType().name()); } - nodeMap.put(protoNode.getSid(), current); + nodeMap.put(protoNode.getVisitSeq(), current); } return current; @@ -171,8 +171,8 @@ private static LogicalRootNode convertRoot(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.RootNode rootProto = protoNode.getRoot(); - LogicalRootNode root = new LogicalRootNode(protoNode.getPid()); - root.setChild(nodeMap.get(rootProto.getChildId())); + LogicalRootNode root = new LogicalRootNode(protoNode.getNodeId()); + root.setChild(nodeMap.get(rootProto.getChildSeq())); if (protoNode.hasInSchema()) { root.setInSchema(convertSchema(protoNode.getInSchema())); } @@ -186,7 +186,7 @@ private static LogicalRootNode convertRoot(Map nodeMap, private static SetSessionNode convertSetSession(PlanProto.LogicalNode protoNode) { PlanProto.SetSessionNode setSessionProto = protoNode.getSetSession(); - SetSessionNode setSession = new SetSessionNode(protoNode.getPid()); + SetSessionNode setSession = new SetSessionNode(protoNode.getNodeId()); setSession.init(setSessionProto.getName(), setSessionProto.hasValue() ? setSessionProto.getValue() : null); return setSession; @@ -195,7 +195,7 @@ private static SetSessionNode convertSetSession(PlanProto.LogicalNode protoNode) private static EvalExprNode convertEvalExpr(OverridableConf context, PlanProto.LogicalNode protoNode) { PlanProto.EvalExprNode evalExprProto = protoNode.getExprEval(); - EvalExprNode evalExpr = new EvalExprNode(protoNode.getPid()); + EvalExprNode evalExpr = new EvalExprNode(protoNode.getNodeId()); evalExpr.setInSchema(convertSchema(protoNode.getInSchema())); evalExpr.setTargets(convertTargets(context, evalExprProto.getTargetsList())); @@ -206,9 +206,9 @@ private static ProjectionNode convertProjection(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.LimitNode limitProto = protoNode.getLimit(); - LimitNode limitNode = new LimitNode(protoNode.getPid()); - limitNode.setChild(nodeMap.get(limitProto.getChildId())); + LimitNode limitNode = new LimitNode(protoNode.getNodeId()); + limitNode.setChild(nodeMap.get(limitProto.getChildSeq())); limitNode.setInSchema(convertSchema(protoNode.getInSchema())); limitNode.setOutSchema(convertSchema(protoNode.getOutSchema())); limitNode.setFetchFirst(limitProto.getFetchFirstNum()); @@ -230,8 +230,8 @@ private static LimitNode convertLimit(Map nodeMap, PlanPro private static SortNode convertSort(Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.SortNode sortProto = protoNode.getSort(); - SortNode sortNode = new SortNode(protoNode.getPid()); - sortNode.setChild(nodeMap.get(sortProto.getChildId())); + SortNode sortNode = new SortNode(protoNode.getNodeId()); + sortNode.setChild(nodeMap.get(sortProto.getChildSeq())); sortNode.setInSchema(convertSchema(protoNode.getInSchema())); sortNode.setOutSchema(convertSchema(protoNode.getOutSchema())); sortNode.setSortSpecs(convertSortSpecs(sortProto.getSortSpecsList())); @@ -243,8 +243,8 @@ private static HavingNode convertHaving(OverridableConf context, Map 0) { windowAgg.setPartitionKeys(convertColumns(windowAggProto.getPartitionKeysList())); @@ -287,8 +287,8 @@ private static GroupbyNode convertGroupby(OverridableConf context, Map 0) { @@ -311,8 +311,8 @@ private static DistinctGroupbyNode convertDistinctGroupby(OverridableConf contex PlanProto.LogicalNode protoNode) { PlanProto.DistinctGroupbyNode distinctGroupbyProto = protoNode.getDistinctGroupby(); - DistinctGroupbyNode distinctGroupby = new DistinctGroupbyNode(protoNode.getPid()); - distinctGroupby.setChild(nodeMap.get(distinctGroupbyProto.getChildId())); + DistinctGroupbyNode distinctGroupby = new DistinctGroupbyNode(protoNode.getNodeId()); + distinctGroupby.setChild(nodeMap.get(distinctGroupbyProto.getChildSeq())); if (distinctGroupbyProto.hasGroupbyNode()) { distinctGroupby.setGroupbyPlan(convertGroupby(context, nodeMap, distinctGroupbyProto.getGroupbyNode())); @@ -352,9 +352,9 @@ private static JoinNode convertJoin(OverridableConf context, Map nodeMap, PlanProto.LogicalNode protoNode) { PlanProto.UnionNode unionProto = protoNode.getUnion(); - UnionNode union = new UnionNode(protoNode.getPid()); + UnionNode union = new UnionNode(protoNode.getNodeId()); union.setInSchema(convertSchema(protoNode.getInSchema())); union.setOutSchema(convertSchema(protoNode.getOutSchema())); - union.setLeftChild(nodeMap.get(unionProto.getLeftChildId())); - union.setRightChild(nodeMap.get(unionProto.getRightChildId())); + union.setLeftChild(nodeMap.get(unionProto.getLeftChildSeq())); + union.setRightChild(nodeMap.get(unionProto.getRightChildSeq())); return union; } private static ScanNode convertScan(OverridableConf context, PlanProto.LogicalNode protoNode) { - ScanNode scan = new ScanNode(protoNode.getPid()); + ScanNode scan = new ScanNode(protoNode.getNodeId()); fillScanNode(context, protoNode, scan); return scan; @@ -421,7 +421,7 @@ private static void fillScanNode(OverridableConf context, PlanProto.LogicalNode } private static PartitionedTableScanNode convertPartitionScan(OverridableConf context, PlanProto.LogicalNode protoNode) { - PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getPid()); + PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getNodeId()); fillScanNode(context, protoNode, partitionedScan); PlanProto.PartitionScanSpec partitionScanProto = protoNode.getPartitionScan(); @@ -438,8 +438,8 @@ private static TableSubQueryNode convertTableSubQuery(OverridableConf context, PlanProto.LogicalNode protoNode) { PlanProto.TableSubQueryNode proto = protoNode.getTableSubQuery(); - TableSubQueryNode tableSubQuery = new TableSubQueryNode(protoNode.getPid()); - tableSubQuery.init(proto.getTableName(), nodeMap.get(proto.getChildId())); + TableSubQueryNode tableSubQuery = new TableSubQueryNode(protoNode.getNodeId()); + tableSubQuery.init(proto.getTableName(), nodeMap.get(proto.getChildSeq())); tableSubQuery.setInSchema(convertSchema(protoNode.getInSchema())); if (proto.getTargetsCount() > 0) { tableSubQuery.setTargets(convertTargets(context, proto.getTargetsList())); @@ -454,14 +454,14 @@ private static CreateTableNode convertCreateTable(Map node PlanProto.StoreTableNodeSpec storeTableNodeSpec = protoNode.getStoreTable(); PlanProto.CreateTableNodeSpec createTableNodeSpec = protoNode.getCreateTable(); - CreateTableNode createTable = new CreateTableNode(protoNode.getPid()); + CreateTableNode createTable = new CreateTableNode(protoNode.getNodeId()); if (protoNode.hasInSchema()) { createTable.setInSchema(convertSchema(protoNode.getInSchema())); } if (protoNode.hasOutSchema()) { createTable.setOutSchema(convertSchema(protoNode.getOutSchema())); } - createTable.setChild(nodeMap.get(persistentStoreProto.getChildId())); + createTable.setChild(nodeMap.get(persistentStoreProto.getChildSeq())); createTable.setStorageType(persistentStoreProto.getStorageType()); createTable.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); @@ -486,14 +486,14 @@ private static InsertNode convertInsert(Map nodeMap, PlanProto.StoreTableNodeSpec storeTableNodeSpec = protoNode.getStoreTable(); PlanProto.InsertNodeSpec insertNodeSpec = protoNode.getInsert(); - InsertNode insertNode = new InsertNode(protoNode.getPid()); + InsertNode insertNode = new InsertNode(protoNode.getNodeId()); if (protoNode.hasInSchema()) { insertNode.setInSchema(convertSchema(protoNode.getInSchema())); } if (protoNode.hasOutSchema()) { insertNode.setOutSchema(convertSchema(protoNode.getOutSchema())); } - insertNode.setChild(nodeMap.get(persistentStoreProto.getChildId())); + insertNode.setChild(nodeMap.get(persistentStoreProto.getChildSeq())); insertNode.setStorageType(persistentStoreProto.getStorageType()); insertNode.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); @@ -520,7 +520,7 @@ private static InsertNode convertInsert(Map nodeMap, } private static DropTableNode convertDropTable(PlanProto.LogicalNode protoNode) { - DropTableNode dropTable = new DropTableNode(protoNode.getPid()); + DropTableNode dropTable = new DropTableNode(protoNode.getNodeId()); PlanProto.DropTableNode dropTableProto = protoNode.getDropTable(); dropTable.init(dropTableProto.getTableName(), dropTableProto.getIfExists(), dropTableProto.getPurge()); @@ -529,7 +529,7 @@ private static DropTableNode convertDropTable(PlanProto.LogicalNode protoNode) { } private static CreateDatabaseNode convertCreateDatabase(PlanProto.LogicalNode protoNode) { - CreateDatabaseNode createDatabase = new CreateDatabaseNode(protoNode.getPid()); + CreateDatabaseNode createDatabase = new CreateDatabaseNode(protoNode.getNodeId()); PlanProto.CreateDatabaseNode createDatabaseProto = protoNode.getCreateDatabase(); createDatabase.init(createDatabaseProto.getDbName(), createDatabaseProto.getIfNotExists()); @@ -538,7 +538,7 @@ private static CreateDatabaseNode convertCreateDatabase(PlanProto.LogicalNode pr } private static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoNode) { - DropDatabaseNode dropDatabase = new DropDatabaseNode(protoNode.getPid()); + DropDatabaseNode dropDatabase = new DropDatabaseNode(protoNode.getNodeId()); PlanProto.DropDatabaseNode dropDatabaseProto = protoNode.getDropDatabase(); dropDatabase.init(dropDatabaseProto.getDbName(), dropDatabaseProto.getIfExists()); @@ -547,7 +547,7 @@ private static DropDatabaseNode convertDropDatabase(PlanProto.LogicalNode protoN } private static AlterTablespaceNode convertAlterTablespace(PlanProto.LogicalNode protoNode) { - AlterTablespaceNode alterTablespace = new AlterTablespaceNode(protoNode.getPid()); + AlterTablespaceNode alterTablespace = new AlterTablespaceNode(protoNode.getNodeId()); PlanProto.AlterTablespaceNode alterTablespaceProto = protoNode.getAlterTablespace(); alterTablespace.setTablespaceName(alterTablespaceProto.getTableSpaceName()); @@ -564,7 +564,7 @@ private static AlterTablespaceNode convertAlterTablespace(PlanProto.LogicalNode } private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) { - AlterTableNode alterTable = new AlterTableNode(protoNode.getPid()); + AlterTableNode alterTable = new AlterTableNode(protoNode.getNodeId()); PlanProto.AlterTableNode alterTableProto = protoNode.getAlterTable(); alterTable.setTableName(alterTableProto.getTableName()); @@ -588,7 +588,7 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) } private static TruncateTableNode convertTruncateTable(PlanProto.LogicalNode protoNode) { - TruncateTableNode truncateTable = new TruncateTableNode(protoNode.getPid()); + TruncateTableNode truncateTable = new TruncateTableNode(protoNode.getNodeId()); PlanProto.TruncateTableNode truncateTableProto = protoNode.getTruncateTableNode(); truncateTable.setTableNames(truncateTableProto.getTableNamesList()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 87970eefca..39a13badfb 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -84,8 +84,8 @@ private static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext } PlanProto.LogicalNode.Builder nodeBuilder = PlanProto.LogicalNode.newBuilder(); - nodeBuilder.setSid(selfId); - nodeBuilder.setPid(node.getPID()); + nodeBuilder.setVisitSeq(selfId); + nodeBuilder.setNodeId(node.getPID()); nodeBuilder.setType(convertType(node.getType())); // some DDL statements like DropTable or DropDatabase do not have in/out schemas @@ -111,7 +111,7 @@ public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, Logical int [] childIds = registerGetChildIds(context, root); PlanProto.RootNode.Builder rootBuilder = PlanProto.RootNode.newBuilder(); - rootBuilder.setChildId(childIds[0]); + rootBuilder.setChildSeq(childIds[0]); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, root); nodeBuilder.setRoot(rootBuilder); @@ -158,7 +158,7 @@ public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, L int [] childIds = registerGetChildIds(context, projection); PlanProto.ProjectionNode.Builder projectionBuilder = PlanProto.ProjectionNode.newBuilder(); - projectionBuilder.setChildId(childIds[0]); + projectionBuilder.setChildSeq(childIds[0]); projectionBuilder.addAllTargets( ProtoUtil.toProtoObjects(projection.getTargets())); projectionBuilder.setDistinct(projection.isDistinct()); @@ -178,7 +178,7 @@ public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, Logica int [] childIds = registerGetChildIds(context, limit); PlanProto.LimitNode.Builder limitBuilder = PlanProto.LimitNode.newBuilder(); - limitBuilder.setChildId(childIds[0]); + limitBuilder.setChildSeq(childIds[0]); limitBuilder.setFetchFirstNum(limit.getFetchFirstNum()); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, limit); @@ -195,7 +195,7 @@ public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, Lo int [] childIds = registerGetChildIds(context, windowAgg); PlanProto.WindowAggNode.Builder windowAggBuilder = PlanProto.WindowAggNode.newBuilder(); - windowAggBuilder.setChildId(childIds[0]); + windowAggBuilder.setChildSeq(childIds[0]); if (windowAgg.hasPartitionKeys()) { windowAggBuilder.addAllPartitionKeys( @@ -232,7 +232,7 @@ public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, Logical int [] childIds = registerGetChildIds(context, sort); PlanProto.SortNode.Builder sortBuilder = PlanProto.SortNode.newBuilder(); - sortBuilder.setChildId(childIds[0]); + sortBuilder.setChildSeq(childIds[0]); for (int i = 0; i < sort.getSortKeys().length; i++) { sortBuilder.addSortSpecs(sort.getSortKeys()[i].getProto()); } @@ -252,7 +252,7 @@ public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, Logic int [] childIds = registerGetChildIds(context, having); PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); - filterBuilder.setChildId(childIds[0]); + filterBuilder.setChildSeq(childIds[0]); filterBuilder.setQual(EvalNodeSerializer.serialize(having.getQual())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, having); @@ -276,7 +276,7 @@ private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, Gro int [] childIds = registerGetChildIds(context, node); PlanProto.GroupbyNode.Builder groupbyBuilder = PlanProto.GroupbyNode.newBuilder(); - groupbyBuilder.setChildId(childIds[0]); + groupbyBuilder.setChildSeq(childIds[0]); groupbyBuilder.setDistinct(node.isDistinct()); if (node.groupingKeyNum() > 0) { @@ -304,7 +304,7 @@ public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan pl int [] childIds = registerGetChildIds(context, node); PlanProto.DistinctGroupbyNode.Builder distGroupbyBuilder = PlanProto.DistinctGroupbyNode.newBuilder(); - distGroupbyBuilder.setChildId(childIds[0]); + distGroupbyBuilder.setChildSeq(childIds[0]); if (node.getGroupbyPlan() != null) { distGroupbyBuilder.setGroupbyNode(buildGroupby(context, node.getGroupbyPlan())); } @@ -343,7 +343,7 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic int [] childIds = registerGetChildIds(context, filter); PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); - filterBuilder.setChildId(childIds[0]); + filterBuilder.setChildSeq(childIds[0]); filterBuilder.setQual(EvalNodeSerializer.serialize(filter.getQual())); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, filter); @@ -362,8 +362,8 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical // building itself PlanProto.JoinNode.Builder joinBuilder = PlanProto.JoinNode.newBuilder(); joinBuilder.setJoinType(convertJoinType(join.getJoinType())); - joinBuilder.setLeftChildId(childIds[0]); - joinBuilder.setRightChildId(childIds[1]); + joinBuilder.setLeftChildSeq(childIds[0]); + joinBuilder.setRightChilSeq(childIds[1]); if (join.hasJoinQual()) { joinBuilder.setJoinQual(EvalNodeSerializer.serialize(join.getJoinQual())); } @@ -391,8 +391,8 @@ public LogicalNode visitUnion(SerializeContext context, LogicalPlan plan, Logica PlanProto.UnionNode.Builder unionBuilder = PlanProto.UnionNode.newBuilder(); unionBuilder.setAll(true); - unionBuilder.setLeftChildId(childIds[0]); - unionBuilder.setRightChildId(childIds[1]); + unionBuilder.setLeftChildSeq(childIds[0]); + unionBuilder.setRightChildSeq(childIds[1]); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setUnion(unionBuilder); @@ -465,7 +465,7 @@ public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan int [] childIds = registerGetChildIds(context, node); PlanProto.TableSubQueryNode.Builder builder = PlanProto.TableSubQueryNode.newBuilder(); - builder.setChildId(childIds[0]); + builder.setChildSeq(childIds[0]); builder.setTableName(node.getTableName()); @@ -623,7 +623,7 @@ public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, Logic private static PlanProto.PersistentStoreNode.Builder buildPersistentStoreBuilder(PersistentStoreNode node, int [] childIds) { PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = PlanProto.PersistentStoreNode.newBuilder(); - persistentStoreBuilder.setChildId(childIds[0]); + persistentStoreBuilder.setChildSeq(childIds[0]); persistentStoreBuilder.setStorageType(node.getStorageType()); if (node.hasOptions()) { persistentStoreBuilder.setTableProperties(node.getOptions().getProto()); diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 7942fd9d99..3e4f07cfcb 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -63,8 +63,8 @@ message LogicalNodeTree { } message LogicalNode { - required int32 sid = 1; - required int32 pid = 2; + required int32 visitSeq = 1; + required int32 nodeId = 2; required NodeType type = 3; optional SchemaProto in_schema = 4; optional SchemaProto out_schema = 5; @@ -111,13 +111,13 @@ message PartitionScanSpec { } message FilterNode { - required int32 childId = 1; + required int32 childSeq = 1; required EvalNodeTree qual = 2; } message JoinNode { - required int32 leftChildId = 1; - required int32 rightChildId = 2; + required int32 leftChildSeq = 1; + required int32 rightChilSeq = 2; required JoinType joinType = 3; optional EvalNodeTree joinQual = 4; required bool existsTargets = 5; @@ -125,7 +125,7 @@ message JoinNode { } message GroupbyNode { - required int32 childId = 1; + required int32 childSeq = 1; required bool distinct = 2; repeated ColumnProto groupingKeys = 3; repeated EvalNodeTree aggFunctions = 4; @@ -133,7 +133,7 @@ message GroupbyNode { } message DistinctGroupbyNode { - required int32 childId = 1; + required int32 childSeq = 1; optional LogicalNode groupbyNode = 2; repeated LogicalNode subPlans = 3; repeated Target targets = 4; @@ -143,17 +143,17 @@ message DistinctGroupbyNode { } message SortNode { - required int32 childId = 1; + required int32 childSeq = 1; repeated SortSpecProto sortSpecs = 2; } message LimitNode { - required int32 childId = 1; + required int32 childSeq = 1; required int64 fetchFirstNum = 2; } message WindowAggNode { - required int32 childId = 1; + required int32 childSeq = 1; repeated ColumnProto partitionKeys = 2; repeated SortSpecProto sortSpecs = 3; repeated EvalNodeTree windowFunctions = 4; @@ -162,19 +162,19 @@ message WindowAggNode { } message UnionNode { - required int32 leftChildId = 1; - required int32 rightChildId = 2; + required int32 leftChildSeq = 1; + required int32 rightChildSeq = 2; required bool all = 3; } message TableSubQueryNode { - required int32 childId = 1; + required int32 childSeq = 1; required string tableName = 2; repeated Target targets = 3; } message ProjectionNode { - required int32 childId = 1; + required int32 childSeq = 1; required bool distinct = 2; repeated Target targets = 3; } @@ -184,7 +184,7 @@ message EvalExprNode { } message RootNode { - required int32 childId = 1; + required int32 childSeq = 1; } message SetSessionNode { @@ -215,7 +215,7 @@ message PartitionTableScanSpec { } message PersistentStoreNode { - required int32 childId = 1; + required int32 childSeq = 1; required StoreType storageType = 2; required KeyValueSetProto tableProperties = 3; } From 27287ab2f3a54308d59a8e178b740eac865b5dd2 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 30 Dec 2014 21:38:45 +0900 Subject: [PATCH 31/31] Simplified getColumnId(). --- .../java/org/apache/tajo/catalog/Schema.java | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 27127faa5c..71c1b01dc2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -205,24 +205,15 @@ public int getColumnId(String name) { } // The following is some workaround code. - String [] parts = name.split("\\."); - if (parts.length == 2 || parts.length == 3) { - if (fieldsByQualifiedName.containsKey(name)) { - return fieldsByQualifiedName.get(name); - } else { - return -1; - } - } else { - List list = fieldsByName.get(name); - if (list == null) { - return -1; - } else if (list.size() == 1) { - return fieldsByName.get(name).get(0); - } else if (list.size() == 0) { - return -1; - } else { // if list.size > 2 - throw throwAmbiguousFieldException(list); - } + List list = fieldsByName.get(name); + if (list == null) { + return -1; + } else if (list.size() == 1) { + return fieldsByName.get(name).get(0); + } else if (list.size() == 0) { + return -1; + } else { // if list.size > 2 + throw throwAmbiguousFieldException(list); } }