From 5bbc7d2816276e1d19349c9f6a0a1dab797dd74e Mon Sep 17 00:00:00 2001 From: "navis.ryu" Date: Wed, 8 Apr 2015 13:45:32 +0900 Subject: [PATCH] TAJO-1505 Transferred plan should not include supplementary of FunctionDesc --- .../tajo/catalog/AbstractCatalogClient.java | 9 +- .../apache/tajo/catalog/AlterTableDesc.java | 11 +- .../org/apache/tajo/catalog/CatalogUtil.java | 4 +- .../java/org/apache/tajo/catalog/Column.java | 6 +- .../org/apache/tajo/catalog/FunctionDesc.java | 14 ++- .../org/apache/tajo/catalog/IndexDesc.java | 5 +- .../java/org/apache/tajo/catalog/Schema.java | 16 +-- .../org/apache/tajo/catalog/SortSpec.java | 7 +- .../org/apache/tajo/catalog/TableDesc.java | 13 +- .../org/apache/tajo/catalog/TableMeta.java | 23 ++-- .../tajo/catalog/partition/PartitionDesc.java | 8 +- .../tajo/catalog/partition/PartitionKey.java | 5 +- .../partition/PartitionMethodDesc.java | 7 +- .../tajo/catalog/statistics/ColumnStats.java | 7 +- .../tajo/catalog/statistics/StatSet.java | 3 +- .../tajo/catalog/statistics/TableStats.java | 7 +- .../function/ClassBaseInvocationDesc.java | 3 +- .../org/apache/tajo/function/Function.java | 3 +- .../tajo/function/FunctionInvocation.java | 13 +- .../tajo/function/FunctionSignature.java | 3 +- .../tajo/function/FunctionSupplement.java | 9 +- .../function/StaticMethodInvocationDesc.java | 3 +- .../org/apache/tajo/catalog/TestColumn.java | 9 +- .../apache/tajo/catalog/TestFunctionDesc.java | 13 +- .../apache/tajo/catalog/TestIndexDesc.java | 5 +- .../apache/tajo/catalog/TestKeyValueSet.java | 5 +- .../org/apache/tajo/catalog/TestSchema.java | 25 ++-- .../apache/tajo/catalog/TestTableDesc.java | 7 +- .../apache/tajo/catalog/TestTableMeta.java | 9 +- .../catalog/statistics/TestColumnStat.java | 7 +- .../tajo/catalog/statistics/TestStatSet.java | 3 +- .../catalog/statistics/TestTableStat.java | 7 +- .../tajo/catalog/store/HCatalogStore.java | 3 +- .../tajo/catalog/store/TestHCatalogStore.java | 25 ++-- .../apache/tajo/catalog/CatalogServer.java | 5 +- .../apache/tajo/catalog/store/MemStore.java | 4 +- .../tajo/client/CatalogAdminClientImpl.java | 7 +- .../apache/tajo/client/SessionConnection.java | 5 +- .../java/org/apache/tajo/SerializeOption.java | 24 ++++ .../org/apache/tajo/common/ProtoObject.java | 4 +- .../java/org/apache/tajo/datum/Datum.java | 3 +- .../java/org/apache/tajo/json/GsonObject.java | 5 +- .../org/apache/tajo/util/KeyValueSet.java | 5 +- .../java/org/apache/tajo/util/ProtoUtil.java | 13 +- .../org/apache/tajo/datum/TestBytesDatum.java | 3 +- .../org/apache/tajo/datum/TestDateDatum.java | 3 +- .../org/apache/tajo/datum/TestInet4Datum.java | 3 +- .../org/apache/tajo/datum/TestTimeDatum.java | 3 +- .../apache/tajo/datum/TestTimestampDatum.java | 3 +- .../engine/codegen/EvalCodeGenContext.java | 9 +- .../engine/planner/PhysicalPlannerImpl.java | 6 +- .../tajo/engine/planner/enforce/Enforcer.java | 7 +- .../engine/planner/global/DataChannel.java | 7 +- .../builder/DistinctGroupbyBuilder.java | 15 +-- .../rules/GlobalPlanEqualityTester.java | 3 +- .../planner/physical/PhysicalPlanUtil.java | 4 +- .../tajo/engine/query/QueryContext.java | 4 +- .../tajo/engine/query/TaskRequestImpl.java | 23 ++-- .../tajo/master/QueryCoordinatorService.java | 3 +- .../apache/tajo/master/QueryInProgress.java | 8 +- .../org/apache/tajo/master/QueryInfo.java | 9 +- .../tajo/master/TajoContainerProxy.java | 7 +- .../tajo/master/TajoMasterClientService.java | 21 ++-- .../master/cluster/WorkerConnectionInfo.java | 3 +- .../apache/tajo/master/exec/DDLExecutor.java | 3 +- .../exec/ExplainPlanPreprocessorForTest.java | 5 +- .../NonForwardQueryResultFileScanner.java | 4 +- .../tajo/master/exec/QueryExecutor.java | 15 +-- .../master/rm/TajoWorkerResourceManager.java | 3 +- .../querymaster/DefaultTaskScheduler.java | 19 +-- .../org/apache/tajo/querymaster/Query.java | 3 +- .../apache/tajo/querymaster/QueryMaster.java | 4 +- .../org/apache/tajo/querymaster/Task.java | 9 +- .../java/org/apache/tajo/session/Session.java | 5 +- .../java/org/apache/tajo/util/JSPUtil.java | 3 +- .../tajo/util/history/HistoryWriter.java | 7 +- .../tajo/util/history/QueryHistory.java | 3 +- .../tajo/util/history/StageHistory.java | 3 +- .../apache/tajo/util/history/TaskHistory.java | 3 +- .../org/apache/tajo/worker/FetchImpl.java | 3 +- .../java/org/apache/tajo/worker/Task.java | 15 +-- .../tajo/worker/TaskAttemptContext.java | 7 +- .../org/apache/tajo/worker/TaskHistory.java | 5 +- .../apache/tajo/worker/TaskRunnerHistory.java | 7 +- .../tajo/worker/WorkerHeartbeatService.java | 3 +- .../apache/tajo/engine/eval/ExprTestBase.java | 3 +- .../apache/tajo/engine/eval/TestEvalTree.java | 3 +- .../engine/planner/TestLogicalPlanner.java | 15 +-- .../apache/tajo/master/TestRepartitioner.java | 5 +- .../master/rm/TestTajoResourceManager.java | 3 +- .../apache/tajo/storage/TestFileFragment.java | 3 +- .../org/apache/tajo/storage/TestRowFile.java | 3 +- .../org/apache/tajo/worker/TestHistory.java | 5 +- .../java/org/apache/tajo/plan/Target.java | 7 +- .../apache/tajo/plan/expr/CaseWhenEval.java | 3 +- .../org/apache/tajo/plan/expr/EvalNode.java | 7 +- .../tajo/plan/function/AggFunction.java | 3 +- .../tajo/plan/function/GeneralFunction.java | 3 +- .../tajo/plan/function/WindowAggFunc.java | 3 +- .../apache/tajo/plan/logical/LogicalNode.java | 3 +- .../rules/LogicalPlanEqualityTester.java | 3 +- .../tajo/plan/serder/EvalNodeSerializer.java | 20 ++-- .../plan/serder/LogicalNodeSerializer.java | 112 ++++++++++-------- .../tajo/storage/BaseTupleComparator.java | 12 +- .../tajo/storage/fragment/Fragment.java | 3 +- .../storage/fragment/FragmentConvertor.java | 9 +- .../tajo/storage/hbase/HBaseFragment.java | 3 +- .../tajo/storage/fragment/FileFragment.java | 3 +- .../tajo/storage/index/bst/BSTIndex.java | 5 +- 109 files changed, 522 insertions(+), 362 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/SerializeOption.java diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 458d6e0489..24e4c1640b 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -21,6 +21,7 @@ import com.google.protobuf.ServiceException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SerializeOption; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService; import org.apache.tajo.catalog.exception.NoSuchFunctionException; @@ -521,7 +522,7 @@ public final boolean createTable(final TableDesc desc) { return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { public Boolean call(NettyClientBase client) throws ServiceException { CatalogProtocolService.BlockingInterface stub = getStub(client); - return stub.createTable(null, desc.getProto()).getValue(); + return stub.createTable(null, desc.getProto(SerializeOption.GENERIC)).getValue(); } }.withRetries(); } catch (ServiceException e) { @@ -589,7 +590,7 @@ public final boolean createIndex(final IndexDesc index) { return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { public Boolean call(NettyClientBase client) throws ServiceException { CatalogProtocolService.BlockingInterface stub = getStub(client); - return stub.createIndex(null, index.getProto()).getValue(); + return stub.createIndex(null, index.getProto(SerializeOption.GENERIC)).getValue(); } }.withRetries(); } catch (ServiceException e) { @@ -724,7 +725,7 @@ public final boolean createFunction(final FunctionDesc funcDesc) { return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { public Boolean call(NettyClientBase client) throws ServiceException { CatalogProtocolService.BlockingInterface stub = getStub(client); - return stub.createFunction(null, funcDesc.getProto()).getValue(); + return stub.createFunction(null, funcDesc.getProto(SerializeOption.GENERIC)).getValue(); } }.withRetries(); } catch (ServiceException e) { @@ -837,7 +838,7 @@ public final boolean alterTable(final AlterTableDesc desc) { return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { public Boolean call(NettyClientBase client) throws ServiceException { CatalogProtocolService.BlockingInterface stub = getStub(client); - return stub.alterTable(null, desc.getProto()).getValue(); + return stub.alterTable(null, desc.getProto(SerializeOption.GENERIC)).getValue(); } }.withRetries(); } catch (ServiceException e) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java index 9133b6f807..9048d4a7eb 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/AlterTableDesc.java @@ -20,6 +20,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -145,12 +146,12 @@ public AlterTableDesc clone() throws CloneNotSupportedException { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, AlterTableDesc.class); } @Override - public AlterTableDescProto getProto() { + public AlterTableDescProto getProto(SerializeOption option) { AlterTableDescProto.Builder builder = AlterTableDescProto.newBuilder(); if (null != this.tableName) { @@ -166,10 +167,10 @@ public AlterTableDescProto getProto() { builder.setAlterColumnName(alterColumnBuilder.build()); } if (null != this.addColumn) { - builder.setAddColumn(addColumn.getProto()); + builder.setAddColumn(addColumn.getProto(option)); } if (null != this.properties) { - builder.setParams(properties.getProto()); + builder.setParams(properties.getProto(option)); } switch (alterTableType) { @@ -195,7 +196,7 @@ public AlterTableDescProto getProto() { } if (null != this.partitionDesc) { - builder.setPartitionDesc(partitionDesc.getProto()); + builder.setPartitionDesc(partitionDesc.getProto(option)); } return builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 628d710f37..07ff95b187 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -20,11 +20,11 @@ import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.common.TajoDataTypes; @@ -328,7 +328,7 @@ public static PartitionMethodDesc newPartitionMethodDesc(CatalogProtos.Partition public static SchemaProto getQualfiedSchema(String tableName, SchemaProto schema) { Schema restored = new Schema(schema); restored.setQualifier(tableName); - return restored.getProto(); + return restored.getProto(SerializeOption.GENERIC); } public static DataType newDataType(Type type, String code) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index 12edaa74cb..52b2098fc8 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.common.ProtoObject; @@ -144,9 +145,10 @@ public int hashCode() { /** * * @return The protocol buffer object for Column + * @param option */ @Override - public ColumnProto getProto() { + public ColumnProto getProto(SerializeOption option) { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder .setName(this.name) @@ -161,7 +163,7 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, Column.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java index 23d39f2528..81ea479881 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.annotation.NotNull; import org.apache.tajo.function.*; import org.apache.tajo.json.GsonObject; @@ -61,6 +62,7 @@ public FunctionDesc(FunctionDescProto proto) throws ClassNotFoundException { this.supplement = new FunctionSupplement(proto.getSupplement()); } + @SuppressWarnings("unchecked") public FunctionDesc(String signature, String className, FunctionType type, DataType retType, @NotNull DataType... argTypes) throws ClassNotFoundException { @@ -165,7 +167,7 @@ public int hashCode() { public boolean equals(Object obj) { if (obj instanceof FunctionDesc) { FunctionDesc other = (FunctionDesc) obj; - if(this.getProto().equals(other.getProto())) + if(this.getProto(SerializeOption.GENERIC).equals(other.getProto(SerializeOption.GENERIC))) return true; } return false; @@ -183,15 +185,15 @@ public Object clone() throws CloneNotSupportedException{ } @Override - public FunctionDescProto getProto() { + public FunctionDescProto getProto(SerializeOption option) { if (builder == null) { builder = FunctionDescProto.newBuilder(); } else { builder.clear(); } - builder.setSignature(signature.getProto()); - builder.setSupplement(supplement.getProto()); - builder.setInvocation(invocation.getProto()); + builder.setSignature(signature.getProto(option)); + builder.setSupplement(supplement.getProto(option)); + builder.setInvocation(invocation.getProto(option)); return builder.build(); } @@ -200,7 +202,7 @@ public String toString() { return getHelpSignature(); } - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, FunctionDesc.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java index 151b919af5..a0deb486b4 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; @@ -89,7 +90,7 @@ public boolean isAscending() { } @Override - public IndexDescProto getProto() { + public IndexDescProto getProto(SerializeOption option) { IndexDescProto.Builder builder = IndexDescProto.newBuilder(); CatalogProtos.TableIdentifierProto.Builder tableIdentifierBuilder = CatalogProtos.TableIdentifierProto.newBuilder(); @@ -102,7 +103,7 @@ public IndexDescProto getProto() { builder.setTableIdentifier(tableIdentifierBuilder.build()); builder.setIndexName(this.indexName); - builder.setColumn(this.column.getProto()); + builder.setColumn(this.column.getProto(option)); builder.setIndexMethod(indexMethod); builder.setIsUnique(this.isUnique); builder.setIsClustered(this.isClustered); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index ed2cd2c651..b4ed84e50c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -23,6 +23,7 @@ import com.google.gson.annotations.Expose; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.SchemaUtil.ColumnVisitor; import org.apache.tajo.catalog.exception.AlreadyExistsFieldException; import org.apache.tajo.catalog.json.CatalogGsonHelper; @@ -362,7 +363,7 @@ public int hashCode() { public boolean equals(Object o) { if (o instanceof Schema) { Schema other = (Schema) o; - return getProto().equals(other.getProto()); + return getProto(SerializeOption.GENERIC).equals(other.getProto(SerializeOption.GENERIC)); } return false; } @@ -379,16 +380,17 @@ public Object clone() throws CloneNotSupportedException { } @Override - public SchemaProto getProto() { + public SchemaProto getProto(SerializeOption option) { SchemaProto.Builder builder = SchemaProto.newBuilder(); - SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder); + SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder, option); SchemaUtil.visitSchema(this, recursiveBuilder); return builder.build(); } private static class SchemaProtoBuilder implements ColumnVisitor { private SchemaProto.Builder builder; - public SchemaProtoBuilder(SchemaProto.Builder builder) { + private SerializeOption option; + public SchemaProtoBuilder(SchemaProto.Builder builder, SerializeOption option) { this.builder = builder; } @@ -399,12 +401,12 @@ public void visit(int depth, Column column) { DataType.Builder updatedType = DataType.newBuilder(column.getDataType()); updatedType.setNumNestedFields(column.typeDesc.nestedRecordSchema.size()); - ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto()); + ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto(option)); updatedColumn.setDataType(updatedType); builder.addFields(updatedColumn.build()); } else { - builder.addFields(column.getProto()); + builder.addFields(column.getProto(option)); } } } @@ -426,7 +428,7 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, Schema.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java index 5b10dfd5d3..1970fb8f23 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SortSpec.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.GsonObject; @@ -103,7 +104,7 @@ public boolean equals(Object object) { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, SortSpec.class); } @@ -112,9 +113,9 @@ public String toString() { } @Override - public SortSpecProto getProto() { + public SortSpecProto getProto(SerializeOption option) { SortSpecProto.Builder builder = SortSpecProto.newBuilder(); - builder.setColumn(sortKey.getProto()); + builder.setColumn(sortKey.getProto(option)); builder.setAscending(ascending); builder.setNullFirst(nullFirst); return builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index ee096a2a54..e055e87cf2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -196,30 +197,30 @@ public String toString() { return gson.toJson(this); } - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, TableDesc.class); } - public TableDescProto getProto() { + public TableDescProto getProto(SerializeOption option) { TableDescProto.Builder builder = TableDescProto.newBuilder(); if (this.tableName != null) { builder.setTableName(this.tableName); } if (this.schema != null) { - builder.setSchema(schema.getProto()); + builder.setSchema(schema.getProto(option)); } if (this.meta != null) { - builder.setMeta(meta.getProto()); + builder.setMeta(meta.getProto(option)); } if (this.uri != null) { builder.setPath(this.uri.toString()); } if (this.stats != null) { - builder.setStats(this.stats.getProto()); + builder.setStats(this.stats.getProto(option)); } if (this.partitionMethodDesc != null) { - builder.setPartition(this.partitionMethodDesc.getProto()); + builder.setPartition(this.partitionMethodDesc.getProto(option)); } if (this.external != null) { builder.setIsExternal(external); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index 2d95e6be09..8629b659a2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -22,6 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -132,17 +133,19 @@ public KeyValueSet getOptions() { public Map toMap() { return getOptions().getAllKeyValus(); } - + + @Override public boolean equals(Object object) { if(object instanceof TableMeta) { TableMeta other = (TableMeta) object; - return this.getProto().equals(other.getProto()); + return this.getProto(SerializeOption.GENERIC).equals(other.getProto(SerializeOption.GENERIC)); } return false; } - + + @Override public int hashCode() { return Objects.hashCode(getStoreType(), getOptions()); } @@ -166,15 +169,15 @@ public String toString() { //////////////////////////////////////////////////////////////////////// // ProtoObject //////////////////////////////////////////////////////////////////////// - public TableProto getProto() { - mergeLocalToProto(); + public TableProto getProto(SerializeOption option) { + mergeLocalToProto(option); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } @Override - public String toJson() { + public String toJson(SerializeOption option) { mergeProtoToLocal(); return CatalogGsonHelper.toJson(this, TableMeta.class); } @@ -191,20 +194,20 @@ private void maybeInitBuilder() { viaProto = true; } - private void mergeLocalToBuilder() { + private void mergeLocalToBuilder(SerializeOption option) { if (storeType != null) { builder.setStoreType(storeType); } if (this.options != null) { - builder.setParams(options.getProto()); + builder.setParams(options.getProto(option)); } } - private void mergeLocalToProto() { + private void mergeLocalToProto(SerializeOption option) { if(viaProto) { maybeInitBuilder(); } - mergeLocalToBuilder(); + mergeLocalToBuilder(option); proto = builder.build(); viaProto = true; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java index b6d883d150..371b6593aa 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java @@ -22,7 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.Column; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; @@ -131,7 +131,7 @@ public boolean equals(Object o) { @Override - public CatalogProtos.PartitionDescProto getProto() { + public CatalogProtos.PartitionDescProto getProto(SerializeOption option) { if (builder == null) { builder = CatalogProtos.PartitionDescProto.newBuilder(); } @@ -143,7 +143,7 @@ public CatalogProtos.PartitionDescProto getProto() { builder.clearPartitionKeys(); if (this.partitionKeys != null) { for(PartitionKey partitionKey : this.partitionKeys) { - builder.addPartitionKeys(partitionKey.getProto()); + builder.addPartitionKeys(partitionKey.getProto(option)); } } @@ -161,7 +161,7 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, PartitionDesc.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java index 085598b7ec..9d61985ddb 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; @@ -105,7 +106,7 @@ public boolean equals(Object o) { } @Override - public CatalogProtos.PartitionKeyProto getProto() { + public CatalogProtos.PartitionKeyProto getProto(SerializeOption option) { if (builder == null) { builder = CatalogProtos.PartitionKeyProto.newBuilder(); } @@ -127,7 +128,7 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, PartitionKey.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java index 721a7a060e..e71a591153 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionMethodDesc.java @@ -22,6 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -114,7 +115,7 @@ public int hashCode() { } @Override - public CatalogProtos.PartitionMethodProto getProto() { + public CatalogProtos.PartitionMethodProto getProto(SerializeOption option) { TableIdentifierProto.Builder tableIdentifierBuilder = TableIdentifierProto.newBuilder(); if (databaseName != null) { tableIdentifierBuilder.setDatabaseName(databaseName); @@ -127,7 +128,7 @@ public CatalogProtos.PartitionMethodProto getProto() { builder.setTableIdentifier(tableIdentifierBuilder.build()); builder.setPartitionType(partitionType); builder.setExpression(expression); - builder.setExpressionSchema(expressionSchema.getProto()); + builder.setExpressionSchema(expressionSchema.getProto(option)); return builder.build(); } @@ -148,7 +149,7 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, PartitionMethodDesc.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java index 1bd0276342..8473c5077f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/ColumnStats.java @@ -24,6 +24,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import com.google.protobuf.ByteString; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -144,17 +145,17 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, ColumnStats.class); } @Override - public CatalogProtos.ColumnStatsProto getProto() { + public CatalogProtos.ColumnStatsProto getProto(SerializeOption option) { CatalogProtos.ColumnStatsProto.Builder builder = CatalogProtos.ColumnStatsProto.newBuilder(); if (this.column != null) { - builder.setColumn(this.column.getProto()); + builder.setColumn(this.column.getProto(option)); } if (this.numDistVals != null) { builder.setNumDistVal(this.numDistVals); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatSet.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatSet.java index 9c95a1d906..3a45eb6174 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatSet.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatSet.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.common.collect.Maps; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StatProto; @@ -122,7 +123,7 @@ private void setModified() { } @Override - public StatSetProto getProto() { + public StatSetProto getProto(SerializeOption option) { if (!viaProto) { mergeLocalToBuilder(); proto = builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java index dd358ae1de..bf47765e15 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java @@ -24,6 +24,7 @@ import com.google.common.base.Objects; import com.google.gson.Gson; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.json.GsonObject; @@ -232,12 +233,12 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, TableStats.class); } @Override - public TableStatsProto getProto() { + public TableStatsProto getProto(SerializeOption option) { TableStatsProto.Builder builder = CatalogProtos.TableStatsProto.newBuilder(); builder.setNumRows(this.numRows); @@ -257,7 +258,7 @@ public TableStatsProto getProto() { } if (this.columnStatses != null) { for (ColumnStats colStat : columnStatses) { - builder.addColStat(colStat.getProto()); + builder.addColStat(colStat.getProto(option)); } } return builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/ClassBaseInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/ClassBaseInvocationDesc.java index dccf789a1a..0922ac0f18 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/ClassBaseInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/ClassBaseInvocationDesc.java @@ -19,6 +19,7 @@ package org.apache.tajo.function; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import static org.apache.tajo.catalog.proto.CatalogProtos.ClassBaseInvocationDescProto; @@ -63,7 +64,7 @@ public String toString() { } @Override - public ClassBaseInvocationDescProto getProto() { + public ClassBaseInvocationDescProto getProto(SerializeOption option) { ClassBaseInvocationDescProto.Builder builder = ClassBaseInvocationDescProto.newBuilder(); builder.setClassName(functionClass.getName()); return builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java index 6a538b8821..1e848d56aa 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/Function.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.json.GsonObject; import org.apache.tajo.catalog.Column; @@ -58,7 +59,7 @@ public Object clone() throws CloneNotSupportedException { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, Function.class); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java index 653bdb6906..e4ae090f74 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionInvocation.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionInvocationProto; @@ -122,22 +123,22 @@ public ClassBaseInvocationDesc getAggregationJIT() { } @Override - public FunctionInvocationProto getProto() { + public FunctionInvocationProto getProto(SerializeOption option) { FunctionInvocationProto.Builder builder = FunctionInvocationProto.newBuilder(); if (hasLegacy()) { - builder.setLegacy(legacy.getProto()); + builder.setLegacy(legacy.getProto(option)); } if (hasScalar()) { - builder.setScalar(scalar.getProto()); + builder.setScalar(scalar.getProto(option)); } if (hasAggregation()) { - builder.setAggregation(aggregation.getProto()); + builder.setAggregation(aggregation.getProto(option)); } if (hasScalarJIT()) { - builder.setScalarJIT(scalarJIT.getProto()); + builder.setScalarJIT(scalarJIT.getProto(option)); } if (hasAggregationJIT()) { - builder.setAggregationJIT(aggregationJIT.getProto()); + builder.setAggregationJIT(aggregationJIT.getProto(option)); } return builder.build(); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java index fc3a0560dc..82b2688f31 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSignature.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.annotation.NotNull; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.util.TUtil; @@ -105,7 +106,7 @@ public FunctionSignature clone() throws CloneNotSupportedException { } @Override - public FunctionSignatureProto getProto() { + public FunctionSignatureProto getProto(SerializeOption option) { FunctionSignatureProto.Builder builder = FunctionSignatureProto.newBuilder(); builder.setType(functionType); builder.setName(name); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSupplement.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSupplement.java index 33cf908c5d..3b9d384762 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSupplement.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/FunctionSupplement.java @@ -19,6 +19,7 @@ package org.apache.tajo.function; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionSupplementProto; @@ -88,15 +89,15 @@ public String getExample() { } @Override - public FunctionSupplementProto getProto() { + public FunctionSupplementProto getProto(SerializeOption option) { FunctionSupplementProto.Builder builder = FunctionSupplementProto.newBuilder(); - if (shortDescription != null) { + if (shortDescription != null && option != SerializeOption.INTERNAL) { builder.setShortDescription(shortDescription); } - if (detail != null) { + if (detail != null && option != SerializeOption.INTERNAL) { builder.setDetail(detail); } - if (example != null) { + if (example != null && option != SerializeOption.INTERNAL) { builder.setExample(example); } return builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/StaticMethodInvocationDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/StaticMethodInvocationDesc.java index b909878048..1a83834eb6 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/StaticMethodInvocationDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/function/StaticMethodInvocationDesc.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.common.collect.Maps; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.util.ClassUtil; import org.apache.tajo.util.TUtil; @@ -121,7 +122,7 @@ public String toString() { @Override - public StaticMethodInvocationDescProto getProto() { + public StaticMethodInvocationDescProto getProto(SerializeOption option) { StaticMethodInvocationDescProto.Builder builder = StaticMethodInvocationDescProto.newBuilder(); builder.setClassName(baseClassName.getName()); builder.setMethodName(methodName); diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestColumn.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestColumn.java index ef08d51628..16f8f2efd9 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestColumn.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestColumn.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -94,16 +95,16 @@ public final void testMultiLevelQualifiedName() { @Test public final void testToJson() { - Column col = new Column(field1.getProto()); - String json = col.toJson(); + Column col = new Column(field1.getProto(SerializeOption.GENERIC)); + String json = col.toJson(SerializeOption.GENERIC); Column fromJson = CatalogGsonHelper.fromJson(json, Column.class); assertEquals(col, fromJson); } @Test public final void testToProto() { - Column column = new Column(field1.getProto()); - CatalogProtos.ColumnProto proto = column.getProto(); + Column column = new Column(field1.getProto(SerializeOption.GENERIC)); + CatalogProtos.ColumnProto proto = column.getProto(SerializeOption.GENERIC); Column fromProto = new Column(proto); assertEquals(column, fromProto); } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java index 92d2aa43e0..fd409f7c50 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestFunctionDesc.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.tajo.SerializeOption; import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -76,7 +77,7 @@ public void testGetSignature() throws IOException, ClassNotFoundException { CommonTestingUtil.getTestDir(TEST_PATH); File save = new File(TEST_PATH + "/save.dat"); - FileUtil.writeProto(save, desc.getProto()); + FileUtil.writeProto(save, desc.getProto(SerializeOption.GENERIC)); FunctionDescProto proto = FunctionDescProto.getDefaultInstance(); proto = (FunctionDescProto) FileUtil.loadProto(save, proto); @@ -91,7 +92,7 @@ public void testGetSignature() throws IOException, ClassNotFoundException { assertArrayEquals(CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8), newDesc.getParamTypes()); - assertEquals(desc.getProto(), newDesc.getProto()); + assertEquals(desc.getProto(SerializeOption.GENERIC), newDesc.getProto(SerializeOption.GENERIC)); } @Test @@ -99,10 +100,10 @@ public void testToJson() throws InternalException { FunctionDesc desc = new FunctionDesc("sum", TestSum.class, FunctionType.GENERAL, CatalogUtil.newSimpleDataType(Type.INT4), CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8)); - String json = desc.toJson(); + String json = desc.toJson(SerializeOption.GENERIC); FunctionDesc fromJson = CatalogGsonHelper.fromJson(json, FunctionDesc.class); assertEquals(desc, fromJson); - assertEquals(desc.getProto(), fromJson.getProto()); + assertEquals(desc.getProto(SerializeOption.GENERIC), fromJson.getProto(SerializeOption.GENERIC)); } @Test @@ -110,10 +111,10 @@ public void testGetProto() throws InternalException, ClassNotFoundException { FunctionDesc desc = new FunctionDesc("sum", TestSum.class, FunctionType.GENERAL, CatalogUtil.newSimpleDataType(Type.INT4), CatalogUtil.newSimpleDataTypeArray(Type.INT4, Type.INT8)); - FunctionDescProto proto = desc.getProto(); + FunctionDescProto proto = desc.getProto(SerializeOption.GENERIC); FunctionDesc fromProto = new FunctionDesc(proto); assertEquals(desc, fromProto); - assertEquals(desc.toJson(), fromProto.toJson()); + assertEquals(desc.toJson(SerializeOption.GENERIC), fromProto.toJson(SerializeOption.GENERIC)); } @Test diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java index 3fa8f5399b..b4dffff174 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes.Type; @@ -58,8 +59,8 @@ public static void tearDown() throws Exception { @Test public void testIndexDescProto() { - IndexDescProto proto = desc1.getProto(); - assertEquals(desc1.getProto(), proto); + IndexDescProto proto = desc1.getProto(SerializeOption.GENERIC); + assertEquals(desc1.getProto(SerializeOption.GENERIC), proto); assertEquals(desc1, new IndexDesc(proto)); } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java index a09275d44e..794c5e2fcc 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.tajo.SerializeOption; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; @@ -88,7 +89,7 @@ public final void testGetProto() { opts.set("name", "abc"); opts.set("delimiter", ","); - PrimitiveProtos.KeyValueSetProto proto = opts.getProto(); + PrimitiveProtos.KeyValueSetProto proto = opts.getProto(SerializeOption.GENERIC); KeyValueSet opts2 = new KeyValueSet(proto); assertEquals(opts, opts2); @@ -109,7 +110,7 @@ public final void testRemove() { assertTrue(true); } - KeyValueSet opts2 = new KeyValueSet(opts.getProto()); + KeyValueSet opts2 = new KeyValueSet(opts.getProto(SerializeOption.GENERIC)); try { opts2.get("name"); assertTrue(false); diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java index edd0f3e5b1..532f1f2e9e 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestSchema.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.exception.AlreadyExistsFieldException; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; @@ -135,7 +136,7 @@ public final void testSchemaSchema() { @Test public final void testSchemaSchemaProto() { - Schema schema2 = new Schema(schema.getProto()); + Schema schema2 = new Schema(schema.getProto(SerializeOption.GENERIC)); assertEquals(schema, schema2); } @@ -167,7 +168,7 @@ public final void testEqualsObject() { @Test public final void testGetProto() { - SchemaProto proto = schema.getProto(); + SchemaProto proto = schema.getProto(SerializeOption.GENERIC); assertEquals("name", proto.getFields(0).getName()); assertEquals("age", proto.getFields(1).getName()); @@ -180,13 +181,13 @@ public final void testClone() throws CloneNotSupportedException { schema.addColumn("abc", Type.FLOAT8); schema.addColumn("bbc", Type.FLOAT8); - Schema schema2 = new Schema(schema.getProto()); - assertEquals(schema.getProto(), schema2.getProto()); + Schema schema2 = new Schema(schema.getProto(SerializeOption.GENERIC)); + assertEquals(schema.getProto(SerializeOption.GENERIC), schema2.getProto(SerializeOption.GENERIC)); assertEquals(schema.getColumn(0), schema2.getColumn(0)); assertEquals(schema.size(), schema2.size()); Schema schema3 = (Schema) schema.clone(); - assertEquals(schema.getProto(), schema3.getProto()); + assertEquals(schema.getProto(SerializeOption.GENERIC), schema3.getProto(SerializeOption.GENERIC)); assertEquals(schema.getColumn(0), schema3.getColumn(0)); assertEquals(schema.size(), schema3.size()); } @@ -201,24 +202,24 @@ public final void testAddExistColumn() { @Test public final void testJson() { - Schema schema2 = new Schema(schema.getProto()); - String json = schema2.toJson(); + Schema schema2 = new Schema(schema.getProto(SerializeOption.GENERIC)); + String json = schema2.toJson(SerializeOption.GENERIC); Schema fromJson = CatalogGsonHelper.fromJson(json, Schema.class); assertEquals(schema2, fromJson); - assertEquals(schema2.getProto(), fromJson.getProto()); + assertEquals(schema2.getProto(SerializeOption.GENERIC), fromJson.getProto(SerializeOption.GENERIC)); } @Test public final void testProto() { - Schema schema2 = new Schema(schema.getProto()); - SchemaProto proto = schema2.getProto(); + Schema schema2 = new Schema(schema.getProto(SerializeOption.GENERIC)); + SchemaProto proto = schema2.getProto(SerializeOption.GENERIC); Schema fromJson = new Schema(proto); assertEquals(schema2, fromJson); } @Test public final void testSetQualifier() { - Schema schema2 = new Schema(schema.getProto()); + Schema schema2 = new Schema(schema.getProto(SerializeOption.GENERIC)); schema2.setQualifier("test1"); Column column = schema2.getColumn(1); assertEquals(1, schema2.getColumnIdByName("age")); @@ -294,7 +295,7 @@ public void testNestedRecord4() { public static void verifySchema(Schema s1) { assertEquals(s1, s1); - SchemaProto proto = s1.getProto(); + SchemaProto proto = s1.getProto(SerializeOption.GENERIC); assertEquals("Proto (de)serialized schema is different from the original: ", s1, new Schema(proto)); Schema cloned = null; diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java index feeaadd5d9..00ce54a793 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java @@ -19,6 +19,7 @@ package org.apache.tajo.catalog; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -89,7 +90,7 @@ public void testGetProto() throws CloneNotSupportedException, IOException { Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); TableDesc desc = new TableDesc("table1", schema, info, path.toUri()); desc.setStats(stats); - CatalogProtos.TableDescProto proto = desc.getProto(); + CatalogProtos.TableDescProto proto = desc.getProto(SerializeOption.GENERIC); TableDesc fromProto = new TableDesc(proto); assertEquals("equality check the object deserialized from json", desc, fromProto); @@ -100,11 +101,11 @@ public void testToJson() throws CloneNotSupportedException, IOException { Path path = new Path(CommonTestingUtil.getTestDir(), "tajo"); TableDesc desc = new TableDesc("table1", schema, info, path.toUri()); desc.setStats(stats); - String json = desc.toJson(); + String json = desc.toJson(SerializeOption.GENERIC); TableDesc fromJson = CatalogGsonHelper.fromJson(json, TableDesc.class); assertEquals("equality check the object deserialized from json", desc, fromJson); - assertEquals("equality between protos", desc.getProto(), fromJson.getProto()); + assertEquals("equality between protos", desc.getProto(SerializeOption.GENERIC), fromJson.getProto(SerializeOption.GENERIC)); } public TableDesc testClone(TableDesc desc) throws CloneNotSupportedException { diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 904b4bc6ee..79b6e43e50 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableProto; @@ -42,7 +43,7 @@ public void testTableMetaTableProto() { schema1.addColumn("addr", Type.TEXT); TableMeta meta1 = CatalogUtil.newTableMeta(StoreType.CSV); - TableMeta meta2 = new TableMeta(meta1.getProto()); + TableMeta meta2 = new TableMeta(meta1.getProto(SerializeOption.GENERIC)); assertEquals(meta1, meta2); } @@ -89,16 +90,16 @@ public void testEqualsObject() { @Test public void testGetProto() { - TableProto proto = meta.getProto(); + TableProto proto = meta.getProto(SerializeOption.GENERIC); TableMeta newMeta = new TableMeta(proto); assertEquals(meta, newMeta); } @Test public void testToJson() { - String json = meta.toJson(); + String json = meta.toJson(SerializeOption.GENERIC); TableMeta fromJson = CatalogGsonHelper.fromJson(json, TableMeta.class); assertEquals(meta, fromJson); - assertEquals(meta.getProto(), fromJson.getProto()); + assertEquals(meta.getProto(SerializeOption.GENERIC), fromJson.getProto(SerializeOption.GENERIC)); } } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java index 6c1c311711..a0493da775 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestColumnStat.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog.statistics; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.common.TajoDataTypes.Type; @@ -38,7 +39,7 @@ public final void testColumnStat() { assertTrue(1000 == stat.getNumDistValues()); assertTrue(999 == stat.getNumNulls()); - ColumnStats stat2 = new ColumnStats(stat.getProto()); + ColumnStats stat2 = new ColumnStats(stat.getProto(SerializeOption.GENERIC)); assertTrue(1000 == stat2.getNumDistValues()); assertTrue(999 == stat2.getNumNulls()); } @@ -51,7 +52,7 @@ public final void testEqualsObject() { stat.setMinValue(DatumFactory.createInt8(5)); stat.setMaxValue(DatumFactory.createInt8(10)); - ColumnStats stat2 = new ColumnStats(stat.getProto()); + ColumnStats stat2 = new ColumnStats(stat.getProto(SerializeOption.GENERIC)); assertEquals(stat, stat2); } @@ -63,7 +64,7 @@ public final void testJson() throws CloneNotSupportedException { stat.setMinValue(DatumFactory.createInt8(5)); stat.setMaxValue(DatumFactory.createInt8(10)); - String json = stat.toJson(); + String json = stat.toJson(SerializeOption.GENERIC); ColumnStats fromJson = CatalogGsonHelper.fromJson(json, ColumnStats.class); assertEquals(stat, fromJson); } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatSet.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatSet.java index 9285c8da9e..09a90ec3bb 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatSet.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestStatSet.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog.statistics; +import org.apache.tajo.SerializeOption; import org.junit.Test; import org.apache.tajo.catalog.proto.CatalogProtos.StatType; @@ -52,7 +53,7 @@ public final void testStatGroup() throws CloneNotSupportedException { assertEquals(101, group.getStat(StatType.TABLE_NUM_ROWS).getValue()); assertEquals(1, group.getStat(StatType.TABLE_NUM_BLOCKS).getValue()); - StatSet group2 = new StatSet(group.getProto()); + StatSet group2 = new StatSet(group.getProto(SerializeOption.GENERIC)); assertEquals(2, group2.getAllStats().size()); assertEquals(stat, group2.getStat(StatType.TABLE_NUM_ROWS)); assertEquals(101, group2.getStat(StatType.TABLE_NUM_ROWS).getValue()); diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java index b649dd7353..b489426a9a 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog.statistics; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; @@ -63,13 +64,13 @@ public final void testTableStat() throws CloneNotSupportedException { assertEquals(cols[i], stat.getColumnStats().get(i)); } - TableStats stat2 = new TableStats(stat.getProto()); + TableStats stat2 = new TableStats(stat.getProto(SerializeOption.GENERIC)); tableStatEquals(stat, stat2); TableStats stat3 = (TableStats) stat.clone(); tableStatEquals(stat, stat3); - String json = stat.toJson(); + String json = stat.toJson(SerializeOption.GENERIC); TableStats fromJson = CatalogGsonHelper.fromJson(json, TableStats.class); tableStatEquals(stat, fromJson); } @@ -108,7 +109,7 @@ public void testGetProtoThreadSafe() throws Exception { public void run() { for (int j = 0; j < 100; j++) { try { - TableStatsProto proto = tableStats.getProto(); + TableStatsProto proto = tableStats.getProto(SerializeOption.GENERIC); if (tableStats.getColumnStats().size() != proto.getColStatList().size()) { success.set(false); break; diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index e829b94afb..950e80b743 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -36,6 +36,7 @@ import org.apache.hcatalog.common.HCatUtil; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; @@ -279,7 +280,7 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St if (partitions != null) { tableDesc.setPartitionMethod(partitions); } - return tableDesc.getProto(); + return tableDesc.getProto(SerializeOption.GENERIC); } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java index 32ab674481..3784e86abd 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionKey; @@ -100,7 +101,7 @@ public void testTableUsingTextFile() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, CUSTOMER)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); @@ -129,7 +130,7 @@ public void testTableUsingRCFileWithBinarySerde() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, REGION)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); @@ -158,7 +159,7 @@ public void testTableUsingRCFileWithTextSerde() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, REGION)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); @@ -192,7 +193,7 @@ public void testTableWithNullValue() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, SUPPLIER)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, SUPPLIER)); @@ -242,7 +243,7 @@ public void testAddTableByPartition() throws Exception { CatalogProtos.PartitionType.COLUMN, "n_nationkey,n_date", expressionSchema); table.setPartitionMethod(partitions); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, NATION)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, NATION)); @@ -298,7 +299,7 @@ private void testAddPartition(URI uri, String tableName, String partitionName) t alterTableDesc.setPartitionDesc(partitionDesc); - store.alterTable(alterTableDesc.getProto()); + store.alterTable(alterTableDesc.getProto(SerializeOption.GENERIC)); CatalogProtos.PartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName); assertNotNull(resultDesc); @@ -324,7 +325,7 @@ private void testDropPartition(String tableName, String partitionName) throws E alterTableDesc.setPartitionDesc(partitionDesc); - store.alterTable(alterTableDesc.getProto()); + store.alterTable(alterTableDesc.getProto(SerializeOption.GENERIC)); } @Test @@ -340,7 +341,7 @@ public void testGetAllTableNames() throws Exception{ for(String tableName : tableNames){ TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema, meta, new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); } List tables = store.getAllTableNames("default"); @@ -365,7 +366,7 @@ public void testDeleteTable() throws Exception { String tableName = "table1"; TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath.toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, tableName)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, tableName)); @@ -390,7 +391,7 @@ public void testTableUsingSequenceFileWithBinarySerde() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, REGION)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); @@ -419,7 +420,7 @@ public void testTableUsingSequenceFileWithTextSerde() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, REGION)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); @@ -451,7 +452,7 @@ public void testTableUsingParquet() throws Exception { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); - store.createTable(table.getProto()); + store.createTable(table.getProto(SerializeOption.GENERIC)); assertTrue(store.existTable(DB_NAME, CUSTOMER)); TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index e9fb177cd4..27c0cb89bb 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -28,12 +28,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TajoConstants; import org.apache.tajo.annotation.ThreadSafe; import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService; import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary; import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.store.CatalogStore; import org.apache.tajo.catalog.store.DerbyStore; @@ -61,7 +61,6 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionType.*; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListProto; -import static org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; /** * This class provides the catalog service. The catalog service enables clients @@ -170,7 +169,7 @@ public String getCatalogServerName() { private void initBuiltinFunctions(Collection functions) throws ServiceException { for (FunctionDesc desc : functions) { - handler.createFunction(null, desc.getProto()); + handler.createFunction(null, desc.getProto(SerializeOption.GENERIC)); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index b0585048cc..d2ba12d360 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -25,12 +25,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; @@ -363,7 +363,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th } TableMeta newMeta = new TableMeta(tableDescProto.getMeta().getStoreType(), properties); - newTableDescProto = tableDescProto.toBuilder().setMeta(newMeta.getProto()).build(); + newTableDescProto = tableDescProto.toBuilder().setMeta(newMeta.getProto(SerializeOption.GENERIC)).build(); database.put(tableName, newTableDescProto); break; default: diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java index 6347ad174b..3faa1b4d22 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java @@ -19,6 +19,7 @@ package org.apache.tajo.client; import com.google.protobuf.ServiceException; +import org.apache.tajo.SerializeOption; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; @@ -144,11 +145,11 @@ public TableDesc call(NettyClientBase client) throws ServiceException, SQLExcept ClientProtos.CreateTableRequest.Builder builder = ClientProtos.CreateTableRequest.newBuilder(); builder.setSessionId(connection.sessionId); builder.setName(tableName); - builder.setSchema(schema.getProto()); - builder.setMeta(meta.getProto()); + builder.setSchema(schema.getProto(SerializeOption.GENERIC)); + builder.setMeta(meta.getProto(SerializeOption.GENERIC)); builder.setPath(path.toString()); if (partitionMethodDesc != null) { - builder.setPartition(partitionMethodDesc.getProto()); + builder.setPartition(partitionMethodDesc.getProto(SerializeOption.GENERIC)); } ClientProtos.TableResponse res = tajoMasterService.createExternalTable(null, builder.build()); if (res.getResultCode() == ClientProtos.ResultCode.OK) { diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 63631989f5..95ff6ad7dc 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -21,6 +21,7 @@ import com.google.protobuf.ServiceException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SerializeOption; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.annotation.Nullable; @@ -164,7 +165,7 @@ public Map call(NettyClientBase client) throws ServiceException keyValueSet.putAll(variables); ClientProtos.UpdateSessionVariableRequest request = ClientProtos.UpdateSessionVariableRequest.newBuilder() .setSessionId(sessionId) - .setSessionVars(keyValueSet.getProto()).build(); + .setSessionVars(keyValueSet.getProto(SerializeOption.GENERIC)).build(); SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); @@ -364,7 +365,7 @@ public Boolean call(NettyClientBase client) throws ServiceException { keyValueSet.putAll(sessionVarsCache); ClientProtos.UpdateSessionVariableRequest request = ClientProtos.UpdateSessionVariableRequest.newBuilder() .setSessionId(sessionId) - .setSessionVars(keyValueSet.getProto()).build(); + .setSessionVars(keyValueSet.getProto(SerializeOption.GENERIC)).build(); if (tajoMasterService.updateSessionVariables(null, request).getResultCode() != ResultCode.OK) { tajoMasterService.removeSession(null, sessionId); diff --git a/tajo-common/src/main/java/org/apache/tajo/SerializeOption.java b/tajo-common/src/main/java/org/apache/tajo/SerializeOption.java new file mode 100644 index 0000000000..a4e000841f --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/SerializeOption.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +// serialization option for protobuf or json +public enum SerializeOption { + GENERIC, INTERNAL +} diff --git a/tajo-common/src/main/java/org/apache/tajo/common/ProtoObject.java b/tajo-common/src/main/java/org/apache/tajo/common/ProtoObject.java index d5f7533dd2..6b7dd62cfa 100644 --- a/tajo-common/src/main/java/org/apache/tajo/common/ProtoObject.java +++ b/tajo-common/src/main/java/org/apache/tajo/common/ProtoObject.java @@ -19,7 +19,9 @@ package org.apache.tajo.common; import com.google.protobuf.Message; +import org.apache.tajo.SerializeOption; public interface ProtoObject

{ - public P getProto(); + + P getProto(SerializeOption option); } diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java index 1ff305df50..64733f2039 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java @@ -19,6 +19,7 @@ package org.apache.tajo.datum; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.SessionVars; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; @@ -208,7 +209,7 @@ public Datum greaterThanEqual(Datum datum) { public abstract int compareTo(Datum datum); @Override - public String toJson() { + public String toJson(SerializeOption option) { return CommonGsonHelper.toJson(this, Datum.class); } diff --git a/tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java b/tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java index 3c49a21034..a28704226a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java +++ b/tajo-common/src/main/java/org/apache/tajo/json/GsonObject.java @@ -18,6 +18,9 @@ package org.apache.tajo.json; +import org.apache.tajo.SerializeOption; + public interface GsonObject { - public String toJson(); + + String toJson(SerializeOption option); } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index 6af0c9e9da..0aa22ad91c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.json.GsonObject; @@ -227,7 +228,7 @@ public Object clone() throws CloneNotSupportedException { } @Override - public KeyValueSetProto getProto() { + public KeyValueSetProto getProto(SerializeOption option) { KeyValueSetProto.Builder builder = KeyValueSetProto.newBuilder(); KeyValueProto.Builder kvBuilder = KeyValueProto.newBuilder(); @@ -243,7 +244,7 @@ public KeyValueSetProto getProto() { return builder.build(); } - public String toJson() { + public String toJson(SerializeOption option) { return CommonGsonHelper.toJson(this, KeyValueSet.class); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java index f9d759bed9..545d0ac6a5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/ProtoUtil.java @@ -19,6 +19,7 @@ package org.apache.tajo.util; import com.google.common.collect.Lists; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import java.util.Collection; @@ -54,20 +55,22 @@ public static Map convertToMap(KeyValueSetProto proto) { } public static KeyValueSetProto convertFromMap(Map map) { - return new KeyValueSet(map).getProto(); + return new KeyValueSet(map).getProto(SerializeOption.GENERIC); } /** - * It converts an array of ProtoObjects into Iteratable one. + * Converts an array of ProtoObjects into an Iterable. * - * @param protoObjects * @param + * @param protoObjects + * @param option * @return */ - public static Iterable toProtoObjects(ProtoObject[] protoObjects) { + @SuppressWarnings("unchecked") + public static Iterable toProtoObjects(ProtoObject[] protoObjects, SerializeOption option) { List converted = Lists.newArrayList(); for (int i = 0; i < protoObjects.length; i++) { - converted.add((T) protoObjects[i].getProto()); + converted.add((T) protoObjects[i].getProto(option)); } return converted; } diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java index 4dcbbee8d2..cafcdfffe3 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestBytesDatum.java @@ -18,6 +18,7 @@ package org.apache.tajo.datum; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.util.Bytes; @@ -52,7 +53,7 @@ public final void testSize() { @Test public final void testJson() { Datum d = DatumFactory.createBlob("12345".getBytes()); - String json = d.toJson(); + String json = d.toJson(SerializeOption.GENERIC); Datum fromJson = CommonGsonHelper.fromJson(json, Datum.class); assertTrue(d.equalsTo(fromJson).asBool()); } diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java index 41b4dcae47..0248cfa69b 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestDateDatum.java @@ -18,6 +18,7 @@ package org.apache.tajo.datum; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.InvalidCastException; import org.apache.tajo.json.CommonGsonHelper; @@ -83,7 +84,7 @@ public final void testAsTextBytes() { @Test public final void testToJson() { Datum d = DatumFactory.createDate(DATE); - Datum copy = CommonGsonHelper.fromJson(d.toJson(), Datum.class); + Datum copy = CommonGsonHelper.fromJson(d.toJson(SerializeOption.GENERIC), Datum.class); assertEquals(d, copy); } diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java index 14ebc7c912..d06056d2a9 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestInet4Datum.java @@ -18,6 +18,7 @@ package org.apache.tajo.datum; +import org.apache.tajo.SerializeOption; import org.apache.tajo.json.CommonGsonHelper; import org.junit.Before; import org.junit.Test; @@ -85,7 +86,7 @@ public final void testSize() { @Test public final void testJson() { Datum d = DatumFactory.createInet4("163.152.163.152"); - String json = d.toJson(); + String json = d.toJson(SerializeOption.GENERIC); Datum fromJson = CommonGsonHelper.fromJson(json, Datum.class); assertTrue(d.equalsTo(fromJson).asBool()); } diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java index ea641ecf7b..cc5cfeb2ca 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimeDatum.java @@ -18,6 +18,7 @@ package org.apache.tajo.datum; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidCastException; @@ -84,7 +85,7 @@ public final void testAsTextBytes() { @Test public final void testToJson() { Datum d = DatumFactory.createTime(TIME); - Datum copy = CommonGsonHelper.fromJson(d.toJson(), Datum.class); + Datum copy = CommonGsonHelper.fromJson(d.toJson(SerializeOption.GENERIC), Datum.class); assertEquals(d, copy); } diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java index 06e50f7563..5f57fd7bf9 100644 --- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java +++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java @@ -18,6 +18,7 @@ package org.apache.tajo.datum; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.InvalidCastException; import org.apache.tajo.json.CommonGsonHelper; @@ -112,7 +113,7 @@ public final void testAsTextBytes() { @Test public final void testToJson() { Datum d = DatumFactory.createTimestmpDatumWithUnixTime(unixtime); - Datum copy = CommonGsonHelper.fromJson(d.toJson(), Datum.class); + Datum copy = CommonGsonHelper.fromJson(d.toJson(SerializeOption.GENERIC), Datum.class); assertEquals(d, copy); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java index c8197b7078..b2f46be149 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.codegen; import com.google.common.collect.Maps; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -75,22 +76,22 @@ public void emitMemberFields() { } public static void emitCreateSchema(TajoGeneratorAdapter adapter, MethodVisitor mv, Schema schema) { - mv.visitLdcInsn(schema.toJson()); + mv.visitLdcInsn(schema.toJson(SerializeOption.GENERIC)); adapter.invokeStatic(EvalCodeGenerator.class, "createSchema", Schema.class, new Class[] {String.class}); } public static void emitCreateEval(TajoGeneratorAdapter adapter, MethodVisitor mv, EvalNode evalNode) { - mv.visitLdcInsn(evalNode.toJson()); + mv.visitLdcInsn(evalNode.toJson(SerializeOption.GENERIC)); adapter.invokeStatic(EvalCodeGenerator.class, "createEval", EvalNode.class, new Class[] {String.class}); } public static void emitConstEval(TajoGeneratorAdapter adapter, MethodVisitor mv, ConstEval evalNode) { - mv.visitLdcInsn(evalNode.toJson()); + mv.visitLdcInsn(evalNode.toJson(SerializeOption.GENERIC)); adapter.invokeStatic(EvalCodeGenerator.class, "createConstEval", ConstEval.class, new Class[] {String.class}); } public static void emitRowConstantEval(TajoGeneratorAdapter adapter, MethodVisitor mv, RowConstantEval evalNode) { - mv.visitLdcInsn(evalNode.toJson()); + mv.visitLdcInsn(evalNode.toJson(SerializeOption.GENERIC)); adapter.invokeStatic(EvalCodeGenerator.class, "createRowConstantEval", RowConstantEval.class, new Class[] {String.class}); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index ac1c9adb5f..1d36211f16 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.SortSpec; @@ -914,14 +915,15 @@ public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, St if (broadcastFlag) { PartitionedTableScanNode partitionedTableScanNode = (PartitionedTableScanNode) scanNode; - List fileFragments = TUtil.newList(); + List fileFragments = TUtil.newList(); FileStorageManager fileStorageManager = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); for (Path path : partitionedTableScanNode.getInputPaths()) { fileFragments.addAll(TUtil.newList(fileStorageManager.split(scanNode.getCanonicalName(), path))); } FragmentProto[] fragments = - FragmentConvertor.toFragmentProtoArray(fileFragments.toArray(new FileFragment[fileFragments.size()])); + FragmentConvertor.toFragmentProtoArray( + SerializeOption.INTERNAL, fileFragments.toArray(new FileFragment[fileFragments.size()])); ctx.addFragments(scanNode.getCanonicalName(), fragments); return new PartitionMergeScanExec(ctx, scanNode, fragments); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java index 8128390933..8c538be300 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.planner.enforce; +import org.apache.tajo.SerializeOption; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -78,7 +79,7 @@ public void addSortedInput(String tableName, SortSpec[] sortSpecs) { SortedInputEnforce.Builder enforce = SortedInputEnforce.newBuilder(); enforce.setTableName(tableName); for (SortSpec sortSpec : sortSpecs) { - enforce.addSortSpecs(sortSpec.getProto()); + enforce.addSortSpecs(sortSpec.getProto(SerializeOption.INTERNAL)); } builder.setType(EnforceType.SORTED_INPUT); @@ -113,7 +114,7 @@ public void enforceSortAggregation(int pid, @Nullable SortSpec[] sortSpecs) { enforce.setAlgorithm(GroupbyAlgorithm.SORT_AGGREGATION); if (sortSpecs != null) { for (SortSpec sortSpec : sortSpecs) { - enforce.addSortSpecs(sortSpec.getProto()); + enforce.addSortSpecs(sortSpec.getProto(SerializeOption.INTERNAL)); } } @@ -238,7 +239,7 @@ public String toString() { } @Override - public EnforcerProto getProto() { + public EnforcerProto getProto(SerializeOption option) { EnforcerProto.Builder builder = EnforcerProto.newBuilder(); builder.addAllProperties(getProperties()); return builder.build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java index 11548d35c1..76c212a54d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.planner.global; import com.google.common.base.Preconditions; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; @@ -152,7 +153,7 @@ public StoreType getStoreType() { return storeType; } - public DataChannelProto getProto() { + public DataChannelProto getProto(SerializeOption option) { DataChannelProto.Builder builder = DataChannelProto.newBuilder(); builder.setSrcId(srcId.getProto()); builder.setTargetId(targetId.getProto()); @@ -161,11 +162,11 @@ public DataChannelProto getProto() { } builder.setShuffleType(shuffleType); if (schema != null) { - builder.setSchema(schema.getProto()); + builder.setSchema(schema.getProto(option)); } if (shuffleKeys != null) { for (Column column : shuffleKeys) { - builder.addShuffleKeys(column.getProto()); + builder.addShuffleKeys(column.getProto(option)); } } if (numOutputs != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index b5e91046e6..68e28bcfc4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; @@ -537,12 +538,12 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou Target target = new Target(new FieldEval(column)); firstGroupbyTargets.add(target); } - firstStageGroupbyNode.setTargets(firstGroupbyTargets.toArray(new Target[]{})); + firstStageGroupbyNode.setTargets(firstGroupbyTargets.toArray(new Target[firstGroupbyTargets.size()])); // SecondStage: // Set grouping column with origin groupby's columns // Remove distinct group column from targets - secondStageGroupbyNode.setGroupingColumns(originGroupColumns.toArray(new Column[]{})); + secondStageGroupbyNode.setGroupingColumns(originGroupColumns.toArray(new Column[originGroupColumns.size()])); Target[] oldTargets = secondStageGroupbyNode.getTargets(); List secondGroupbyTargets = new ArrayList(); @@ -573,7 +574,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou } columnIdIndex++; } - secondStageGroupbyNode.setTargets(secondGroupbyTargets.toArray(new Target[]{})); + secondStageGroupbyNode.setTargets(secondGroupbyTargets.toArray(new Target[secondGroupbyTargets.size()])); } else { // FirstStage: Change target of aggFunction to function name expr List firstGroupbyTargets = new ArrayList(); @@ -602,7 +603,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou columnIdIndex++; aggFuncIdx++; } - firstStageGroupbyNode.setTargets(firstGroupbyTargets.toArray(new Target[]{})); + firstStageGroupbyNode.setTargets(firstGroupbyTargets.toArray(new Target[firstGroupbyTargets.size()])); secondStageGroupbyNode.setInSchema(firstStageGroupbyNode.getOutSchema()); } grpIdx++; @@ -661,7 +662,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou } } } - firstStageDistinctNode.setTargets(firstTargets.toArray(new Target[]{})); + firstStageDistinctNode.setTargets(firstTargets.toArray(new Target[firstTargets.size()])); firstStageDistinctNode.setResultColumnIds(TUtil.toArray(firstStageColumnIds)); //Set SecondStage ColumnId and Input schema @@ -694,7 +695,7 @@ private void setDistinctAggregationEnforcer( for (GroupbyNode groupbyNode: firstStageDistinctNode.getSubPlans()) { List sortSpecs = new ArrayList(); for (Column column: groupbyNode.getGroupingColumns()) { - sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); + sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto(SerializeOption.INTERNAL)).build()); } sortSpecArrays.add( SortSpecArray.newBuilder() .setNodeId(secondStageDistinctNode.getSubPlans().get(index).getPID()) @@ -722,7 +723,7 @@ private void setMultiStageAggregationEnforcer( for (GroupbyNode groupbyNode: firstStageDistinctNode.getSubPlans()) { List sortSpecs = new ArrayList(); for (Column column: groupbyNode.getGroupingColumns()) { - sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); + sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto(SerializeOption.INTERNAL)).build()); } sortSpecArrays.add( SortSpecArray.newBuilder() .setNodeId(thirdStageDistinctNode.getSubPlans().get(index).getPID()) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java index e2fd47f944..ceabde9659 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanEqualityTester.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.global.rewriter.rules; +import org.apache.tajo.SerializeOption; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; import org.apache.tajo.engine.planner.global.MasterPlan; @@ -50,7 +51,7 @@ public MasterPlan rewrite(MasterPlan plan) { ExecutionBlock eb = cursor.nextBlock(); LogicalNode node = eb.getPlan(); if (node != null) { - PlanProto.LogicalNodeTree tree = LogicalNodeSerializer.serialize(node); + PlanProto.LogicalNodeTree tree = LogicalNodeSerializer.serialize(node, SerializeOption.GENERIC); LogicalNode deserialize = LogicalNodeDeserializer.deserialize(plan.getContext(), tree); assert node.deepEquals(deserialize); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 247b37305c..705c4b991b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; @@ -116,7 +117,8 @@ public static CatalogProtos.FragmentProto[] getNonZeroLengthDataFiles(TajoConf t fragments.add(fileFragment); } } - return FragmentConvertor.toFragmentProtoArray(fragments.toArray(new FileFragment[]{})); + return FragmentConvertor.toFragmentProtoArray( + SerializeOption.INTERNAL, fragments.toArray(new FileFragment[fragments.size()])); } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 7b3c00db30..4f063dbb20 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; +import org.apache.tajo.SerializeOption; import org.apache.tajo.OverridableConf; import org.apache.tajo.QueryVars; import org.apache.tajo.SessionVars; @@ -107,7 +108,8 @@ public boolean hasPartition() { } public void setPartitionMethod(PartitionMethodDesc partitionMethodDesc) { - put(QueryVars.OUTPUT_PARTITIONS, partitionMethodDesc != null ? partitionMethodDesc.toJson() : null); + put(QueryVars.OUTPUT_PARTITIONS, + partitionMethodDesc != null ? partitionMethodDesc.toJson(SerializeOption.GENERIC) : null); } // public PartitionMethodDesc getPartitionMethod() { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index b4727dcff3..b4bf607586 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.query; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.enforce.Enforcer; @@ -89,8 +90,8 @@ public void set(TaskAttemptId id, List fragments, } @Override - public TaskRequestProto getProto() { - mergeLocalToProto(); + public TaskRequestProto getProto(SerializeOption option) { + mergeLocalToProto(option); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; @@ -115,9 +116,7 @@ public List getFragments() { if (fragments != null) { return fragments; } - if (fragments == null) { - fragments = new ArrayList(); - } + fragments = new ArrayList(p.getFragmentsCount()); for (int i = 0; i < p.getFragmentsCount(); i++) { fragments.add(p.getFragments(i)); } @@ -279,7 +278,7 @@ private void maybeInitBuilder() { viaProto = true; } - private void mergeLocalToBuilder() { + private void mergeLocalToBuilder(SerializeOption option) { if (id != null) { builder.setId(this.id.getProto()); } @@ -302,28 +301,28 @@ private void mergeLocalToBuilder() { } if (this.fetches != null) { for (int i = 0; i < fetches.size(); i++) { - builder.addFetches(fetches.get(i).getProto()); + builder.addFetches(fetches.get(i).getProto(option)); } } if (this.shouldDie != null) { builder.setShouldDie(this.shouldDie); } if (this.queryContext != null) { - builder.setQueryContext(queryContext.getProto()); + builder.setQueryContext(queryContext.getProto(option)); } if (this.dataChannel != null) { - builder.setDataChannel(dataChannel.getProto()); + builder.setDataChannel(dataChannel.getProto(option)); } if (this.enforcer != null) { - builder.setEnforcer(enforcer.getProto()); + builder.setEnforcer(enforcer.getProto(option)); } } - private void mergeLocalToProto() { + private void mergeLocalToProto(SerializeOption option) { if(viaProto) { maybeInitBuilder(); } - mergeLocalToBuilder(); + mergeLocalToBuilder(option); proto = builder.build(); viaProto = true; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java index 1b1d49e499..088c31e474 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.service.AbstractService; +import org.apache.tajo.SerializeOption; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ContainerProtocol; import org.apache.tajo.ipc.QueryCoordinatorProtocol; @@ -148,7 +149,7 @@ public void getAllWorkerResource(RpcController controller, PrimitiveProtos.NullP WorkerResourceProto.Builder workerResource = WorkerResourceProto.newBuilder(); - workerResource.setConnectionInfo(worker.getConnectionInfo().getProto()); + workerResource.setConnectionInfo(worker.getConnectionInfo().getProto(SerializeOption.INTERNAL)); workerResource.setMemoryMB(resource.getMemoryMB()); workerResource.setDiskSlots(resource.getDiskSlots()); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java index 668a770a5a..4c799849ff 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java @@ -21,6 +21,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.StringUtils; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.TajoProtos; import org.apache.tajo.engine.query.QueryContext; @@ -187,10 +188,11 @@ public void submitQueryToMaster() { QueryExecutionRequestProto.Builder builder = TajoWorkerProtocol.QueryExecutionRequestProto.newBuilder(); builder.setQueryId(queryId.getProto()) - .setQueryContext(queryInfo.getQueryContext().getProto()) - .setSession(session.getProto()) + .setQueryContext(queryInfo.getQueryContext().getProto(SerializeOption.INTERNAL)) + .setSession(session.getProto(SerializeOption.INTERNAL)) .setExprInJson(PrimitiveProtos.StringProto.newBuilder().setValue(queryInfo.getJsonExpr())) - .setLogicalPlanJson(PrimitiveProtos.StringProto.newBuilder().setValue(plan.toJson()).build()); + .setLogicalPlanJson(PrimitiveProtos.StringProto.newBuilder().setValue(plan.toJson(SerializeOption.INTERNAL)) + .build()); queryMasterRpcClient.executeQuery(null, builder.build(), NullCallback.get()); querySubmitted.set(true); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInfo.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInfo.java index 38e9403131..ffe5263ad2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInfo.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInfo.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.TajoProtos; import org.apache.tajo.catalog.TableDesc; @@ -182,7 +183,7 @@ public String getJsonExpr() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CoreGsonHelper.toJson(this, QueryInfo.class); } @@ -201,12 +202,12 @@ public String getQueryIdStr() { return queryIdStr; } - public QueryInfoProto getProto() { + public QueryInfoProto getProto(SerializeOption option) { QueryInfoProto.Builder builder = QueryInfoProto.newBuilder(); builder.setQueryId(queryId.toString()) .setQueryState(queryState) - .setContextVars(context.getProto()) + .setContextVars(context.getProto(option)) .setProgress(progress) .setStartTime(startTime) .setFinishTime(finishTime) @@ -215,7 +216,7 @@ public QueryInfoProto getProto() { .setQueryMasterInfoPort(queryMasterInfoPort); if (resultDesc != null) { - builder.setResultDesc(resultDesc.getProto()); + builder.setResultDesc(resultDesc.getProto(option)); } if (sql != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java index 7ed9fc5aa0..65545a16c3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java @@ -21,11 +21,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.TaskAttemptId; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.ha.HAServiceUtil; import org.apache.tajo.ipc.ContainerProtocol; import org.apache.tajo.ipc.QueryCoordinatorProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -108,11 +107,11 @@ private void assignExecutionBlock(ExecutionBlockId executionBlockId, TajoContain TajoWorkerProtocol.RunExecutionBlockRequestProto request = TajoWorkerProtocol.RunExecutionBlockRequestProto.newBuilder() .setExecutionBlockId(executionBlockId.getProto()) - .setQueryMaster(context.getQueryMasterContext().getWorkerContext().getConnectionInfo().getProto()) + .setQueryMaster(context.getQueryMasterContext().getWorkerContext().getConnectionInfo().getProto(SerializeOption.INTERNAL)) .setNodeId(container.getNodeId().toString()) .setContainerId(container.getId().toString()) .setQueryOutputPath(context.getStagingDir().toString()) - .setQueryContext(queryContext.getProto()) + .setQueryContext(queryContext.getProto(SerializeOption.INTERNAL)) .setPlanJson(planJson) .build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 4fcdc888e0..61b51a0030 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.TajoIdProtos; @@ -177,7 +178,7 @@ public BoolProto removeSession(RpcController controller, TajoIdProtos.SessionIdP public SessionUpdateResponse buildSessionUpdateOnSuccess(Map variables) { SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder(); builder.setResultCode(ResultCode.OK); - builder.setSessionVars(new KeyValueSet(variables).getProto()); + builder.setSessionVars(new KeyValueSet(variables).getProto(SerializeOption.GENERIC)); return builder.build(); } @@ -242,7 +243,7 @@ public KeyValueSetProto getAllSessionVariables(RpcController controller, String sessionId = request.getId(); KeyValueSet keyValueSet = new KeyValueSet(); keyValueSet.putAll(context.getSessionManager().getAllVariables(sessionId)); - return keyValueSet.getProto(); + return keyValueSet.getProto(SerializeOption.GENERIC); } catch (Throwable t) { throw new ServiceException(t); } @@ -361,7 +362,7 @@ public GetQueryResultResponse getQueryResult(RpcController controller, switch (queryInfo.getQueryState()) { case QUERY_SUCCEEDED: if (queryInfo.hasResultdesc()) { - builder.setTableDesc(queryInfo.getResultDesc().getProto()); + builder.setTableDesc(queryInfo.getResultDesc().getProto(SerializeOption.GENERIC)); } break; case QUERY_FAILED: @@ -550,7 +551,7 @@ public GetQueryResultDataResponse getQueryResultData(RpcController controller, G List rows = queryResultScanner.getNextRows(request.getFetchRowNum()); - resultSetBuilder.setSchema(queryResultScanner.getLogicalSchema().getProto()); + resultSetBuilder.setSchema(queryResultScanner.getLogicalSchema().getProto(SerializeOption.GENERIC)); resultSetBuilder.addAllSerializedTuples(rows); builder.setResultSet(resultSetBuilder.build()); @@ -603,7 +604,7 @@ public GetQueryInfoResponse getQueryInfo(RpcController controller, QueryIdReques } if (queryInfo != null) { - builder.setQueryInfo(queryInfo.getProto()); + builder.setQueryInfo(queryInfo.getProto(SerializeOption.GENERIC)); } builder.setResultCode(ResultCode.OK); } catch (Throwable t) { @@ -652,7 +653,7 @@ public GetClusterInfoResponse getClusterInfo(RpcController controller, for(Worker worker: workers.values()) { WorkerResource workerResource = worker.getResource(); - workerBuilder.setConnectionInfo(worker.getConnectionInfo().getProto()); + workerBuilder.setConnectionInfo(worker.getConnectionInfo().getProto(SerializeOption.GENERIC)); workerBuilder.setDiskSlots(workerResource.getDiskSlots()); workerBuilder.setCpuCoreSlots(workerResource.getCpuCoreSlots()); workerBuilder.setMemoryMB(workerResource.getMemoryMB()); @@ -802,7 +803,7 @@ public TableResponse getTableDesc(RpcController controller, GetTableDescRequest if (catalog.existsTable(databaseName, tableName)) { return TableResponse.newBuilder() .setResultCode(ResultCode.OK) - .setTableDesc(catalog.getTableDesc(databaseName, tableName).getProto()) + .setTableDesc(catalog.getTableDesc(databaseName, tableName).getProto(SerializeOption.GENERIC)) .build(); } else { return TableResponse.newBuilder() @@ -849,7 +850,7 @@ public TableResponse createExternalTable(RpcController controller, CreateTableRe return TableResponse.newBuilder() .setResultCode(ResultCode.OK) - .setTableDesc(desc.getProto()).build(); + .setTableDesc(desc.getProto(SerializeOption.GENERIC)).build(); } catch (InvalidSessionException ise) { return TableResponse.newBuilder() .setResultCode(ResultCode.ERROR) @@ -889,10 +890,10 @@ public FunctionResponse getFunctionList(RpcController controller, SessionedStrin for (FunctionDesc eachFunction: functions) { if (functionName == null || functionName.isEmpty()) { - functionProtos.add(eachFunction.getProto()); + functionProtos.add(eachFunction.getProto(SerializeOption.GENERIC)); } else { if(functionName.equals(eachFunction.getFunctionName())) { - functionProtos.add(eachFunction.getProto()); + functionProtos.add(eachFunction.getProto(SerializeOption.GENERIC)); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java b/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java index 78d4978908..b0ea4060ee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java @@ -18,6 +18,7 @@ package org.apache.tajo.master.cluster; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import static org.apache.tajo.TajoProtos.WorkerConnectionInfoProto; @@ -112,7 +113,7 @@ public String getHostAndPeerRpcPort() { } @Override - public WorkerConnectionInfoProto getProto() { + public WorkerConnectionInfoProto getProto(SerializeOption option) { WorkerConnectionInfoProto.Builder builder = WorkerConnectionInfoProto.newBuilder(); builder.setId(id) .setHost(host) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 9264c68a01..c270f1ef1a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; @@ -102,7 +103,7 @@ public boolean execute(QueryContext queryContext, LogicalPlan plan) throws IOExc return true; default: - throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson()); + throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson(SerializeOption.GENERIC)); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java index ab37e22112..2e57c90991 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java @@ -18,6 +18,7 @@ package org.apache.tajo.master.exec; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.plan.LogicalPlan; @@ -182,7 +183,7 @@ private static class EvalNodeComparator implements Comparator { @Override public int compare(EvalNode o1, EvalNode o2) { - return o1.toJson().compareTo(o2.toJson()); + return o1.toJson(SerializeOption.GENERIC).compareTo(o2.toJson(SerializeOption.GENERIC)); } } @@ -190,7 +191,7 @@ private static class TargetComparator implements Comparator { @Override public int compare(Target o1, Target o2) { - return o1.toJson().compareTo(o2.toJson()); + return o1.toJson(SerializeOption.GENERIC).compareTo(o2.toJson(SerializeOption.GENERIC)); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index 804821bf77..f64ca140a3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -22,6 +22,7 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryId; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.TaskId; import org.apache.tajo.catalog.Column; @@ -110,7 +111,8 @@ private void initSeqScanExec() throws IOException { fragments = storageManager.getNonForwardSplit(tableDesc, currentFragmentIndex, MAX_FRAGMENT_NUM_PER_SCAN); if (fragments != null && !fragments.isEmpty()) { - FragmentProto[] fragmentProtos = FragmentConvertor.toFragmentProtoArray(fragments.toArray(new Fragment[] {})); + FragmentProto[] fragmentProtos = FragmentConvertor.toFragmentProtoArray( + SerializeOption.INTERNAL, fragments.toArray(new Fragment[fragments.size()])); this.taskContext = new TaskAttemptContext( new QueryContext(tajoConf), null, new TaskAttemptId(new TaskId(new ExecutionBlockId(queryId, 1), 0), 0), diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 2eb3c5f752..1612c5b375 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.SessionVars; @@ -197,7 +198,7 @@ public void execExplain(LogicalPlan plan, QueryContext queryContext, boolean isG bytesNum += encodedData.length; serializedResBuilder.addSerializedTuples(ByteString.copyFrom(encodedData)); } - serializedResBuilder.setSchema(schema.getProto()); + serializedResBuilder.setSchema(schema.getProto(SerializeOption.GENERIC)); serializedResBuilder.setBytesNum(bytesNum); response.setResultSet(serializedResBuilder.build()); @@ -223,7 +224,7 @@ public void execQueryOnVirtualTable(QueryContext queryContext, Session session, response.setQueryId(queryId.getProto()); response.setMaxRowNum(maxRow); - response.setTableDesc(queryResultScanner.getTableDesc().getProto()); + response.setTableDesc(queryResultScanner.getTableDesc().getProto(SerializeOption.GENERIC)); response.setResultCode(ClientProtos.ResultCode.OK); } @@ -259,7 +260,7 @@ public void execSimpleQuery(QueryContext queryContext, Session session, String q response.setQueryId(queryInfo.getQueryId().getProto()); response.setMaxRowNum(maxRow); - response.setTableDesc(desc.getProto()); + response.setTableDesc(desc.getProto(SerializeOption.GENERIC)); response.setResultCode(ClientProtos.ResultCode.OK); } @@ -287,7 +288,7 @@ public void execNonFromQuery(QueryContext queryContext, Session session, String byte[] serializedBytes = encoder.toBytes(outTuple); ClientProtos.SerializedResultSet.Builder serializedResBuilder = ClientProtos.SerializedResultSet.newBuilder(); serializedResBuilder.addSerializedTuples(ByteString.copyFrom(serializedBytes)); - serializedResBuilder.setSchema(schema.getProto()); + serializedResBuilder.setSchema(schema.getProto(SerializeOption.GENERIC)); serializedResBuilder.setBytesNum(serializedBytes.length); responseBuilder.setResultSet(serializedResBuilder); @@ -369,11 +370,11 @@ private void insertNonFromQuery(QueryContext queryContext, CatalogProtos.UpdateTableStatsProto.Builder builder = CatalogProtos.UpdateTableStatsProto.newBuilder(); builder.setTableName(tableDesc.getName()); - builder.setStats(stats.getProto()); + builder.setStats(stats.getProto(SerializeOption.GENERIC)); catalog.updateTableStats(builder.build()); - responseBuilder.setTableDesc(tableDesc.getProto()); + responseBuilder.setTableDesc(tableDesc.getProto(SerializeOption.GENERIC)); } else { TableStats stats = new TableStats(); long volume = Query.getTableVolume(context.getConf(), finalOutputDir); @@ -386,7 +387,7 @@ private void insertNonFromQuery(QueryContext queryContext, .setTableName(nodeUniqName) .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(CatalogProtos.StoreType.CSV).build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) - .setStats(stats.getProto()) + .setStats(stats.getProto(SerializeOption.GENERIC)) .build(); responseBuilder.setTableDesc(tableDescProto); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java index 091da2f3e2..339239fe78 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ContainerProtocol; @@ -332,7 +333,7 @@ public void run() { ContainerProtocol.TajoContainerIdProto containerIdProto = containerId.getProto(); allocatedResources.add(WorkerAllocatedResource.newBuilder() .setContainerId(containerIdProto) - .setConnectionInfo(allocatedResource.worker.getConnectionInfo().getProto()) + .setConnectionInfo(allocatedResource.worker.getConnectionInfo().getProto(SerializeOption.INTERNAL)) .setAllocatedMemoryMB(allocatedResource.allocatedMemoryMB) .setAllocatedDiskSlots(allocatedResource.allocatedDiskSlots) .build()); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 351856fde9..b726ce90f4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.util.RackResolver; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.TaskAttemptId; @@ -195,10 +196,10 @@ public void handle(TaskSchedulerEvent event) { if (context.isLeafQuery()) { TaskAttemptScheduleContext taskContext = new TaskAttemptScheduleContext(); Task task = Stage.newEmptyTask(context, taskContext, stage, nextTaskId++); - task.addFragment(castEvent.getLeftFragment(), true); + task.addFragment(castEvent.getLeftFragment(), SerializeOption.INTERNAL, true); scheduledObjectNum++; if (castEvent.hasRightFragments()) { - task.addFragments(castEvent.getRightFragments()); + task.addFragments(castEvent.getRightFragments(), SerializeOption.INTERNAL); } stage.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE)); } else { @@ -223,13 +224,13 @@ public void handle(TaskSchedulerEvent event) { scheduledObjectNum++; for (Entry> eachFetch : fetches.entrySet()) { task.addFetches(eachFetch.getKey(), eachFetch.getValue()); - task.addFragment(fragmentsForNonLeafTask[0], true); + task.addFragment(fragmentsForNonLeafTask[0], SerializeOption.INTERNAL, true); if (fragmentsForNonLeafTask[1] != null) { - task.addFragment(fragmentsForNonLeafTask[1], true); + task.addFragment(fragmentsForNonLeafTask[1], SerializeOption.INTERNAL, true); } } if (broadcastFragmentsForNonLeafTask != null && broadcastFragmentsForNonLeafTask.length > 0) { - task.addFragments(Arrays.asList(broadcastFragmentsForNonLeafTask)); + task.addFragments(Arrays.asList(broadcastFragmentsForNonLeafTask), SerializeOption.INTERNAL); } stage.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE)); } else if (event instanceof TaskAttemptToSchedulerEvent) { @@ -840,7 +841,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { new ArrayList(task.getAllFragments()), "", false, - LogicalNodeSerializer.serialize(task.getLogicalPlan()), + LogicalNodeSerializer.serialize(task.getLogicalPlan(), SerializeOption.INTERNAL), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer()); if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) { @@ -852,7 +853,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { assignedRequest.add(attemptId); scheduledObjectNum--; - taskRequest.getCallback().run(taskAssign.getProto()); + taskRequest.getCallback().run(taskAssign.getProto(SerializeOption.INTERNAL)); } else { throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!"); } @@ -896,7 +897,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { Lists.newArrayList(task.getAllFragments()), "", false, - LogicalNodeSerializer.serialize(task.getLogicalPlan()), + LogicalNodeSerializer.serialize(task.getLogicalPlan(), SerializeOption.INTERNAL), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer()); @@ -916,7 +917,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { getWorkerConnectionInfo(taskRequest.getWorkerId()); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, taskRequest.getContainerId(), connectionInfo)); - taskRequest.getCallback().run(taskAssign.getProto()); + taskRequest.getCallback().run(taskAssign.getProto(SerializeOption.INTERNAL)); totalAssigned++; scheduledObjectNum--; } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 1ce15fccef..09de0e8fd7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.state.*; import org.apache.hadoop.yarn.util.Clock; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryId; import org.apache.tajo.SessionVars; @@ -599,7 +600,7 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo if (insertNode.hasTargetTable()) { UpdateTableStatsProto.Builder builder = UpdateTableStatsProto.newBuilder(); builder.setTableName(finalTable.getName()); - builder.setStats(stats.getProto()); + builder.setStats(stats.getProto(SerializeOption.GENERIC)); catalog.updateTableStats(builder.build()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 9cbfb95cc9..bb9e774d7d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -382,12 +382,12 @@ public void stopQuery(QueryId queryId) { private TajoHeartbeat buildTajoHeartBeat(QueryMasterTask queryMasterTask) { TajoHeartbeat.Builder builder = TajoHeartbeat.newBuilder(); - builder.setConnectionInfo(workerContext.getConnectionInfo().getProto()); + builder.setConnectionInfo(workerContext.getConnectionInfo().getProto(SerializeOption.INTERNAL)); builder.setQueryId(queryMasterTask.getQueryId().getProto()); builder.setState(queryMasterTask.getState()); if (queryMasterTask.getQuery() != null) { if (queryMasterTask.getQuery().getResultDesc() != null) { - builder.setResultDesc(queryMasterTask.getQuery().getResultDesc().getProto()); + builder.setResultDesc(queryMasterTask.getQuery().getResultDesc().getProto(SerializeOption.INTERNAL)); } builder.setQueryProgress(queryMasterTask.getQuery().getProgress()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index 1da623e89a..2d11128c87 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.state.*; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.TajoProtos.TaskAttemptState; @@ -332,7 +333,7 @@ private void addDataLocation(Fragment fragment) { } } - public void addFragment(Fragment fragment, boolean useDataLocation) { + public void addFragment(Fragment fragment, SerializeOption option, boolean useDataLocation) { Set fragmentProtos; if (fragMap.containsKey(fragment.getTableName())) { fragmentProtos = fragMap.get(fragment.getTableName()); @@ -340,16 +341,16 @@ public void addFragment(Fragment fragment, boolean useDataLocation) { fragmentProtos = new HashSet(); fragMap.put(fragment.getTableName(), fragmentProtos); } - fragmentProtos.add(fragment.getProto()); + fragmentProtos.add(fragment.getProto(option)); if (useDataLocation) { addDataLocation(fragment); } totalFragmentNum++; } - public void addFragments(Collection fragments) { + public void addFragments(Collection fragments, SerializeOption option) { for (Fragment eachFragment: fragments) { - addFragment(eachFragment, false); + addFragment(eachFragment, option, false); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/session/Session.java index 7ac4f858c8..d785900b6e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/session/Session.java +++ b/tajo-core/src/main/java/org/apache/tajo/session/Session.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.SessionVars; import org.apache.tajo.master.exec.NonForwardQueryResultScanner; @@ -122,7 +123,7 @@ public synchronized String getCurrentDatabase() { } @Override - public SessionProto getProto() { + public SessionProto getProto(SerializeOption option) { SessionProto.Builder builder = SessionProto.newBuilder(); builder.setSessionId(getSessionId()); builder.setUsername(getUserName()); @@ -132,7 +133,7 @@ public SessionProto getProto() { synchronized (sessionVariables) { variables.putAll(this.sessionVariables); - builder.setVariables(variables.getProto()); + builder.setVariables(variables.getProto(option)); return builder.build(); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java index 875d12bab7..97bfe20d34 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java @@ -19,6 +19,7 @@ package org.apache.tajo.util; import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; @@ -332,7 +333,7 @@ public static String percentFormat(float value) { public static String tableStatToString(TableStats tableStats) { if(tableStats != null){ - return tableStatToString(tableStats.getProto()); + return tableStatToString(tableStats.getProto(SerializeOption.GENERIC)); } else { return "No input statistics"; diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java index f0c6c1134f..d40f4a2d21 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.service.AbstractService; +import org.apache.tajo.SerializeOption; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.master.QueryInfo; import org.apache.tajo.util.Bytes; @@ -361,7 +362,7 @@ private synchronized void writeQueryHistory(QueryHistory queryHistory) throws Ex try { LOG.info("Saving query summary: " + queryHistoryFile); out = fs.create(queryHistoryFile, queryReplication); - out.write(queryHistory.toJson().getBytes(Bytes.UTF8_CHARSET)); + out.write(queryHistory.toJson(SerializeOption.GENERIC).getBytes(Bytes.UTF8_CHARSET)); } finally { IOUtils.cleanup(LOG, out); } @@ -397,7 +398,7 @@ private synchronized WriterHolder writeQuerySummary(QueryInfo queryInfo) throws rollingQuerySummaryWriter(); } } - byte[] jsonBytes = ("\n" + queryInfo.toJson() + "\n").getBytes(Bytes.UTF8_CHARSET); + byte[] jsonBytes = ("\n" + queryInfo.toJson(SerializeOption.GENERIC) + "\n").getBytes(Bytes.UTF8_CHARSET); try { querySummaryWriter.out.writeInt(jsonBytes.length); querySummaryWriter.out.write(jsonBytes); @@ -459,7 +460,7 @@ private synchronized WriterHolder writeTaskHistory(TaskHistory taskHistory) thro if (writerHolder.out != null) { try { - byte[] taskHistoryBytes = taskHistory.getProto().toByteArray(); + byte[] taskHistoryBytes = taskHistory.getProto(SerializeOption.GENERIC).toByteArray(); writerHolder.out.writeInt(taskHistoryBytes.length); writerHolder.out.write(taskHistoryBytes); } catch (IOException ie) { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java index fdc45a1dbe..979ead6b6a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java @@ -19,6 +19,7 @@ package org.apache.tajo.util.history; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.ipc.ClientProtos.QueryHistoryProto; @@ -102,7 +103,7 @@ public void setDistributedPlan(String distributedPlan) { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CoreGsonHelper.toJson(this, QueryHistory.class); } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java index e760f86912..4c70f5f188 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java @@ -20,6 +20,7 @@ import com.google.gson.annotations.Expose; import com.google.gson.reflect.TypeToken; +import org.apache.tajo.SerializeOption; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.ipc.ClientProtos.StageHistoryProto; import org.apache.tajo.json.GsonObject; @@ -222,7 +223,7 @@ public void setTasks(List tasks) { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CoreGsonHelper.toJson(this, StageHistory.class); } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java index 35c56e1d67..f361a3b2c0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java @@ -19,6 +19,7 @@ package org.apache.tajo.util.history; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.json.GsonObject; @@ -153,7 +154,7 @@ public void setFetchs(String[][] fetchs) { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CoreGsonHelper.toJson(this, TaskHistory.class); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java index 89c34046d7..50235050c2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -112,7 +113,7 @@ public int hashCode() { } @Override - public TajoWorkerProtocol.FetchProto getProto() { + public TajoWorkerProtocol.FetchProto getProto(SerializeOption option) { TajoWorkerProtocol.FetchProto.Builder builder = TajoWorkerProtocol.FetchProto.newBuilder(); builder.setHost(host.getHost()); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index b08af2b3a7..98eeab6abd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.TajoProtos; import org.apache.tajo.TajoProtos.TaskAttemptState; @@ -152,7 +153,7 @@ public void initPlan() throws IOException { } } - interQuery = request.getProto().getInterQuery(); + interQuery = request.getProto(SerializeOption.GENERIC).getInterQuery(); if (interQuery) { context.setInterQuery(); this.shuffleType = context.getDataChannel().getShuffleType(); @@ -286,7 +287,7 @@ public TaskStatusProto getReport() { builder.setInputStats(reloadInputStats()); if (context.getResultStats() != null) { - builder.setResultStats(context.getResultStats().getProto()); + builder.setResultStats(context.getResultStats().getProto(SerializeOption.GENERIC)); } return builder.build(); } @@ -312,7 +313,7 @@ public void updateProgress() { private CatalogProtos.TableStatsProto reloadInputStats() { synchronized(inputStats) { if (this.executor == null) { - return inputStats.getProto(); + return inputStats.getProto(SerializeOption.GENERIC); } TableStats executorInputStats = this.executor.getInputStats(); @@ -320,7 +321,7 @@ private CatalogProtos.TableStatsProto reloadInputStats() { if (executorInputStats != null) { inputStats.setValues(executorInputStats); } - return inputStats.getProto(); + return inputStats.getProto(SerializeOption.GENERIC); } } @@ -331,9 +332,9 @@ private TaskCompletionReport getTaskCompletionReport() { builder.setInputStats(reloadInputStats()); if (context.hasResultStats()) { - builder.setResultStats(context.getResultStats().getProto()); + builder.setResultStats(context.getResultStats().getProto(SerializeOption.GENERIC)); } else { - builder.setResultStats(new TableStats().getProto()); + builder.setResultStats(new TableStats().getProto(SerializeOption.GENERIC)); } Iterator> it = context.getShuffleFileOutputs(); @@ -507,7 +508,7 @@ public TaskHistory createTaskHistory() { } if (context.getResultStats() != null) { - taskHistory.setOutputStats(context.getResultStats().getProto()); + taskHistory.setOutputStats(context.getResultStats().getProto(SerializeOption.GENERIC)); } if (hasFetchPhase()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 706e9b87f9..5fc415bb3c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TajoProtos.TaskAttemptState; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.catalog.Schema; @@ -129,7 +130,7 @@ public TaskAttemptContext(QueryContext queryContext, final ExecutionBlockContext @VisibleForTesting public TaskAttemptContext(final QueryContext queryContext, final TaskAttemptId queryId, final Fragment [] fragments, final Path workDir) { - this(queryContext, null, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir); + this(queryContext, null, queryId, FragmentConvertor.toFragmentProtoArray(SerializeOption.GENERIC, fragments), workDir); } public TajoConf getConf() { @@ -257,10 +258,10 @@ public void updateAssignedFragments(String tableId, Fragment[] fragments) { fragmentMap.remove(tableId); for(Fragment t : fragments) { if (fragmentMap.containsKey(t.getTableName())) { - fragmentMap.get(t.getTableName()).add(t.getProto()); + fragmentMap.get(t.getTableName()).add(t.getProto(SerializeOption.GENERIC)); } else { List frags = new ArrayList(); - frags.add(t.getProto()); + frags.add(t.getProto(SerializeOption.GENERIC)); fragmentMap.put(t.getTableName(), frags); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java index c2432ebaff..02a6b9dd83 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.common.collect.Lists; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; @@ -106,13 +107,13 @@ public int hashCode() { public boolean equals(Object o) { if (o instanceof TaskHistory) { TaskHistory other = (TaskHistory) o; - return getProto().equals(other.getProto()); + return getProto(SerializeOption.GENERIC).equals(other.getProto(SerializeOption.GENERIC)); } return false; } @Override - public TaskHistoryProto getProto() { + public TaskHistoryProto getProto(SerializeOption option) { TaskHistoryProto.Builder builder = TaskHistoryProto.newBuilder(); builder.setTaskAttemptId(taskAttemptId.getProto()); builder.setState(state); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java index 5c97ba81d8..cbc78121d8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.common.collect.Maps; import org.apache.hadoop.service.Service; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.TaskAttemptId; import org.apache.tajo.common.ProtoObject; @@ -82,13 +83,13 @@ public int hashCode() { public boolean equals(Object o) { if (o instanceof TaskRunnerHistory) { TaskRunnerHistory other = (TaskRunnerHistory) o; - return getProto().equals(other.getProto()); + return getProto(SerializeOption.GENERIC).equals(other.getProto(SerializeOption.GENERIC)); } return false; } @Override - public TaskRunnerHistoryProto getProto() { + public TaskRunnerHistoryProto getProto(SerializeOption option) { TaskRunnerHistoryProto.Builder builder = TaskRunnerHistoryProto.newBuilder(); builder.setContainerId(containerId.toString()); builder.setState(state.toString()); @@ -96,7 +97,7 @@ public TaskRunnerHistoryProto getProto() { builder.setStartTime(startTime); builder.setFinishTime(finishTime); for (TaskHistory taskHistory : taskHistoryMap.values()){ - builder.addTaskHistories(taskHistory.getProto()); + builder.addTaskHistories(taskHistory.getProto(option)); } return builder.build(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java index e9f90ca848..c245666877 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; +import org.apache.tajo.SerializeOption; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.QueryCoordinatorProtocol.ClusterResourceSummary; import org.apache.tajo.ipc.QueryCoordinatorProtocol.ServerStatusProto; @@ -172,7 +173,7 @@ public void run() { .build(); NodeHeartbeat heartbeatProto = NodeHeartbeat.newBuilder() - .setConnectionInfo(context.getConnectionInfo().getProto()) + .setConnectionInfo(context.getConnectionInfo().getProto(SerializeOption.INTERNAL)) .setServerStatus(serverStatus) .build(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index e2dac05750..454c76ad7a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -20,6 +20,7 @@ import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.OverridableConf; +import org.apache.tajo.SerializeOption; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; @@ -322,7 +323,7 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S } public static void assertEvalTreeProtoSerDer(OverridableConf context, EvalNode evalNode) { - PlanProto.EvalNodeTree converted = EvalNodeSerializer.serialize(evalNode); + PlanProto.EvalNodeTree converted = EvalNodeSerializer.serialize(evalNode, SerializeOption.GENERIC); assertEquals(evalNode, EvalNodeDeserializer.deserialize(context, converted)); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index a2d05989b5..72448869cb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.eval; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; @@ -355,7 +356,7 @@ public final void testJson() throws CloneNotSupportedException { BinaryEval gth = new BinaryEval(EvalType.GTH, e3, plus3); assertCloneEqual(gth); - String json = gth.toJson(); + String json = gth.toJson(SerializeOption.GENERIC); BinaryEval eval = (BinaryEval) CoreGsonHelper.fromJson(json, EvalNode.class); assertCloneEqual(eval); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 0b59bc7a35..00fc7bfc6e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.tajo.SerializeOption; import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; @@ -520,7 +521,7 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, PlanningEx for (Map.Entry entry : qualMap.entrySet()) { if (!entry.getValue().booleanValue()) { Preconditions.checkArgument(false, - "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); + "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson(SerializeOption.GENERIC)); } } } @@ -561,7 +562,7 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, PlanningEx for (Map.Entry entry : qualMap.entrySet()) { if (!entry.getValue().booleanValue()) { Preconditions.checkArgument(false, - "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson()); + "SelectionQual not found. -> required JoinQual:" + entry.getKey().toJson(SerializeOption.GENERIC)); } } } @@ -607,7 +608,7 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, PlanningEx for (Map.Entry entry : qualMap.entrySet()) { if (!entry.getValue().booleanValue()) { Preconditions.checkArgument(false, - "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); + "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson(SerializeOption.GENERIC)); } } } @@ -677,14 +678,14 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, PlanningEx for (Map.Entry entry : joinQualMap.entrySet()) { if (!entry.getValue().booleanValue()) { Preconditions.checkArgument(false, - "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson()); + "JoinQual not found. -> required JoinQual:" + entry.getKey().toJson(SerializeOption.GENERIC)); } } for (Map.Entry entry : scanMap.entrySet()) { if (!entry.getValue().booleanValue()) { Preconditions.checkArgument(false, - "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson()); + "ScanQual not found. -> required JoinQual:" + entry.getKey().toJson(SerializeOption.GENERIC)); } } } @@ -817,7 +818,7 @@ public final void testJson() throws PlanningException { LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); - String json = plan.toJson(); + String json = plan.toJson(SerializeOption.GENERIC); LogicalNode fromJson = CoreGsonHelper.fromJson(json, LogicalNode.class); assertEquals(NodeType.ROOT, fromJson.getType()); LogicalNode project = ((LogicalRootNode)fromJson).getChild(); @@ -1098,7 +1099,7 @@ public void testSetQualifier() throws PlanningException { } public void testJsonSerDerObject(LogicalNode rootNode) { - String json = rootNode.toJson(); + String json = rootNode.toJson(SerializeOption.GENERIC); LogicalNode fromJson = CoreGsonHelper.fromJson(json, LogicalNode.class); assertTrue("JSON (de) serialization equivalence check", rootNode.deepEquals(fromJson)); } diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java index 9910d79f27..fb48b2e548 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java @@ -20,6 +20,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.tajo.SerializeOption; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.QueryId; @@ -76,9 +77,9 @@ public void testCreateHashFetchURL() throws Exception { fetch.setName(sid.toString()); - TajoWorkerProtocol.FetchProto proto = fetch.getProto(); + TajoWorkerProtocol.FetchProto proto = fetch.getProto(SerializeOption.GENERIC); fetch = new FetchImpl(proto); - assertEquals(proto, fetch.getProto()); + assertEquals(proto, fetch.getProto(SerializeOption.GENERIC)); Map> ebEntries = new HashMap>(); ebEntries.put(sid, eachEntry.getValue()); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java b/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java index 2c997a3203..5ffd1607f9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java @@ -19,6 +19,7 @@ package org.apache.tajo.master.rm; import com.google.protobuf.RpcCallback; +import org.apache.tajo.SerializeOption; import org.apache.tajo.QueryId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.conf.TajoConf; @@ -96,7 +97,7 @@ private TajoWorkerResourceManager initResourceManager() throws Exception { WorkerConnectionInfo connectionInfo = new WorkerConnectionInfo("host" + (i + 1), 28091, 28092, 21000 + i, 28093, 28080); NodeHeartbeat tajoHeartbeat = NodeHeartbeat.newBuilder() - .setConnectionInfo(connectionInfo.getProto()) + .setConnectionInfo(connectionInfo.getProto(SerializeOption.GENERIC)) .setServerStatus(serverStatus) .build(); diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java index d0ab1c0d50..3a574093ae 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestFileFragment.java @@ -20,6 +20,7 @@ import com.google.common.collect.Sets; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.CommonTestingUtil; @@ -54,7 +55,7 @@ public final void testGetAndSetFields() { public final void testGetProtoAndRestore() { FileFragment fragment = new FileFragment("table1_1", new Path(path, "table0"), 0, 500); - FileFragment fragment1 = FragmentConvertor.convert(FileFragment.class, fragment.getProto()); + FileFragment fragment1 = FragmentConvertor.convert(FileFragment.class, fragment.getProto(SerializeOption.GENERIC)); assertEquals("table1_1", fragment1.getTableName()); assertEquals(new Path(path, "table0"), fragment1.getPath()); assertTrue(0 == fragment1.getStartKey()); diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index 859a1f2f24..04befe5cfe 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.CatalogUtil; @@ -77,7 +78,7 @@ public void test() throws IOException { FileSystem fs = sm.getFileSystem(); fs.mkdirs(tablePath); - FileUtil.writeProto(fs, metaPath, meta.getProto()); + FileUtil.writeProto(fs, metaPath, meta.getProto(SerializeOption.GENERIC)); Appender appender = sm.getAppender(meta, schema, dataPath); appender.enableStats(); diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java index df6d714732..d4f2c9b3d8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java @@ -20,6 +20,7 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.service.Service; +import org.apache.tajo.SerializeOption; import org.apache.tajo.TajoProtos; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TaskAttemptId; @@ -82,13 +83,13 @@ public final void testTaskRunnerHistory() throws IOException, ServiceException, TaskRunnerHistory history = histories.iterator().next(); assertEquals(Service.STATE.STOPPED, history.getState()); - TaskRunnerHistory fromProto = new TaskRunnerHistory(history.getProto()); + TaskRunnerHistory fromProto = new TaskRunnerHistory(history.getProto(SerializeOption.GENERIC)); assertEquals(history.getExecutionBlockId(), fromProto.getExecutionBlockId()); assertEquals(history.getFinishTime(), fromProto.getFinishTime()); assertEquals(history.getStartTime(), fromProto.getStartTime()); assertEquals(history.getState(), fromProto.getState()); assertEquals(history.getContainerId(), fromProto.getContainerId()); - assertEquals(history.getProto().getTaskHistoriesCount(), fromProto.getProto().getTaskHistoriesCount()); + assertEquals(history.getProto(SerializeOption.GENERIC).getTaskHistoriesCount(), fromProto.getProto(SerializeOption.GENERIC).getTaskHistoriesCount()); } @Test diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java b/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java index a5c39b8d88..4a449b6f85 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -126,12 +127,12 @@ public Object clone() throws CloneNotSupportedException { return target; } - public String toJson() { + public String toJson(SerializeOption option) { return PlanGsonHelper.toJson(this, Target.class); } @Override - public PlanProto.Target getProto() { - return LogicalNodeSerializer.convertTarget(this); + public PlanProto.Target getProto(SerializeOption option) { + return LogicalNodeSerializer.convertTarget(this, option); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java index 2f40974c7b..4de201ceda 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -279,7 +280,7 @@ public String toString() { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return PlanGsonHelper.toJson(IfThenEval.this, IfThenEval.class); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java index 154b0fd8a7..701c6b6d8f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -57,7 +58,7 @@ public EvalType getType() { public abstract String getName(); @Override - public String toJson() { + public String toJson(SerializeOption option) { return PlanGsonHelper.toJson(this, EvalNode.class); } @@ -91,7 +92,7 @@ public Object clone() throws CloneNotSupportedException { } @Override - public PlanProto.EvalNodeTree getProto() { - return EvalNodeSerializer.serialize(this); + public PlanProto.EvalNodeTree getProto(SerializeOption option) { + return EvalNodeSerializer.serialize(this, option); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java index 08ea6a7f73..14fe9bfe4d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.function; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.CatalogGsonHelper; @@ -48,7 +49,7 @@ public void merge(FunctionContext ctx, Tuple part) { public abstract T terminate(FunctionContext ctx); @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, AggFunction.class); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index 39db5c6a33..1eb0fd3ccc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.function; +import org.apache.tajo.SerializeOption; import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; @@ -48,7 +49,7 @@ public enum Type { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, GeneralFunction.class); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java index 82f725765b..cdca51a258 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.function; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -51,7 +52,7 @@ public DataType getPartialResultType() { public abstract T terminate(FunctionContext ctx); @Override - public String toJson() { + public String toJson(SerializeOption option) { return CatalogGsonHelper.toJson(this, WindowAggFunc.class); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java index 450d3e0afc..ec17418d80 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java @@ -23,6 +23,7 @@ import com.google.gson.annotations.Expose; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Schema; import org.apache.tajo.json.GsonObject; import org.apache.tajo.plan.PlanString; @@ -132,7 +133,7 @@ public Object clone() throws CloneNotSupportedException { } @Override - public String toJson() { + public String toJson(SerializeOption option) { return PlanGsonHelper.toJson(this, LogicalNode.class); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java index 8a24add6c1..bcf7ffea7b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan.rewrite.rules; import org.apache.tajo.OverridableConf; +import org.apache.tajo.SerializeOption; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.LogicalNode; @@ -47,7 +48,7 @@ public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { @Override public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { LogicalNode root = plan.getRootBlock().getRoot(); - PlanProto.LogicalNodeTree serialized = LogicalNodeSerializer.serialize(plan.getRootBlock().getRoot()); + PlanProto.LogicalNodeTree serialized = LogicalNodeSerializer.serialize(root, SerializeOption.GENERIC); LogicalNode deserialized = LogicalNodeDeserializer.deserialize(queryContext, serialized); assert root.deepEquals(deserialized); return plan; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java index c7702c5117..cb503c42f8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeSerializer.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; +import org.apache.tajo.SerializeOption; import org.apache.tajo.algebra.WindowSpec.WindowFrameEndBoundType; import org.apache.tajo.algebra.WindowSpec.WindowFrameStartBoundType; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -51,13 +52,18 @@ public class EvalNodeSerializer public static class EvalTreeProtoBuilderContext { private int seqId = 0; - private Map idMap = Maps.newHashMap(); - private PlanProto.EvalNodeTree.Builder treeBuilder = PlanProto.EvalNodeTree.newBuilder(); + private final Map idMap = Maps.newHashMap(); + private final PlanProto.EvalNodeTree.Builder treeBuilder = PlanProto.EvalNodeTree.newBuilder(); + private final SerializeOption option; + + public EvalTreeProtoBuilderContext(SerializeOption option) { + this.option = option; + } } - public static PlanProto.EvalNodeTree serialize(EvalNode evalNode) { + public static PlanProto.EvalNodeTree serialize(EvalNode evalNode, SerializeOption option) { EvalNodeSerializer.EvalTreeProtoBuilderContext context = - new EvalNodeSerializer.EvalTreeProtoBuilderContext(); + new EvalNodeSerializer.EvalTreeProtoBuilderContext(option); instance.visit(context, evalNode, new Stack()); return context.treeBuilder.build(); } @@ -183,7 +189,7 @@ public EvalNode visitRowConstant(EvalTreeProtoBuilderContext context, RowConstan public EvalNode visitField(EvalTreeProtoBuilderContext context, Stack stack, FieldEval field) { PlanProto.EvalNode.Builder builder = createEvalBuilder(context, field); - builder.setField(field.getColumnRef().getProto()); + builder.setField(field.getColumnRef().getProto(context.option)); context.treeBuilder.addNodes(builder); return field; } @@ -261,7 +267,7 @@ public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval // building itself PlanProto.FunctionEval.Builder funcBuilder = PlanProto.FunctionEval.newBuilder(); - funcBuilder.setFuncion(function.getFuncDesc().getProto()); + funcBuilder.setFuncion(function.getFuncDesc().getProto(context.option)); for (int i = 0; i < childIds.length; i++) { funcBuilder.addParamIds(childIds[i]); } @@ -290,7 +296,7 @@ public EvalNode visitFuncCall(EvalTreeProtoBuilderContext context, FunctionEval if (winFunc.hasSortSpecs()) { windowFuncBuilder.addAllSortSpec(ProtoUtil.toProtoObjects - (winFunc.getSortSpecs())); + (winFunc.getSortSpecs(), context.option)); } windowFuncBuilder.setWindowFrame(buildWindowFrame(winFunc.getWindowFrame())); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 60171de085..b32b864496 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -20,6 +20,7 @@ import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.exception.UnimplementedException; @@ -62,10 +63,11 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor()); } catch (PlanningException e) { @@ -90,20 +92,26 @@ private static PlanProto.LogicalNode.Builder createNodeBuilder(SerializeContext // some DDL statements like DropTable or DropDatabase do not have in/out schemas if (node.getInSchema() != null) { - nodeBuilder.setInSchema(node.getInSchema().getProto()); + nodeBuilder.setInSchema(node.getInSchema().getProto(context.option)); } if (node.getOutSchema() != null) { - nodeBuilder.setOutSchema(node.getOutSchema().getProto()); + nodeBuilder.setOutSchema(node.getOutSchema().getProto(context.option)); } return nodeBuilder; } public static class SerializeContext { private int seqId = 0; - private Map idMap = Maps.newHashMap(); - private LogicalNodeTree.Builder treeBuilder = LogicalNodeTree.newBuilder(); + private final Map idMap = Maps.newHashMap(); + private final LogicalNodeTree.Builder treeBuilder = LogicalNodeTree.newBuilder(); + private final SerializeOption option; + + public SerializeContext(SerializeOption option) { + this.option = option; + } } + @Override public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, LogicalRootNode root, Stack stack) throws PlanningException { super.visitRoot(context, plan, block, root, stack); @@ -138,11 +146,12 @@ public LogicalNode visitSetSession(SerializeContext context, LogicalPlan plan, L return node; } + @Override public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, EvalExprNode exprEval, Stack stack) throws PlanningException { PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder(); exprEvalBuilder.addAllTargets( - ProtoUtil.toProtoObjects(exprEval.getTargets())); + ProtoUtil.toProtoObjects(exprEval.getTargets(), context.option)); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, exprEval); nodeBuilder.setExprEval(exprEvalBuilder); @@ -151,6 +160,7 @@ public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, Log return exprEval; } + @Override public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode projection, Stack stack) throws PlanningException { super.visitProjection(context, plan, block, projection, stack); @@ -160,7 +170,7 @@ public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, L PlanProto.ProjectionNode.Builder projectionBuilder = PlanProto.ProjectionNode.newBuilder(); projectionBuilder.setChildSeq(childIds[0]); projectionBuilder.addAllTargets( - ProtoUtil.toProtoObjects(projection.getTargets())); + ProtoUtil.toProtoObjects(projection.getTargets(), context.option)); projectionBuilder.setDistinct(projection.isDistinct()); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, projection); @@ -188,6 +198,7 @@ public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, Logica return limit; } + @Override public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, WindowAggNode windowAgg, Stack stack) throws PlanningException { super.visitWindowAgg(context, plan, block, windowAgg, stack); @@ -199,22 +210,22 @@ public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, Lo if (windowAgg.hasPartitionKeys()) { windowAggBuilder.addAllPartitionKeys( - ProtoUtil.toProtoObjects(windowAgg.getPartitionKeys())); + ProtoUtil.toProtoObjects(windowAgg.getPartitionKeys(), context.option)); } if (windowAgg.hasAggFunctions()) { windowAggBuilder.addAllWindowFunctions( - ProtoUtil.toProtoObjects(windowAgg.getWindowFunctions())); + ProtoUtil.toProtoObjects(windowAgg.getWindowFunctions(), context.option)); } windowAggBuilder.setDistinct(windowAgg.isDistinct()); if (windowAgg.hasSortSpecs()) { windowAggBuilder.addAllSortSpecs( - ProtoUtil.toProtoObjects(windowAgg.getSortSpecs())); + ProtoUtil.toProtoObjects(windowAgg.getSortSpecs(), context.option)); } if (windowAgg.hasTargets()) { windowAggBuilder.addAllTargets( - ProtoUtil.toProtoObjects(windowAgg.getTargets())); + ProtoUtil.toProtoObjects(windowAgg.getTargets(), context.option)); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, windowAgg); @@ -234,7 +245,7 @@ public LogicalNode visitSort(SerializeContext context, LogicalPlan plan, Logical PlanProto.SortNode.Builder sortBuilder = PlanProto.SortNode.newBuilder(); sortBuilder.setChildSeq(childIds[0]); for (int i = 0; i < sort.getSortKeys().length; i++) { - sortBuilder.addSortSpecs(sort.getSortKeys()[i].getProto()); + sortBuilder.addSortSpecs(sort.getSortKeys()[i].getProto(context.option)); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, sort); @@ -253,7 +264,7 @@ public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, Logic PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); filterBuilder.setChildSeq(childIds[0]); - filterBuilder.setQual(EvalNodeSerializer.serialize(having.getQual())); + filterBuilder.setQual(EvalNodeSerializer.serialize(having.getQual(), context.option)); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, having); nodeBuilder.setFilter(filterBuilder); @@ -262,6 +273,7 @@ public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, Logic return having; } + @Override public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, GroupbyNode node, Stack stack) throws PlanningException { super.visitGroupBy(context, plan, block, node, new Stack()); @@ -281,14 +293,14 @@ private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, Gro if (node.groupingKeyNum() > 0) { groupbyBuilder.addAllGroupingKeys( - ProtoUtil.toProtoObjects(node.getGroupingColumns())); + ProtoUtil.toProtoObjects(node.getGroupingColumns(), context.option)); } if (node.hasAggFunctions()) { groupbyBuilder.addAllAggFunctions( - ProtoUtil.toProtoObjects(node.getAggFunctions())); + ProtoUtil.toProtoObjects(node.getAggFunctions(), context.option)); } if (node.hasTargets()) { - groupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); + groupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets(), context.option)); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); @@ -297,6 +309,7 @@ private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, Gro return nodeBuilder; } + @Override public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, Stack stack) throws PlanningException { super.visitDistinctGroupby(context, plan, block, node, new Stack()); @@ -315,14 +328,14 @@ public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan pl if (node.getGroupingColumns().length > 0) { distGroupbyBuilder.addAllGroupingKeys( - ProtoUtil.toProtoObjects(node.getGroupingColumns())); + ProtoUtil.toProtoObjects(node.getGroupingColumns(), context.option)); } if (node.getAggFunctions().length > 0) { distGroupbyBuilder.addAllAggFunctions( - ProtoUtil.toProtoObjects(node.getAggFunctions())); + ProtoUtil.toProtoObjects(node.getAggFunctions(), context.option)); } if (node.hasTargets()) { - distGroupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); + distGroupbyBuilder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets(), context.option)); } for (int cid : node.getResultColumnIds()) { distGroupbyBuilder.addResultId(cid); @@ -344,7 +357,7 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic PlanProto.FilterNode.Builder filterBuilder = PlanProto.FilterNode.newBuilder(); filterBuilder.setChildSeq(childIds[0]); - filterBuilder.setQual(EvalNodeSerializer.serialize(filter.getQual())); + filterBuilder.setQual(EvalNodeSerializer.serialize(filter.getQual(), context.option)); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, filter); nodeBuilder.setFilter(filterBuilder); @@ -353,6 +366,7 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic return filter; } + @Override public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode join, Stack stack) throws PlanningException { super.visitJoin(context, plan, block, join, stack); @@ -365,12 +379,12 @@ public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, Logical joinBuilder.setLeftChildSeq(childIds[0]); joinBuilder.setRightChilSeq(childIds[1]); if (join.hasJoinQual()) { - joinBuilder.setJoinQual(EvalNodeSerializer.serialize(join.getJoinQual())); + joinBuilder.setJoinQual(EvalNodeSerializer.serialize(join.getJoinQual(), context.option)); } if (join.hasTargets()) { joinBuilder.setExistsTargets(true); - joinBuilder.addAllTargets(ProtoUtil.toProtoObjects(join.getTargets())); + joinBuilder.addAllTargets(ProtoUtil.toProtoObjects(join.getTargets(), context.option)); } else { joinBuilder.setExistsTargets(false); } @@ -405,7 +419,7 @@ public LogicalNode visitUnion(SerializeContext context, LogicalPlan plan, Logica public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scan, Stack stack) throws PlanningException { - PlanProto.ScanNode.Builder scanBuilder = buildScanNode(scan); + PlanProto.ScanNode.Builder scanBuilder = buildScanNode(scan, context.option); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, scan); nodeBuilder.setScan(scanBuilder); @@ -414,22 +428,22 @@ public LogicalNode visitScan(SerializeContext context, LogicalPlan plan, Logical return scan; } - public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan) { + public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan, SerializeOption option) { PlanProto.ScanNode.Builder scanBuilder = PlanProto.ScanNode.newBuilder(); - scanBuilder.setTable(scan.getTableDesc().getProto()); + scanBuilder.setTable(scan.getTableDesc().getProto(option)); if (scan.hasAlias()) { scanBuilder.setAlias(scan.getAlias()); } if (scan.hasTargets()) { scanBuilder.setExistTargets(true); - scanBuilder.addAllTargets(ProtoUtil.toProtoObjects(scan.getTargets())); + scanBuilder.addAllTargets(ProtoUtil.toProtoObjects(scan.getTargets(), option)); } else { scanBuilder.setExistTargets(false); } if (scan.hasQual()) { - scanBuilder.setQual(EvalNodeSerializer.serialize(scan.getQual())); + scanBuilder.setQual(EvalNodeSerializer.serialize(scan.getQual(), option)); } scanBuilder.setBroadcast(scan.isBroadcastTable()); @@ -441,7 +455,7 @@ public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPl PartitionedTableScanNode node, Stack stack) throws PlanningException { - PlanProto.ScanNode.Builder scanBuilder = buildScanNode(node); + PlanProto.ScanNode.Builder scanBuilder = buildScanNode(node, context.option); PlanProto.PartitionScanSpec.Builder partitionScan = PlanProto.PartitionScanSpec.newBuilder(); List pathStrs = TUtil.newList(); @@ -460,6 +474,7 @@ public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPl return node; } + @Override public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack stack) throws PlanningException { super.visitTableSubQuery(context, plan, block, node, stack); @@ -472,7 +487,7 @@ public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan builder.setTableName(node.getTableName()); if (node.hasTargets()) { - builder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets())); + builder.addAllTargets(ProtoUtil.toProtoObjects(node.getTargets(), context.option)); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); @@ -482,17 +497,18 @@ public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan return node; } + @Override public LogicalNode visitCreateTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, CreateTableNode node, Stack stack) throws PlanningException { super.visitCreateTable(context, plan, block, node, stack); int [] childIds = registerGetChildIds(context, node); - PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds); - PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node); + PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds, context.option); + PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node, context.option); PlanProto.CreateTableNodeSpec.Builder createTableBuilder = PlanProto.CreateTableNodeSpec.newBuilder(); - createTableBuilder.setSchema(node.getTableSchema().getProto()); + createTableBuilder.setSchema(node.getTableSchema().getProto(context.option)); createTableBuilder.setExternal(node.isExternal()); if (node.isExternal() && node.hasPath()) { createTableBuilder.setPath(node.getPath().toString()); @@ -559,7 +575,7 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L break; case ADD_COLUMN: alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.ADD_COLUMN); - alterTableBuilder.setAddColumn(AddColumn.newBuilder().setAddColumn(node.getAddNewColumn().getProto())); + alterTableBuilder.setAddColumn(AddColumn.newBuilder().setAddColumn(node.getAddNewColumn().getProto(context.option))); break; case RENAME_COLUMN: alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.RENAME_COLUMN); @@ -569,7 +585,7 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L break; case SET_PROPERTY: alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.SET_PROPERTY); - alterTableBuilder.setProperties(node.getProperties().getProto()); + alterTableBuilder.setProperties(node.getProperties().getProto(context.option)); break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + node.getAlterTableOpType().name()); @@ -595,23 +611,24 @@ public LogicalNode visitTruncateTable(SerializeContext context, LogicalPlan plan return node; } + @Override public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, InsertNode node, Stack stack) throws PlanningException { super.visitInsert(context, plan, block, node, stack); int [] childIds = registerGetChildIds(context, node); - PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds); - PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node); + PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds, context.option); + PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node, context.option); PlanProto.InsertNodeSpec.Builder insertNodeSpec = PlanProto.InsertNodeSpec.newBuilder(); insertNodeSpec.setOverwrite(node.isOverwrite()); - insertNodeSpec.setTableSchema(node.getTableSchema().getProto()); + insertNodeSpec.setTableSchema(node.getTableSchema().getProto(context.option)); if (node.hasProjectedSchema()) { - insertNodeSpec.setProjectedSchema(node.getProjectedSchema().getProto()); + insertNodeSpec.setProjectedSchema(node.getProjectedSchema().getProto(context.option)); } if (node.hasTargetSchema()) { - insertNodeSpec.setTargetSchema(node.getTargetSchema().getProto()); + insertNodeSpec.setTargetSchema(node.getTargetSchema().getProto(context.option)); } if (node.hasPath()) { insertNodeSpec.setPath(node.getPath().toString()); @@ -626,21 +643,22 @@ public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, Logic return node; } - private static PlanProto.PersistentStoreNode.Builder buildPersistentStoreBuilder(PersistentStoreNode node, - int [] childIds) { + private static PlanProto.PersistentStoreNode.Builder buildPersistentStoreBuilder( + PersistentStoreNode node, int[] childIds, SerializeOption option) { PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = PlanProto.PersistentStoreNode.newBuilder(); persistentStoreBuilder.setChildSeq(childIds[0]); persistentStoreBuilder.setStorageType(node.getStorageType()); if (node.hasOptions()) { - persistentStoreBuilder.setTableProperties(node.getOptions().getProto()); + persistentStoreBuilder.setTableProperties(node.getOptions().getProto(option)); } return persistentStoreBuilder; } - private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { + private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec( + StoreTableNode node, SerializeOption option) { PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); if (node.hasPartition()) { - storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); + storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto(option)); } if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' storeTableBuilder.setTableName(node.getTableName()); @@ -707,9 +725,9 @@ public static PlanProto.JoinType convertJoinType(JoinType type) { } } - public static PlanProto.Target convertTarget(Target target) { + public static PlanProto.Target convertTarget(Target target, SerializeOption option) { PlanProto.Target.Builder targetBuilder = PlanProto.Target.newBuilder(); - targetBuilder.setExpr(EvalNodeSerializer.serialize(target.getEvalTree())); + targetBuilder.setExpr(EvalNodeSerializer.serialize(target.getEvalTree(), option)); if (target.hasAlias()) { targetBuilder.setAlias(target.getAlias()); } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java index cba9ee0572..4f8d922a2d 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java @@ -18,13 +18,15 @@ package org.apache.tajo.storage; -import com.google.common.base.Objects; import com.google.common.base.Preconditions; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.datum.Datum; +import java.util.Arrays; + import static org.apache.tajo.catalog.proto.CatalogProtos.TupleComparatorSpecProto; import static org.apache.tajo.index.IndexProtos.TupleComparatorProto; @@ -145,7 +147,7 @@ public int compare(Tuple tuple1, Tuple tuple2) { @Override public int hashCode() { - return Objects.hashCode(sortKeyIds); + return Arrays.hashCode(sortKeyIds); } @Override @@ -171,11 +173,11 @@ public boolean equals(Object obj) { } @Override - public TupleComparatorProto getProto() { + public TupleComparatorProto getProto(SerializeOption option) { TupleComparatorProto.Builder builder = TupleComparatorProto.newBuilder(); - builder.setSchema(schema.getProto()); + builder.setSchema(schema.getProto(option)); for (int i = 0; i < sortSpecs.length; i++) { - builder.addSortSpecs(sortSpecs[i].getProto()); + builder.addSortSpecs(sortSpecs[i].getProto(option)); } TupleComparatorSpecProto.Builder sortSpecBuilder; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/Fragment.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/Fragment.java index ac43197296..c3a0f8126c 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/Fragment.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/Fragment.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage.fragment; +import org.apache.tajo.SerializeOption; import org.apache.tajo.common.ProtoObject; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; @@ -27,7 +28,7 @@ public interface Fragment extends ProtoObject { public abstract String getTableName(); @Override - public abstract FragmentProto getProto(); + public abstract FragmentProto getProto(SerializeOption option); public abstract long getLength(); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java index 07720c708c..926867ee20 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java @@ -22,6 +22,7 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import org.apache.hadoop.conf.Configuration; +import org.apache.tajo.SerializeOption; import org.apache.tajo.annotation.ThreadSafe; import java.io.IOException; @@ -111,19 +112,19 @@ public static List convert(Configuration conf, FragmentP return list; } - public static List toFragmentProtoList(Fragment... fragments) { + public static List toFragmentProtoList(SerializeOption option, Fragment... fragments) { List list = Lists.newArrayList(); if (fragments == null) { return list; } for (Fragment fragment : fragments) { - list.add(fragment.getProto()); + list.add(fragment.getProto(option)); } return list; } - public static FragmentProto [] toFragmentProtoArray(Fragment... fragments) { - List list = toFragmentProtoList(fragments); + public static FragmentProto [] toFragmentProtoArray(SerializeOption option, Fragment... fragments) { + List list = toFragmentProtoList(option, fragments); return list.toArray(new FragmentProto[list.size()]); } } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 9ea0bf6598..5fa1238546 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -23,6 +23,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -144,7 +145,7 @@ public String toString() { } @Override - public FragmentProto getProto() { + public FragmentProto getProto(SerializeOption option) { HBaseFragmentProto.Builder builder = HBaseFragmentProto.newBuilder(); builder.setTableName(tableName) .setHbaseTableName(hbaseTableName) diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index ab1c808865..a2d9b5dc4c 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -24,6 +24,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.storage.StorageFragmentProtos.*; @@ -211,7 +212,7 @@ public String toString() { + getLength() + "}" ; } - public FragmentProto getProto() { + public FragmentProto getProto(SerializeOption option) { FileFragmentProto.Builder builder = FileFragmentProto.newBuilder(); builder.setId(this.tableName); builder.setStartOffset(this.startOffset); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java index 81a1ffd2cc..27262c7281 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SerializeOption; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.storage.*; @@ -147,12 +148,12 @@ public void flush() throws IOException { public void writeHeader(int entryNum) throws IOException { // schema - byte [] schemaBytes = keySchema.getProto().toByteArray(); + byte [] schemaBytes = keySchema.getProto(SerializeOption.GENERIC).toByteArray(); out.writeInt(schemaBytes.length); out.write(schemaBytes); // comparator - byte [] comparatorBytes = compartor.getProto().toByteArray(); + byte [] comparatorBytes = compartor.getProto(SerializeOption.GENERIC).toByteArray(); out.writeInt(comparatorBytes.length); out.write(comparatorBytes);