From 3f1023abce29be38de820af427a8f6db58b14cf3 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 24 Sep 2015 16:47:09 -0700 Subject: [PATCH 1/5] Renamed StoreType enum type to DataFormat. --- .../org/apache/tajo/catalog/CatalogUtil.java | 51 +++++++++---------- .../src/main/proto/CatalogProtos.proto | 2 +- .../tajo/catalog/store/HiveCatalogUtil.java | 8 ++- .../InfoSchemaMetadataDictionary.java | 4 +- .../org/apache/tajo/plan/TestLogicalNode.java | 2 - .../org/apache/tajo/storage/Tablespace.java | 2 +- .../apache/tajo/storage/TestMergeScanner.java | 2 +- 7 files changed, 33 insertions(+), 38 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 996bfaf733..d50696c0f6 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -30,13 +30,12 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.proto.CatalogProtos.DataFormat; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; -import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedOperatorException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; @@ -286,37 +285,37 @@ public static String getBackwardCompitablityStoreType(String storeType) { return getStoreTypeString(getStoreType(storeType)); } - public static String getStoreTypeString(final StoreType type) { - if (type == StoreType.TEXTFILE) { + public static String getStoreTypeString(final DataFormat type) { + if (type == DataFormat.TEXTFILE) { return BuiltinStorages.TEXT; } else { return type.name(); } } - public static StoreType getStoreType(final String typeStr) { + public static DataFormat getStoreType(final String typeStr) { if (typeStr.equalsIgnoreCase("CSV")) { - return StoreType.TEXTFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.RAW.name())) { - return StoreType.RAW; - } else if (typeStr.equalsIgnoreCase(StoreType.ROWFILE.name())) { - return StoreType.ROWFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.RCFILE.name())) { - return StoreType.RCFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.ORC.name())) { - return StoreType.ORC; - } else if (typeStr.equalsIgnoreCase(StoreType.PARQUET.name())) { - return StoreType.PARQUET; - } else if (typeStr.equalsIgnoreCase(StoreType.SEQUENCEFILE.name())) { - return StoreType.SEQUENCEFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.AVRO.name())) { - return StoreType.AVRO; + return DataFormat.TEXTFILE; + } else if (typeStr.equalsIgnoreCase(DataFormat.RAW.name())) { + return CatalogProtos.DataFormat.RAW; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.ROWFILE.name())) { + return DataFormat.ROWFILE; + } else if (typeStr.equalsIgnoreCase(DataFormat.RCFILE.name())) { + return DataFormat.RCFILE; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.ORC.name())) { + return CatalogProtos.DataFormat.ORC; + } else if (typeStr.equalsIgnoreCase(DataFormat.PARQUET.name())) { + return DataFormat.PARQUET; + } else if (typeStr.equalsIgnoreCase(DataFormat.SEQUENCEFILE.name())) { + return DataFormat.SEQUENCEFILE; + } else if (typeStr.equalsIgnoreCase(DataFormat.AVRO.name())) { + return CatalogProtos.DataFormat.AVRO; } else if (typeStr.equalsIgnoreCase(BuiltinStorages.TEXT)) { - return StoreType.TEXTFILE; - } else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) { - return StoreType.JSON; - } else if (typeStr.equalsIgnoreCase(StoreType.HBASE.name())) { - return StoreType.HBASE; + return CatalogProtos.DataFormat.TEXTFILE; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.JSON.name())) { + return CatalogProtos.DataFormat.JSON; + } else if (typeStr.equalsIgnoreCase(CatalogProtos.DataFormat.HBASE.name())) { + return CatalogProtos.DataFormat.HBASE; } else { return null; } @@ -980,7 +979,7 @@ public static Pair, String> getPartitionKeyNamePair(Stri /** * Create new table property with default configs. It is used to make TableMeta to be stored in Catalog. * - * @param storeType StoreType + * @param storeType DataFormat * @return Table properties */ public static KeyValueSet newDefaultProperty(String storeType) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 08818b1839..28b08a066c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -25,7 +25,7 @@ option java_generate_equals_and_hash = true; import "DataTypes.proto"; import "PrimitiveProtos.proto"; -enum StoreType { +enum DataFormat { MEM = 0; TEXTFILE = 1; RAW = 2; diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index 67a1488143..a878e0178f 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -32,8 +32,6 @@ import org.apache.thrift.TException; import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; -import static org.apache.tajo.exception.ExceptionUtil.makeNotSupported; - public class HiveCatalogUtil { public static void validateSchema(Table tblSchema) { for (FieldSchema fieldSchema : tblSchema.getCols()) { @@ -113,11 +111,11 @@ public static String getStoreType(String fileFormat) { if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) { return BuiltinStorages.TEXT; } else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.SEQUENCEFILE.name(); + return CatalogProtos.DataFormat.SEQUENCEFILE.name(); } else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.RCFILE.name(); + return CatalogProtos.DataFormat.RCFILE.name(); } else if(outputFormatClass.equals(DeprecatedParquetOutputFormat.class.getSimpleName())) { - return CatalogProtos.StoreType.PARQUET.name(); + return CatalogProtos.DataFormat.PARQUET.name(); } else { throw new TajoRuntimeException(new UnknownDataFormatException(fileFormat)); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java index 2de98075d1..be2913ffd4 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java @@ -24,7 +24,7 @@ import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.proto.CatalogProtos.DataFormat; import org.apache.tajo.util.TUtil; public class InfoSchemaMetadataDictionary { @@ -131,6 +131,6 @@ public boolean existTable(String tableName) { } protected String getTablePath() { - return StoreType.SYSTEM.name().toUpperCase(); + return DataFormat.SYSTEM.name().toUpperCase(); } } diff --git a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java index 92cb39952a..56edccce62 100644 --- a/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java @@ -22,11 +22,9 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.plan.logical.JoinNode; -import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.ScanNode; import org.junit.Test; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 1c6f433051..15130414e9 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -274,7 +274,7 @@ public Appender getAppender(OverridableConf queryContext, } /** - * Return the Scanner class for the StoreType that is defined in storage-default.xml. + * Return the Scanner class for the DataFormat that is defined in storage-default.xml. * * @param storeType store type * @return The Scanner class diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index 4faf8b9d75..c93f9e8cd1 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -83,7 +83,7 @@ public static Collection generateParameters() { {"SEQUENCEFILE"}, {"AVRO"}, // RowFile requires Byte-buffer read support, so we omitted RowFile. - //{StoreType.ROWFILE}, + //{DataFormat.ROWFILE}, }); } From f5424251f7c597b48bc94687950d283925bbdec4 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 24 Sep 2015 17:06:03 -0700 Subject: [PATCH 2/5] Change StoreType to DataFormat in Fragment and TableMeta. --- .../org/apache/tajo/catalog/DDLBuilder.java | 6 ++-- .../org/apache/tajo/catalog/TableMeta.java | 22 ++++++------- .../tajo/catalog/json/TableMetaAdapter.java | 3 +- .../src/main/proto/CatalogProtos.proto | 4 +-- .../apache/tajo/catalog/TestTableMeta.java | 6 ++-- .../tajo/catalog/store/HiveCatalogStore.java | 10 +++--- .../dictionary/AbstractTableDescriptor.java | 2 +- .../tajo/catalog/store/AbstractDBStore.java | 5 ++- .../TestCatalogAgainstCaseSensitivity.java | 2 +- .../org/apache/tajo/cli/tools/TajoDump.java | 3 +- .../cli/tsql/commands/DescTableCommand.java | 2 +- .../tajo/engine/query/TestCTASQuery.java | 6 ++-- .../tajo/engine/query/TestCreateTable.java | 4 +-- .../planner/physical/BSTIndexScanExec.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 4 +-- .../planner/physical/StoreTableExec.java | 3 +- .../tajo/master/exec/QueryExecutor.java | 2 +- .../tajo/querymaster/Repartitioner.java | 2 +- .../org/apache/tajo/querymaster/Task.java | 2 +- .../resources/webapps/admin/catalogview.jsp | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 7 ++-- .../tajo/plan/logical/CreateTableNode.java | 2 +- .../apache/tajo/plan/logical/InsertNode.java | 2 +- .../apache/tajo/plan/util/PlannerUtil.java | 2 +- .../tajo/storage/OldStorageManager.java | 4 +-- .../org/apache/tajo/storage/Tablespace.java | 6 ++-- .../storage/fragment/FragmentConvertor.java | 4 +-- .../tajo/storage/hbase/HBaseFragment.java | 2 +- .../tajo/storage/fragment/FileFragment.java | 2 +- .../tajo/storage/text/TextLineSerDe.java | 2 +- .../apache/tajo/storage/TestMergeScanner.java | 2 +- .../tajo/storage/index/TestBSTIndex.java | 32 +++++++++---------- .../index/TestSingleCSVFileBSTIndex.java | 8 ++--- .../tajo/storage/jdbc/JdbcFragment.java | 2 +- 34 files changed, 80 insertions(+), 89 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index baa1388028..6c0f171757 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -35,7 +35,7 @@ public static String buildDDLForExternalTable(TableDesc desc) { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(desc.getMeta().getStoreType()); + .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n-- Path: ").append(desc.getUri()); sb.append("\n--\n"); sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); @@ -58,7 +58,7 @@ public static String buildDDLForBaseTable(TableDesc desc) { sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") - .append(" Storage: ").append(desc.getMeta().getStoreType()); + .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); @@ -114,7 +114,7 @@ public static void buildSchema(StringBuilder sb, Schema schema) { } private static void buildUsingClause(StringBuilder sb, TableMeta meta) { - sb.append(" USING " + CatalogUtil.getBackwardCompitablityStoreType(meta.getStoreType())); + sb.append(" USING " + CatalogUtil.getBackwardCompitablityStoreType(meta.getDataFormat())); } private static void buildWithClause(final StringBuilder sb, TableMeta meta) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index 6838fe04a0..cfbbf75553 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -35,21 +35,21 @@ * It contains all information for scanning a fragmented table */ public class TableMeta implements ProtoObject, GsonObject, Cloneable { - @Expose protected String storeType; + @Expose protected String dataFormat; @Expose protected KeyValueSet options; - public TableMeta(String storeType, KeyValueSet options) { - this.storeType = storeType; + public TableMeta(String dataFormat, KeyValueSet options) { + this.dataFormat = dataFormat; this.options = new KeyValueSet(options); } public TableMeta(TableProto proto) { - this.storeType = proto.getStoreType(); + this.dataFormat = proto.getDataFormat(); this.options = new KeyValueSet(proto.getParams()); } - public String getStoreType() { - return this.storeType; + public String getDataFormat() { + return this.dataFormat; } public void setOptions(KeyValueSet options) { @@ -84,7 +84,7 @@ public boolean equals(Object object) { if(object instanceof TableMeta) { TableMeta other = (TableMeta) object; - boolean eq = this.getStoreType().equals(other.getStoreType()); + boolean eq = this.getDataFormat().equals(other.getDataFormat()); eq = eq && this.getOptions().equals(other.getOptions()); return eq; } @@ -93,13 +93,13 @@ public boolean equals(Object object) { } public int hashCode() { - return Objects.hashCode(getStoreType(), getOptions()); + return Objects.hashCode(getDataFormat(), getOptions()); } @Override public Object clone() throws CloneNotSupportedException { TableMeta meta = (TableMeta) super.clone(); - meta.storeType = getStoreType(); + meta.dataFormat = getDataFormat(); meta.options = (KeyValueSet) (toMap() != null ? options.clone() : null); return meta; } @@ -115,7 +115,7 @@ public String toString() { //////////////////////////////////////////////////////////////////////// public TableProto getProto() { TableProto.Builder builder = TableProto.newBuilder(); - builder.setStoreType(storeType); + builder.setDataFormat(dataFormat); builder.setParams(options.getProto()); return builder.build(); } @@ -127,7 +127,7 @@ public String toJson() { } public void mergeProtoToLocal() { - getStoreType(); + getDataFormat(); toMap(); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java index 2d164f6603..56bdc63ea2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java @@ -23,7 +23,6 @@ import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.json.GsonSerDerAdapter; import java.lang.reflect.Type; @@ -46,7 +45,7 @@ public TableMeta deserialize(JsonElement json, Type typeOfT, public JsonElement serialize(TableMeta src, Type typeOfSrc, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); - jsonObj.addProperty("store", src.getStoreType()); + jsonObj.addProperty("store", src.getDataFormat()); jsonObj.add("options", context.serialize(src.getOptions(), KeyValueSet.class)); return jsonObj; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 28b08a066c..b02c1a8e3a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -75,12 +75,12 @@ message SchemaProto { message FragmentProto { required string id = 1; - required string storeType = 2; + required string dataFormat = 2; required bytes contents = 3; } message TableProto { - required string storeType = 1; + required string dataFormat = 1; optional KeyValueSetProto params = 2; } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 9087262f7a..3f1d5c5ac9 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -55,7 +55,7 @@ public final void testClone() throws CloneNotSupportedException { TableMeta meta1 = CatalogUtil.newTableMeta("TEXT"); TableMeta meta2 = (TableMeta) meta1.clone(); - assertEquals(meta1.getStoreType(), meta2.getStoreType()); + assertEquals(meta1.getDataFormat(), meta2.getDataFormat()); assertEquals(meta1, meta2); } @@ -73,7 +73,7 @@ public void testSchema() throws CloneNotSupportedException { @Test public void testGetStorageType() { - assertEquals("TEXT", meta.getStoreType()); + assertEquals("TEXT", meta.getDataFormat()); } @Test @@ -108,7 +108,7 @@ public void testEqualsObject2() { optionBuilder.addKeyval(keyValueBuilder); } TableProto.Builder builder = TableProto.newBuilder(); - builder.setStoreType(BuiltinStorages.TEXT); + builder.setDataFormat(BuiltinStorages.TEXT); builder.setParams(optionBuilder); TableMeta meta2 = new TableMeta(builder.build()); assertTrue(meta1.equals(meta2)); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 72f7485ab2..49680150aa 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -479,7 +479,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) table.setPartitionKeys(partitionKeys); } - if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.RCFILE)) { + if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.RCFILE)) { String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); @@ -494,7 +494,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } - } else if (tableDesc.getMeta().getStoreType().equals(BuiltinStorages.TEXT)) { + } else if (tableDesc.getMeta().getDataFormat().equals(BuiltinStorages.TEXT)) { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName()); @@ -518,7 +518,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL))); table.getParameters().remove(StorageConstants.TEXT_NULL); } - } else if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { + } else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE); sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName()); @@ -549,12 +549,12 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); } } else { - if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.PARQUET)) { + if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.PARQUET)) { sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName()); sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName()); sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName()); } else { - throw new UnsupportedException(tableDesc.getMeta().getStoreType() + " in HivecatalogStore"); + throw new UnsupportedException(tableDesc.getMeta().getDataFormat() + " in HivecatalogStore"); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java index f9b7c62f78..cb071aa783 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java @@ -55,7 +55,7 @@ protected SchemaProto getSchemaProto() { protected TableProto getTableProto() { TableProto.Builder metaBuilder = TableProto.newBuilder(); - metaBuilder.setStoreType("SYSTEM"); + metaBuilder.setDataFormat("SYSTEM"); metaBuilder.setParams(KeyValueSetProto.newBuilder().build()); return metaBuilder.build(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 700ef1a9ae..ceb0dd9768 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -38,7 +38,6 @@ import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; -import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -854,7 +853,7 @@ public void createTable(final CatalogProtos.TableDescProto table) pstmt.setString(3, TableType.MANAGED.name()); } pstmt.setString(4, table.getPath()); - pstmt.setString(5, table.getMeta().getStoreType()); + pstmt.setString(5, table.getMeta().getDataFormat()); pstmt.setBoolean(6, table.getSchema() == null); pstmt.executeUpdate(); pstmt.close(); @@ -1741,7 +1740,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa ////////////////////////////////////////// CatalogProtos.TableProto.Builder metaBuilder = CatalogProtos.TableProto.newBuilder(); - metaBuilder.setStoreType(storeType); + metaBuilder.setDataFormat(storeType); sql = "SELECT key_, value_ FROM " + TB_OPTIONS + " WHERE " + COL_TABLES_PK + " = ?"; if (LOG.isDebugEnabled()) { diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 0ca6baa55a..cbee4770a0 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -115,7 +115,7 @@ public void testTable() throws Exception { assertTrue(tableDescs.containsKey(qualifiedTableName)); TableDesc desc = tableDescs.get(qualifiedTableName); assertEquals(desc.getUri().toString(), eachTableDescriptor.getPath()); - assertEquals(desc.getMeta().getStoreType(), eachTableDescriptor.getStoreType()); + assertEquals(desc.getMeta().getDataFormat(), eachTableDescriptor.getStoreType()); } ////////////////////////////////////////////////////////////////////////////// diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index c21e2949c5..43ebeb0f79 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -22,7 +22,6 @@ import org.apache.commons.cli.*; import org.apache.tajo.auth.UserRoleInfo; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.client.TajoClient; @@ -178,7 +177,7 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr String fqName = CatalogUtil.buildFQName(databaseName, tableName); TableDesc table = client.getTableDesc(fqName); - if (table.getMeta().getStoreType().equalsIgnoreCase("SYSTEM")) { + if (table.getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")) { continue; } diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index b17a336526..55553aaec5 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -99,7 +99,7 @@ protected String toFormattedString(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("\ntable name: ").append(desc.getName()).append("\n"); sb.append("table uri: ").append(desc.getUri()).append("\n"); - sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n"); + sb.append("store type: ").append(desc.getMeta().getDataFormat()).append("\n"); if (desc.getStats() != null) { long row = desc.getStats().getNumRows(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index f327c85296..4005d867a5 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -220,7 +220,7 @@ public final void testCtasWithStoreType() throws Exception { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("RCFILE".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("RCFILE".equalsIgnoreCase(desc.getMeta().getDataFormat())); } @Test @@ -234,7 +234,7 @@ public final void testCtasWithTextFile() throws Exception { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); } @Test @@ -248,7 +248,7 @@ public final void testCtasWithOptions() throws Exception { TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); - assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getStoreType())); + assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); KeyValueSet options = desc.getMeta().getOptions(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index f2f54baaab..3450b03719 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -20,7 +20,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -28,7 +27,6 @@ import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; -import org.junit.experimental.categories.Category; import java.sql.ResultSet; import java.util.List; @@ -434,7 +432,7 @@ private boolean isClonedTable(String orignalTable, String newTable) throws Excep if(origTableDesc.getMeta() != null) { TableMeta origMeta = origTableDesc.getMeta(); TableMeta newMeta = newTableDesc.getMeta(); - if(origMeta.getStoreType().equals(newMeta.getStoreType()) == false) { + if(origMeta.getDataFormat().equals(newMeta.getDataFormat()) == false) { fail("Store type of input tables not equal"); return false; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 19b0c9a199..53b5875c11 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -181,7 +181,7 @@ private void initScanner(Schema projected) throws IOException { Schema fileScanOutSchema = mergeSubSchemas(projected, keySchema, plan.getTargets(), qual); this.fileScanner = OldStorageManager.getStorageManager(context.getConf(), - plan.getTableDesc().getMeta().getStoreType()) + plan.getTableDesc().getMeta().getDataFormat()) .getSeekableScanner(plan.getTableDesc().getMeta(), plan.getPhysicalSchema(), fragment, fileScanOutSchema); this.fileScanner.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 2fc4fb417a..467ade1d1d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -209,7 +209,7 @@ public PhysicalExec visit(PhysicalExec exec, Stack stack, Class columns = new ArrayList<>(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) - .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(BuiltinStorages.TEXT).build()) + .setMeta(CatalogProtos.TableProto.newBuilder().setDataFormat(BuiltinStorages.TEXT).build()) .setSchema(CatalogProtos.SchemaProto.newBuilder().addAllFields(columns).build()) .setStats(stats.getProto()) .build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 9e05882bf8..05cc7e362d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -195,7 +195,7 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC int maxStatsScanIdx = -1; StringBuilder nonLeafScanNamesBuilder = new StringBuilder(); for (int i = 0; i < scans.length; i++) { - if (scans[i].getTableDesc().getMeta().getStoreType().equalsIgnoreCase("RAW")) { + if (scans[i].getTableDesc().getMeta().getDataFormat().equalsIgnoreCase("RAW")) { // Intermediate data scan hasNonLeafNode = true; largeScanIndexList.add(i); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index 7a9c8b1826..8e0675a0e4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -276,7 +276,7 @@ private TaskHistory makeTaskHistory() { fragmentList.add(fragment.toString()); } catch (Exception e) { LOG.error(e.getMessage(), e); - fragmentList.add("ERROR: " + eachFragment.getStoreType() + "," + eachFragment.getId() + ": " + e.getMessage()); + fragmentList.add("ERROR: " + eachFragment.getDataFormat() + "," + eachFragment.getId() + ": " + e.getMessage()); } } taskHistory.setFragments(fragmentList.toArray(new String[]{})); diff --git a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp index 6771912762..2388f711c2 100644 --- a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp +++ b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp @@ -186,7 +186,7 @@
Detail
- + diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 7593a9b789..bf089157c9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -24,9 +24,6 @@ import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.OverridableConf; @@ -1401,7 +1398,7 @@ private static LinkedHashSet createFieldTargetsFromRelation(QueryBlock b private void updatePhysicalInfo(TableDesc desc) { // FAKEFILE is used for test - if (!desc.getMeta().getStoreType().equals("SYSTEM") && !desc.getMeta().getStoreType().equals("FAKEFILE")) { + if (!desc.getMeta().getDataFormat().equals("SYSTEM") && !desc.getMeta().getDataFormat().equals("FAKEFILE")) { try { if (desc.getStats() != null) { desc.getStats().setNumBytes(storage.getTableVolumn(desc.getUri())); @@ -1891,7 +1888,7 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, createTableNode.setTableSchema(baseTable.getSchema()); createTableNode.setPartitionMethod(partitionDesc); - createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(baseTable.getMeta().getStoreType())); + createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(baseTable.getMeta().getDataFormat())); createTableNode.setOptions(baseTable.getMeta().getOptions()); createTableNode.setExternal(baseTable.isExternal()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java index 9a169dd171..bf29a2df6a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java @@ -124,7 +124,7 @@ public Object clone() throws CloneNotSupportedException { } public String toString() { - return "CreateTable (table=" + tableName + ", external=" + external + ", storeType=" + storageType + + return "CreateTable (table=" + tableName + ", external=" + external + ", dataFormat=" + storageType + ", ifNotExists=" + ifNotExists +")"; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java index cb3dcee996..1ce334c656 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java @@ -48,7 +48,7 @@ public void setTargetTable(TableDesc desc) { setUri(desc.getUri()); } setOptions(desc.getMeta().getOptions()); - setStorageType(desc.getMeta().getStoreType()); + setStorageType(desc.getMeta().getDataFormat()); if (desc.hasPartition()) { this.setPartitionMethod(desc.getPartitionMethod()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index d722884b67..3d0866aa8a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -183,7 +183,7 @@ public static boolean checkIfQueryTargetIsVirtualTable(LogicalPlan plan) { for (LogicalNode node: scanNodes) { scanNode = (ScanNode) node; - isVirtualTable &= (scanNode.getTableDesc().getMeta().getStoreType().equalsIgnoreCase("SYSTEM")); + isVirtualTable &= (scanNode.getTableDesc().getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")); } return !checkIfDDLPlan(rootNode) && hasScanNode && isVirtualTable; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 9716744c65..81a572a8f5 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -103,7 +103,7 @@ public static void shutdown() throws IOException { } /** - * Returns the proper Tablespace instance according to the storeType. + * Returns the proper Tablespace instance according to the dataFormat. * * @param tajoConf Tajo system property. * @param storeType Storage type @@ -120,7 +120,7 @@ public static Tablespace getStorageManager(TajoConf tajoConf, String storeType) } /** - * Returns the proper Tablespace instance according to the storeType + * Returns the proper Tablespace instance according to the dataFormat * * @param tajoConf Tajo system property. * @param uri Key that can identify each storage manager(may be a path) diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 15130414e9..86ddfe2222 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -209,7 +209,7 @@ public Scanner getScanner(TableMeta meta, Scanner scanner; - Class scannerClass = getScannerClass(meta.getStoreType()); + Class scannerClass = getScannerClass(meta.getDataFormat()); scanner = OldStorageManager.newScannerInstance(scannerClass, conf, schema, meta, fragment); scanner.setTarget(target.toArray()); @@ -256,7 +256,7 @@ public Appender getAppender(OverridableConf queryContext, Class appenderClass; - String handlerName = meta.getStoreType().toLowerCase(); + String handlerName = meta.getDataFormat().toLowerCase(); appenderClass = OldStorageManager.APPENDER_HANDLER_CACHE.get(handlerName); if (appenderClass == null) { appenderClass = conf.getClass( @@ -265,7 +265,7 @@ public Appender getAppender(OverridableConf queryContext, } if (appenderClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getStoreType()); + throw new IOException("Unknown Storage Type: " + meta.getDataFormat()); } appender = OldStorageManager.newAppenderInstance(appenderClass, conf, taskAttemptId, meta, schema, workDir); diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java index bd46551776..739b7036bc 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java @@ -82,9 +82,9 @@ public static T convert(Class clazz, FragmentProto fragm public static T convert(Configuration conf, FragmentProto fragment) throws IOException { - Class fragmentClass = (Class) getFragmentClass(conf, fragment.getStoreType().toLowerCase()); + Class fragmentClass = (Class) getFragmentClass(conf, fragment.getDataFormat().toLowerCase()); if (fragmentClass == null) { - throw new IOException("No such a fragment class for " + fragment.getStoreType()); + throw new IOException("No such a fragment class for " + fragment.getDataFormat()); } return convert(fragmentClass, fragment); } diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java index 2d3f2c3096..18aa515b93 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseFragment.java @@ -172,7 +172,7 @@ public FragmentProto getProto() { FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); fragmentBuilder.setContents(builder.buildPartial().toByteString()); - fragmentBuilder.setStoreType(BuiltinStorages.HBASE); + fragmentBuilder.setDataFormat(BuiltinStorages.HBASE); return fragmentBuilder.build(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 6ac5fc3259..79b2080516 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -230,7 +230,7 @@ public FragmentProto getProto() { FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(this.tableName); - fragmentBuilder.setStoreType(BuiltinStorages.TEXT); + fragmentBuilder.setDataFormat(BuiltinStorages.TEXT); fragmentBuilder.setContents(builder.buildPartial().toByteString()); return fragmentBuilder.build(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java index 4cdf7df7f3..3143ec208b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java @@ -43,7 +43,7 @@ public TextLineSerDe() { public static ByteBuf getNullChars(TableMeta meta) { byte[] nullCharByteArray; - if (meta.getStoreType().equals("SEQUENCEFILE")) { + if (meta.getDataFormat().equals("SEQUENCEFILE")) { nullCharByteArray = getNullCharsAsBytes(meta, StorageConstants.SEQUENCEFILE_NULL, "\\"); } else { nullCharByteArray = getNullCharsAsBytes(meta); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index c93f9e8cd1..0a5e8e39b4 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -166,7 +166,7 @@ public void testMultipleFiles() throws IOException { targetSchema.addColumn(schema.getColumn(2)); Scanner scanner = new MergeScanner(conf, schema, meta, TUtil.newList(fragment), targetSchema); - assertEquals(isProjectableStorage(meta.getStoreType()), scanner.isProjectable()); + assertEquals(isProjectableStorage(meta.getDataFormat()), scanner.isProjectable()); scanner.init(); int totalCounts = 0; diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java index 5437b0dd2a..6880f5937d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java @@ -124,7 +124,7 @@ public void testFindValue() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -148,7 +148,7 @@ public void testFindValue() throws IOException { tuple = new VTuple(keySchema.size()); BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindValue_" + storeType + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -228,7 +228,7 @@ public void testBuildIndexWithAppender() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testBuildIndexWithAppender_" + storeType + ".idx"), keySchema, comp); reader.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -292,7 +292,7 @@ public void testFindOmittedValue() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -365,7 +365,7 @@ public void testFindNextKeyValue() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -389,7 +389,7 @@ public void testFindNextKeyValue() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyValue_" + storeType + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -458,7 +458,7 @@ public void testFindNextKeyOmittedValue() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -482,7 +482,7 @@ public void testFindNextKeyOmittedValue() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindNextKeyOmittedValue_" + storeType + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -540,7 +540,7 @@ public void testFindMinValue() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -566,7 +566,7 @@ public void testFindMinValue() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindMinValue_" + storeType + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -626,7 +626,7 @@ public void testMinMax() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -733,7 +733,7 @@ public void testConcurrentAccess() throws IOException, InterruptedException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -816,7 +816,7 @@ public void testFindValueDescOrder() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -842,7 +842,7 @@ public void testFindValueDescOrder() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "testFindValueDescOrder_" + storeType + ".idx"), keySchema, comp); reader.open(); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -908,7 +908,7 @@ public void testFindNextKeyValueDescOrder() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + SeekableScanner scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); @@ -937,7 +937,7 @@ public void testFindNextKeyValueDescOrder() throws IOException { assertEquals(keySchema, reader.getKeySchema()); assertEquals(comp, reader.getComparator()); - scanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()). + scanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()). getSeekableScanner(meta, schema, tablet.getProto(), schema); scanner.init(); diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index a00149234c..8262073eba 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -111,7 +111,7 @@ public void testFindValueInSingleCSV() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); Tuple keyTuple; @@ -136,7 +136,7 @@ public void testFindValueInSingleCSV() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindValueInCSV.idx"), keySchema, comp); reader.open(); - fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); for (int i = 0; i < TUPLE_NUM - 1; i++) { @@ -202,7 +202,7 @@ public void testFindNextKeyValueInSingleCSV() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + SeekableScanner fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); Tuple keyTuple; @@ -224,7 +224,7 @@ public void testFindNextKeyValueInSingleCSV() throws IOException { BSTIndexReader reader = bst.getIndexReader(new Path(testDir, "FindNextKeyValueInCSV.idx"), keySchema, comp); reader.open(); - fileScanner = OldStorageManager.getStorageManager(conf, meta.getStoreType()) + fileScanner = OldStorageManager.getStorageManager(conf, meta.getDataFormat()) .getSeekableScanner(meta, schema, tablet.getProto(), schema); fileScanner.init(); Tuple result; diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java index bf9536e7e4..1c963fdac7 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcFragment.java @@ -94,7 +94,7 @@ public CatalogProtos.FragmentProto getProto() { CatalogProtos.FragmentProto.Builder fragmentBuilder = CatalogProtos.FragmentProto.newBuilder(); fragmentBuilder.setId(this.inputSourceId); - fragmentBuilder.setStoreType("JDBC"); + fragmentBuilder.setDataFormat("JDBC"); fragmentBuilder.setContents(builder.buildPartial().toByteString()); return fragmentBuilder.build(); } From 0f3c6c9f9a0d71ac4e50561b92a81160f2feb18e Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 24 Sep 2015 17:36:36 -0700 Subject: [PATCH 3/5] Rename StoreType in TableDescriptorProto to DataFormat. --- .../src/main/proto/CatalogProtos.proto | 2 +- .../org/apache/tajo/catalog/store/AbstractDBStore.java | 8 ++++---- .../tajo/catalog/TestCatalogAgainstCaseSensitivity.java | 2 +- .../master/exec/NonForwardQueryResultSystemScanner.java | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index b02c1a8e3a..9a7b66a1c7 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -293,7 +293,7 @@ message TableDescriptorProto { required string name = 3; optional string tableType = 4; required string path = 5; - required string storeType = 6; + required string dataFormat = 6; } message AlterTablespaceProto { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index ceb0dd9768..1a0b02681a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1877,10 +1877,10 @@ public List getAllTables() { } else { builder.setPath(resultSet.getString("PATH")); } - String storeType = resultSet.getString("STORE_TYPE"); - if (storeType != null) { - storeType = storeType.trim(); - builder.setStoreType(storeType); + String dataFormat = resultSet.getString("STORE_TYPE"); + if (dataFormat != null) { + dataFormat = dataFormat.trim(); + builder.setDataFormat(dataFormat); } tables.add(builder.build()); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index cbee4770a0..794ae4a21c 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -115,7 +115,7 @@ public void testTable() throws Exception { assertTrue(tableDescs.containsKey(qualifiedTableName)); TableDesc desc = tableDescs.get(qualifiedTableName); assertEquals(desc.getUri().toString(), eachTableDescriptor.getPath()); - assertEquals(desc.getMeta().getDataFormat(), eachTableDescriptor.getStoreType()); + assertEquals(desc.getMeta().getDataFormat(), eachTableDescriptor.getDataFormat()); } ////////////////////////////////////////////////////////////////////////////// diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 82bc679caf..44f2098f80 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -246,7 +246,7 @@ private List getTables(Schema outSchema) { } else if ("path".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getPath())); } else if ("store_type".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createText(table.getStoreType())); + aTuple.put(fieldId, DatumFactory.createText(table.getDataFormat())); } } From d65c18aedb11e5d493e04c5ecbbef9e500676994 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 25 Sep 2015 04:06:44 -0700 Subject: [PATCH 4/5] Completely changed all storeType names. --- .../java/org/apache/tajo/algebra/Insert.java | 2 +- .../org/apache/tajo/catalog/CatalogUtil.java | 32 +++---- .../org/apache/tajo/catalog/DDLBuilder.java | 2 +- .../org/apache/tajo/catalog/TableDesc.java | 4 +- .../tajo/catalog/json/TableMetaAdapter.java | 4 +- .../tajo/catalog/store/HiveCatalogStore.java | 12 +-- .../tajo/catalog/store/HiveCatalogUtil.java | 2 +- .../dictionary/TablesTableDescriptor.java | 2 +- .../tajo/catalog/store/AbstractDBStore.java | 14 +-- .../main/resources/schemas/derby/derby.xml | 2 +- .../resources/schemas/mariadb/mariadb.xml | 2 +- .../main/resources/schemas/mysql/mysql.xml | 2 +- .../main/resources/schemas/oracle/oracle.xml | 2 +- .../schemas/postgresql/postgresql.xml | 2 +- .../schemas/derbytest/loadtest/derby.xml | 2 +- .../engine/planner/TestQueryValidation.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 4 +- .../tajo/engine/query/TestCTASQuery.java | 4 +- ...thStoreType.sql => CtasWithDataFormat.sql} | 2 +- .../TestCTASQuery/testCtasWithDataFormat.sql | 1 + .../TestCTASQuery/testCtasWithStoreType.sql | 1 - .../engine/planner/PhysicalPlannerImpl.java | 2 +- .../engine/planner/global/DataChannel.java | 22 ++--- .../engine/planner/global/GlobalPlanner.java | 36 +++---- .../builder/DistinctGroupbyBuilder.java | 6 +- .../rewriter/rules/GlobalPlanRewriteUtil.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 16 ++-- .../org/apache/tajo/master/GlobalEngine.java | 4 +- .../NonForwardQueryResultSystemScanner.java | 2 +- .../tajo/querymaster/Repartitioner.java | 4 +- .../org/apache/tajo/querymaster/Stage.java | 8 +- .../org/apache/tajo/plan/LogicalPlanner.java | 8 +- .../apache/tajo/plan/TablePropertyUtil.java | 4 +- .../apache/tajo/plan/logical/InsertNode.java | 2 +- .../plan/logical/PersistentStoreNode.java | 2 +- .../plan/serder/LogicalNodeDeserializer.java | 4 +- .../apache/tajo/plan/util/PlannerUtil.java | 2 +- .../plan/verifier/PreLogicalPlanVerifier.java | 6 +- tajo-plan/src/main/proto/Plan.proto | 2 +- .../tajo/storage/OldStorageManager.java | 16 ++-- .../org/apache/tajo/storage/Tablespace.java | 8 +- .../storage/fragment/FragmentConvertor.java | 10 +- .../tajo/storage/TestCompressionStorages.java | 26 ++--- .../apache/tajo/storage/TestMergeScanner.java | 18 ++-- .../org/apache/tajo/storage/TestStorages.java | 94 +++++++++---------- .../tajo/storage/index/TestBSTIndex.java | 84 ++++++++--------- 46 files changed, 244 insertions(+), 244 deletions(-) rename tajo-core-tests/src/test/resources/queries/TestCTASQuery/{CtasWithStoreType.sql => CtasWithDataFormat.sql} (63%) create mode 100644 tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql delete mode 100644 tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java index 8b9df9763c..4f76886589 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java @@ -98,7 +98,7 @@ public void setStorageType(String storageType) { this.storageType = storageType; } - public String getStorageType() { + public String getDataFormat() { return storageType; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index d50696c0f6..6dd12e6a2a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -281,11 +281,11 @@ public static String getCanonicalTableName(String databaseName, String tableName } - public static String getBackwardCompitablityStoreType(String storeType) { - return getStoreTypeString(getStoreType(storeType)); + public static String getBackwardCompitableDataFormat(String dataFormat) { + return getDataFormatAsString(asDataFormat(dataFormat)); } - public static String getStoreTypeString(final DataFormat type) { + public static String getDataFormatAsString(final DataFormat type) { if (type == DataFormat.TEXTFILE) { return BuiltinStorages.TEXT; } else { @@ -293,7 +293,7 @@ public static String getStoreTypeString(final DataFormat type) { } } - public static DataFormat getStoreType(final String typeStr) { + public static DataFormat asDataFormat(final String typeStr) { if (typeStr.equalsIgnoreCase("CSV")) { return DataFormat.TEXTFILE; } else if (typeStr.equalsIgnoreCase(DataFormat.RAW.name())) { @@ -321,13 +321,13 @@ public static DataFormat getStoreType(final String typeStr) { } } - public static TableMeta newTableMeta(String storeType) { - KeyValueSet defaultProperties = CatalogUtil.newDefaultProperty(storeType); - return new TableMeta(storeType, defaultProperties); + public static TableMeta newTableMeta(String dataFormat) { + KeyValueSet defaultProperties = CatalogUtil.newDefaultProperty(dataFormat); + return new TableMeta(dataFormat, defaultProperties); } - public static TableMeta newTableMeta(String storeType, KeyValueSet options) { - return new TableMeta(storeType, options); + public static TableMeta newTableMeta(String dataFormat, KeyValueSet options) { + return new TableMeta(dataFormat, options); } public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) { @@ -979,21 +979,21 @@ public static Pair, String> getPartitionKeyNamePair(Stri /** * Create new table property with default configs. It is used to make TableMeta to be stored in Catalog. * - * @param storeType DataFormat + * @param dataFormat DataFormat * @return Table properties */ - public static KeyValueSet newDefaultProperty(String storeType) { + public static KeyValueSet newDefaultProperty(String dataFormat) { KeyValueSet options = new KeyValueSet(); - if (storeType.equalsIgnoreCase(BuiltinStorages.TEXT)) { + if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - } else if (storeType.equalsIgnoreCase("JSON")) { + } else if (dataFormat.equalsIgnoreCase("JSON")) { options.set(StorageConstants.TEXT_SERDE_CLASS, "org.apache.tajo.storage.json.JsonLineSerDe"); - } else if (storeType.equalsIgnoreCase("RCFILE")) { + } else if (dataFormat.equalsIgnoreCase("RCFILE")) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); - } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - } else if (storeType.equalsIgnoreCase("PARQUET")) { + } else if (dataFormat.equalsIgnoreCase("PARQUET")) { options.set(BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE); options.set(PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE); options.set(COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index 6c0f171757..a5923cf941 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -114,7 +114,7 @@ public static void buildSchema(StringBuilder sb, Schema schema) { } private static void buildUsingClause(StringBuilder sb, TableMeta meta) { - sb.append(" USING " + CatalogUtil.getBackwardCompitablityStoreType(meta.getDataFormat())); + sb.append(" USING " + CatalogUtil.getBackwardCompitableDataFormat(meta.getDataFormat())); } private static void buildWithClause(final StringBuilder sb, TableMeta meta) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index ccf084edd0..3cdc00b80f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -64,9 +64,9 @@ public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta, @Nul this(tableName, schema, meta, path, true); } - public TableDesc(String tableName, @Nullable Schema schema, String storeType, KeyValueSet options, + public TableDesc(String tableName, @Nullable Schema schema, String dataFormat, KeyValueSet options, @Nullable URI path) { - this(tableName, schema, new TableMeta(storeType, options), path); + this(tableName, schema, new TableMeta(dataFormat, options), path); } public TableDesc(TableDescProto proto) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java index 56bdc63ea2..766a8a9a2d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java @@ -35,10 +35,10 @@ public TableMeta deserialize(JsonElement json, Type typeOfT, Preconditions.checkNotNull(json); JsonObject jsonObject = json.getAsJsonObject(); - String storeType = CommonGsonHelper.getOrDie(jsonObject, "store").getAsString(); + String dataFormat = CommonGsonHelper.getOrDie(jsonObject, "store").getAsString(); KeyValueSet keyValueSet = context.deserialize(CommonGsonHelper.getOrDie(jsonObject, "options"), KeyValueSet.class); - return new TableMeta(storeType, keyValueSet); + return new TableMeta(dataFormat, keyValueSet); } @Override diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 49680150aa..8a370b95cd 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -111,7 +111,7 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St org.apache.hadoop.hive.ql.metadata.Table table = null; HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; Path path = null; - String storeType = null; + String dataFormat = null; org.apache.tajo.catalog.Schema schema = null; KeyValueSet options = null; TableStats stats = null; @@ -186,12 +186,12 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); - storeType = HiveCatalogUtil.getStoreType(fileOutputformat); + dataFormat = HiveCatalogUtil.getDataFormat(fileOutputformat); - if (storeType.equalsIgnoreCase("TEXT")) { + if (dataFormat.equalsIgnoreCase("TEXT")) { options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); - } else if (storeType.equals("RCFILE")) { + } else if (dataFormat.equals("RCFILE")) { options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { @@ -199,7 +199,7 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St } else if (ColumnarSerDe.class.getName().equals(serde)) { options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } - } else if (storeType.equals("SEQUENCEFILE")) { + } else if (dataFormat.equals("SEQUENCEFILE")) { options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); @@ -258,7 +258,7 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St } finally { if(client != null) client.release(); } - TableMeta meta = new TableMeta(storeType, options); + TableMeta meta = new TableMeta(dataFormat, options); TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { tableDesc.setExternal(true); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index a878e0178f..9e1da2bf1f 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -99,7 +99,7 @@ public static String getHiveFieldType(TajoDataTypes.DataType dataType) throws LM } } - public static String getStoreType(String fileFormat) { + public static String getDataFormat(String fileFormat) { Preconditions.checkNotNull(fileFormat); String[] fileFormatArrary = fileFormat.split("\\."); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java index 74852482b9..5522e00480 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/TablesTableDescriptor.java @@ -29,7 +29,7 @@ class TablesTableDescriptor extends AbstractTableDescriptor { new ColumnDescriptor("table_name", Type.TEXT, 0), new ColumnDescriptor("table_type", Type.TEXT, 0), new ColumnDescriptor("path", Type.TEXT, 0), - new ColumnDescriptor("store_type", Type.TEXT, 0) + new ColumnDescriptor("data_format", Type.TEXT, 0) }; public TablesTableDescriptor(InfoSchemaMetadataDictionary metadataDictionary) { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 1a0b02681a..c808a50c8c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -838,7 +838,7 @@ public void createTable(final CatalogProtos.TableDescProto table) conn.setAutoCommit(false); String sql = "INSERT INTO TABLES (DB_ID, " + COL_TABLES_NAME + - ", TABLE_TYPE, PATH, STORE_TYPE, HAS_SELF_DESCRIBE_SCHEMA) VALUES(?, ?, ?, ?, ?, ?) "; + ", TABLE_TYPE, PATH, DATA_FORMAT, HAS_SELF_DESCRIBE_SCHEMA) VALUES(?, ?, ?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(sql); @@ -1666,7 +1666,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa ResultSet res = null; PreparedStatement pstmt = null; CatalogProtos.TableDescProto.Builder tableBuilder = null; - String storeType; + String dataFormat; Pair databaseIdAndUri = getDatabaseIdAndUri(databaseName); @@ -1677,7 +1677,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa // Geting Table Description ////////////////////////////////////////// String sql = - "SELECT TID, " + COL_TABLES_NAME + ", TABLE_TYPE, PATH, STORE_TYPE, HAS_SELF_DESCRIBE_SCHEMA FROM TABLES " + + "SELECT TID, " + COL_TABLES_NAME + ", TABLE_TYPE, PATH, DATA_FORMAT, HAS_SELF_DESCRIBE_SCHEMA FROM TABLES " + "WHERE DB_ID = ? AND " + COL_TABLES_NAME + "=?"; if (LOG.isDebugEnabled()) { @@ -1702,7 +1702,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa } tableBuilder.setPath(res.getString(4).trim()); - storeType = res.getString(5).trim(); + dataFormat = res.getString(5).trim(); boolean hasSelfDescSchema = res.getBoolean(6); res.close(); @@ -1740,7 +1740,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa ////////////////////////////////////////// CatalogProtos.TableProto.Builder metaBuilder = CatalogProtos.TableProto.newBuilder(); - metaBuilder.setDataFormat(storeType); + metaBuilder.setDataFormat(dataFormat); sql = "SELECT key_, value_ FROM " + TB_OPTIONS + " WHERE " + COL_TABLES_PK + " = ?"; if (LOG.isDebugEnabled()) { @@ -1854,7 +1854,7 @@ public List getAllTables() { List tables = new ArrayList<>(); try { - String sql = "SELECT t.TID, t.DB_ID, t." + COL_TABLES_NAME + ", t.TABLE_TYPE, t.PATH, t.STORE_TYPE, " + + String sql = "SELECT t.TID, t.DB_ID, t." + COL_TABLES_NAME + ", t.TABLE_TYPE, t.PATH, t.DATA_FORMAT, " + " s.SPACE_URI FROM " + TB_TABLES + " t, " + TB_DATABASES + " d, " + TB_SPACES + " s WHERE t.DB_ID = d.DB_ID AND d.SPACE_ID = s.SPACE_ID"; @@ -1877,7 +1877,7 @@ public List getAllTables() { } else { builder.setPath(resultSet.getString("PATH")); } - String dataFormat = resultSet.getString("STORE_TYPE"); + String dataFormat = resultSet.getString("DATA_FORMAT"); if (dataFormat != null) { dataFormat = dataFormat.trim(); builder.setDataFormat(dataFormat); diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index 921d98e406..bb16444887 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -68,7 +68,7 @@ TABLE_NAME VARCHAR(128) NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096), - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, CONSTRAINT TABLES_PK PRIMARY KEY (TID), CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME) diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml index a23d393a59..82a81d5ec5 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml @@ -66,7 +66,7 @@ TABLE_NAME VARCHAR(128) BINARY NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096) BINARY, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), INDEX IDX_DB_ID (DB_ID), diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml index 126a5e40b3..33c9b64763 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml @@ -66,7 +66,7 @@ TABLE_NAME VARCHAR(128) BINARY NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096) BINARY, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), INDEX IDX_DB_ID (DB_ID), diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml index b5396ef2ea..26e2e3f28a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml @@ -98,7 +98,7 @@ TABLE_TYPE VARCHAR2(128) NOT NULL, PATH VARCHAR2(4000), HAS_SELF_DESCRIBE_SCHEMA CHAR NOT NULL, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME) )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml index 033c233894..92177d966d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml @@ -75,7 +75,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096), HAS_SELF_DESCRIBE_SCHEMA BOOLEAN, - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID), UNIQUE (DB_ID, TABLE_NAME) )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml b/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml index 86fd5e8f53..f7a2e14e88 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/test/resources/schemas/derbytest/loadtest/derby.xml @@ -54,7 +54,7 @@ TABLE_NAME VARCHAR(128) NOT NULL, TABLE_TYPE VARCHAR(128) NOT NULL, PATH VARCHAR(4096), - STORE_TYPE CHAR(16), + DATA_FORMAT CHAR(16), CONSTRAINT TABLES_PK PRIMARY KEY (TID), CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME) )]]> diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java index cb299b487e..0ef1150789 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java @@ -63,7 +63,7 @@ public void testCaseWhenExprs() throws IOException { } @Test - public void testUnsupportedStoreType() throws IOException { + public void testUnsupportedDataFormat() throws IOException { // See TAJO-1249 assertInvalidSQLFromFile("invalid_store_format.sql"); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 8bc619c17b..70c19b204d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -645,7 +645,7 @@ public final void testPartitionedStorePlan() throws IOException, TajoException { ctx.setDataChannel(dataChannel); LogicalNode rootNode = optimizer.optimize(plan); - TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType()); + TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat()); FileSystem fs = sm.getFileSystem(); QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId(); @@ -779,7 +779,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() ctx.setDataChannel(dataChannel); optimizer.optimize(plan); - TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType()); + TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getDataFormat()); FileSystem fs = sm.getFileSystem(); QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index 4005d867a5..667d193bac 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -210,8 +210,8 @@ public final void testCtasWithMultipleUnions() throws Exception { } @Test - public final void testCtasWithStoreType() throws Exception { - ResultSet res = executeFile("CtasWithStoreType.sql"); + public final void testCtasWithDataFormat() throws Exception { + ResultSet res = executeFile("CtasWithDataFormat.sql"); res.close(); ResultSet res2 = executeQuery(); diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql similarity index 63% rename from tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql rename to tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql index bee1a8910e..c09305ebc8 100644 --- a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithStoreType.sql +++ b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/CtasWithDataFormat.sql @@ -1,4 +1,4 @@ -create table testCtasWithStoreType (col1 float, col2 float) using rcfile as +create table testCtasWithDataFormat (col1 float, col2 float) using rcfile as select sum(l_orderkey) as total1, avg(l_partkey) as total2 diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql new file mode 100644 index 0000000000..5c484f3b52 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithDataFormat.sql @@ -0,0 +1 @@ +select * from testCtasWithDataFormat; \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql b/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql deleted file mode 100644 index 9e7e337b0f..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestCTASQuery/testCtasWithStoreType.sql +++ /dev/null @@ -1 +0,0 @@ -select * from testCtasWithStoreType; \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 74f978b983..cdc21d21d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -106,7 +106,7 @@ private PhysicalExec buildOutputOperator(TaskAttemptContext context, LogicalNode PhysicalExec execPlan) throws IOException { DataChannel channel = context.getDataChannel(); ShuffleFileWriteNode shuffleFileWriteNode = LogicalPlan.createNodeWithoutPID(ShuffleFileWriteNode.class); - shuffleFileWriteNode.setStorageType(context.getDataChannel().getStoreType()); + shuffleFileWriteNode.setDataFormat(context.getDataChannel().getDataFormat()); shuffleFileWriteNode.setInSchema(plan.getOutSchema()); shuffleFileWriteNode.setOutSchema(plan.getOutSchema()); shuffleFileWriteNode.setShuffle(channel.getShuffleType(), channel.getShuffleKeys(), channel.getShuffleOutputNum()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java index 2d32785b23..c779d2f362 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java @@ -40,7 +40,7 @@ public class DataChannel { private Schema schema; - private String storeType = BuiltinStorages.RAW; + private String dataFormat = BuiltinStorages.RAW; public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId) { this.srcId = srcId; @@ -82,8 +82,8 @@ public DataChannel(DataChannelProto proto) { this.numOutputs = proto.getNumOutputs(); } - if (proto.hasStoreType()) { - this.storeType = proto.getStoreType(); + if (proto.hasDataFormat()) { + this.dataFormat = proto.getDataFormat(); } } @@ -144,16 +144,16 @@ public int getShuffleOutputNum() { return numOutputs; } - public boolean hasStoreType() { - return this.storeType != null; + public boolean hasDataFormat() { + return this.dataFormat != null; } - public void setStoreType(String storeType) { - this.storeType = storeType; + public void setDataFormat(String dataFormat) { + this.dataFormat = dataFormat; } - public String getStoreType() { - return storeType; + public String getDataFormat() { + return dataFormat; } public DataChannelProto getProto() { @@ -176,8 +176,8 @@ public DataChannelProto getProto() { builder.setNumOutputs(numOutputs); } - if(storeType != null){ - builder.setStoreType(storeType); + if(dataFormat != null){ + builder.setDataFormat(dataFormat); } return builder.build(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 33277b0d9d..576c880c73 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -68,8 +68,8 @@ public class GlobalPlanner { private static Log LOG = LogFactory.getLog(GlobalPlanner.class); private final TajoConf conf; - private final String storeType; - private final String finalOutputStoreType; + private final String dataFormat; + private final String finalOutputDataFormat; private final CatalogService catalog; private final GlobalPlanRewriteEngine rewriteEngine; @@ -77,8 +77,8 @@ public class GlobalPlanner { public GlobalPlanner(final TajoConf conf, final CatalogService catalog) throws IOException { this.conf = conf; this.catalog = catalog; - this.storeType = conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase(); - this.finalOutputStoreType = conf.getVar(ConfVars.QUERY_OUTPUT_DEFAULT_FILE_FORMAT).toUpperCase(); + this.dataFormat = conf.getVar(ConfVars.SHUFFLE_FILE_FORMAT).toUpperCase(); + this.finalOutputDataFormat = conf.getVar(ConfVars.QUERY_OUTPUT_DEFAULT_FILE_FORMAT).toUpperCase(); Class clazz = (Class) conf.getClassVar(GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); @@ -99,8 +99,8 @@ public CatalogService getCatalog() { return catalog; } - public String getStoreType() { - return storeType; + public String getDataFormat() { + return dataFormat; } public static class GlobalPlanContext { @@ -163,7 +163,7 @@ public void build(QueryContext queryContext, MasterPlan masterPlan) throws IOExc private void setFinalOutputChannel(DataChannel outputChannel, Schema outputSchema) { outputChannel.setShuffleType(NONE_SHUFFLE); outputChannel.setShuffleOutputNum(1); - outputChannel.setStoreType(finalOutputStoreType); + outputChannel.setDataFormat(finalOutputDataFormat); outputChannel.setSchema(outputSchema); } @@ -171,7 +171,7 @@ public static ScanNode buildInputExecutor(LogicalPlan plan, DataChannel channel) Preconditions.checkArgument(channel.getSchema() != null, "Channel schema (" + channel.getSrcId().getId() + " -> " + channel.getTargetId().getId() + ") is not initialized"); - TableMeta meta = new TableMeta(channel.getStoreType(), new KeyValueSet()); + TableMeta meta = new TableMeta(channel.getDataFormat(), new KeyValueSet()); TableDesc desc = new TableDesc( channel.getSrcId().toString(), channel.getSchema(), meta, StorageConstants.LOCAL_FS_URI); ScanNode scanNode = plan.createNode(ScanNode.class); @@ -184,7 +184,7 @@ private DataChannel createDataChannelFromJoin(ExecutionBlock leftBlock, Executio ExecutionBlock childBlock = leftTable ? leftBlock : rightBlock; DataChannel channel = new DataChannel(childBlock, parent, HASH_SHUFFLE, 32); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); if (join.getJoinType() != JoinType.CROSS) { // ShuffleKeys need to not have thea-join condition because Tajo supports only equi-join. Column [][] joinColumns = PlannerUtil.joinJoinKeyForEachTable(join.getJoinQual(), @@ -331,7 +331,7 @@ private void buildJoinPlanWithUnionChannel(GlobalPlanContext context, JoinNode j // create other side channel if (otherSideBlock != null) { DataChannel otherSideChannel = new DataChannel(otherSideBlock, targetBlock, HASH_SHUFFLE, 32); - otherSideChannel.setStoreType(storeType); + otherSideChannel.setDataFormat(dataFormat); if (otherSideShuffleKeys != null) { otherSideChannel.setShuffleKeys(otherSideShuffleKeys); } @@ -587,7 +587,7 @@ private ExecutionBlock buildGroupByIncludingDistinctFunctionsMultiStage(GlobalPl channel = new DataChannel(firstStage, secondStage, HASH_SHUFFLE, 32); channel.setShuffleKeys(secondPhaseGroupby.getGroupingColumns().clone()); channel.setSchema(firstStage.getPlan().getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); // Setting for the second phase's logical plan ScanNode scanNode = buildInputExecutor(context.plan.getLogicalPlan(), channel); @@ -735,7 +735,7 @@ private ExecutionBlock buildTwoPhaseGroupby(MasterPlan masterPlan, ExecutionBloc channel.setShuffleKeys(firstPhaseGroupby.getGroupingColumns()); } channel.setSchema(firstPhaseGroupby.getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); ScanNode scanNode = buildInputExecutor(masterPlan.getLogicalPlan(), channel); secondPhaseGroupby.setChild(scanNode); @@ -888,7 +888,7 @@ private ExecutionBlock buildShuffleAndStorePlanToPartitionedTableWithUnion(Globa ExecutionBlock childBlock = masterPlan.getExecBlock(channel.getSrcId()); setShuffleKeysFromPartitionedTableStore(currentNode, channel); channel.setSchema(childBlock.getPlan().getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); lastChannel = channel; } @@ -911,7 +911,7 @@ private ExecutionBlock buildShuffleAndStorePlanToPartitionedTable(GlobalPlanCont DataChannel channel = new DataChannel(lastBlock, nextBlock, HASH_SHUFFLE, 32); setShuffleKeysFromPartitionedTableStore(currentNode, channel); channel.setSchema(lastBlock.getPlan().getOutSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); ScanNode scanNode = buildInputExecutor(masterPlan.getLogicalPlan(), channel); currentNode.setChild(scanNode); @@ -990,7 +990,7 @@ public LogicalNode visitProjection(GlobalPlanContext context, LogicalPlan plan, MasterPlan masterPlan = context.plan; for (DataChannel dataChannel : masterPlan.getIncomingChannels(execBlock.getId())) { - dataChannel.setStoreType(finalOutputStoreType); + dataChannel.setDataFormat(finalOutputDataFormat); ExecutionBlock subBlock = masterPlan.getExecBlock(dataChannel.getSrcId()); ProjectionNode copy = PlannerUtil.clone(plan, node); @@ -1035,7 +1035,7 @@ public LogicalNode visitLimit(GlobalPlanContext context, LogicalPlan plan, Logic DataChannel newChannel = new DataChannel(execBlock, newExecBlock, HASH_SHUFFLE, 1); newChannel.setShuffleKeys(new Column[]{}); newChannel.setSchema(node.getOutSchema()); - newChannel.setStoreType(storeType); + newChannel.setDataFormat(dataFormat); ScanNode scanNode = buildInputExecutor(plan, newChannel); LimitNode parentLimit = PlannerUtil.clone(context.plan.getLogicalPlan(), node); @@ -1103,7 +1103,7 @@ private ExecutionBlock buildWindowAgg(GlobalPlanContext context, ExecutionBlock channel.setShuffleKeys(null); } channel.setSchema(windowAgg.getInSchema()); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); LogicalNode childNode = windowAgg.getChild(); ScanNode scanNode = buildInputExecutor(masterPlan.getLogicalPlan(), channel); @@ -1236,7 +1236,7 @@ public LogicalNode visitUnion(GlobalPlanContext context, LogicalPlan plan, Logic for (ExecutionBlock childBlocks : queryBlockBlocks) { DataChannel channel = new DataChannel(childBlocks, execBlock, NONE_SHUFFLE, 1); - channel.setStoreType(storeType); + channel.setDataFormat(dataFormat); masterPlan.addConnect(channel); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 459d9af8f0..075ec15162 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -142,7 +142,7 @@ public ExecutionBlock buildMultiLevelPlan(GlobalPlanContext context, firstChannel.setShuffleKeys(firstStageDistinctNode.getFirstStageShuffleKeyColumns()); firstChannel.setSchema(firstStageDistinctNode.getOutSchema()); - firstChannel.setStoreType(globalPlanner.getStoreType()); + firstChannel.setDataFormat(globalPlanner.getDataFormat()); ScanNode scanNode = GlobalPlanner.buildInputExecutor(context.getPlan().getLogicalPlan(), firstChannel); secondStageDistinctNode.setChild(scanNode); @@ -161,7 +161,7 @@ public ExecutionBlock buildMultiLevelPlan(GlobalPlanContext context, secondChannel.setShuffleKeys(firstStageDistinctNode.getGroupingColumns()); } secondChannel.setSchema(secondStageDistinctNode.getOutSchema()); - secondChannel.setStoreType(globalPlanner.getStoreType()); + secondChannel.setDataFormat(globalPlanner.getDataFormat()); scanNode = GlobalPlanner.buildInputExecutor(context.getPlan().getLogicalPlan(), secondChannel); thirdStageDistinctNode.setChild(scanNode); @@ -333,7 +333,7 @@ public ExecutionBlock buildPlan(GlobalPlanContext context, channel.setShuffleKeys(firstStageDistinctNode.getGroupingColumns()); } channel.setSchema(firstStageDistinctNode.getOutSchema()); - channel.setStoreType(globalPlanner.getStoreType()); + channel.setDataFormat(globalPlanner.getDataFormat()); ScanNode scanNode = GlobalPlanner.buildInputExecutor(context.getPlan().getLogicalPlan(), channel); secondStageDistinctNode.setChild(scanNode); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java index df1f33bfed..b13cb0f1a8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/rules/GlobalPlanRewriteUtil.java @@ -48,7 +48,7 @@ public static ExecutionBlock mergeExecutionBlocks(MasterPlan plan, ExecutionBloc originalChannel.getShuffleOutputNum()); newChannel.setSchema(originalChannel.getSchema()); newChannel.setShuffleKeys(originalChannel.getShuffleKeys()); - newChannel.setStoreType(originalChannel.getStoreType()); + newChannel.setDataFormat(originalChannel.getDataFormat()); newChannel.setTransmitType(originalChannel.getTransmitType()); plan.addConnect(newChannel); plan.disconnect(eachGrandChild, child); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 467ade1d1d..6bd084b444 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -209,12 +209,12 @@ public PhysicalExec visit(PhysicalExec exec, Stack stack, Class getTables(Schema outSchema) { } } else if ("path".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getPath())); - } else if ("store_type".equalsIgnoreCase(column.getSimpleName())) { + } else if ("data_format".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(table.getDataFormat())); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 05cc7e362d..4fe7718f00 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -636,7 +636,7 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo TupleRange mergedRange = TupleUtil.columnStatToRange(sortSpecs, sortSchema, totalStat.getColumnStats(), false); if (sortNode.getSortPurpose() == SortPurpose.STORAGE_SPECIFIED) { - String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); + String dataFormat = PlannerUtil.getDataFormat(masterPlan.getLogicalPlan()); CatalogService catalog = stage.getContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalRootNode rootNode = masterPlan.getLogicalPlan().getRootBlock().getRoot(); TableDesc tableDesc = null; @@ -647,7 +647,7 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo PlannerUtil.getStoreTableName(masterPlan.getLogicalPlan())); } - Tablespace space = TablespaceManager.getAnyByScheme(storeType).get(); + Tablespace space = TablespaceManager.getAnyByScheme(dataFormat).get(); ranges = space.getInsertSortRanges( stage.getContext().getQueryContext(), tableDesc, diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 74edc480f9..ece1616b9f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -775,14 +775,14 @@ private void finalizeStats() { DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0); // if store plan (i.e., CREATE or INSERT OVERWRITE) - String storeType = PlannerUtil.getStoreType(masterPlan.getLogicalPlan()); - if (storeType == null) { + String dataFormat = PlannerUtil.getDataFormat(masterPlan.getLogicalPlan()); + if (dataFormat == null) { // get final output store type (i.e., SELECT) - storeType = channel.getStoreType(); + dataFormat = channel.getDataFormat(); } schema = channel.getSchema(); - meta = CatalogUtil.newTableMeta(storeType, new KeyValueSet()); + meta = CatalogUtil.newTableMeta(dataFormat, new KeyValueSet()); inputStatistics = statsArray[0]; resultStatistics = statsArray[1]; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index bf089157c9..a140bd2219 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1841,7 +1841,7 @@ private InsertNode buildInsertIntoLocationPlan(PlanContext context, InsertNode i insertNode.setUri(targetUri); if (expr.hasStorageType()) { - insertNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(expr.getStorageType())); + insertNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(expr.getDataFormat())); } if (expr.hasParams()) { KeyValueSet options = new KeyValueSet(); @@ -1888,7 +1888,7 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, createTableNode.setTableSchema(baseTable.getSchema()); createTableNode.setPartitionMethod(partitionDesc); - createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(baseTable.getMeta().getDataFormat())); + createTableNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(baseTable.getMeta().getDataFormat())); createTableNode.setOptions(baseTable.getMeta().getOptions()); createTableNode.setExternal(baseTable.isExternal()); @@ -1924,9 +1924,9 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea createTableNode.setUri(getCreatedTableURI(context, expr)); if (expr.hasStorageType()) { // If storage type (using clause) is specified - createTableNode.setStorageType(CatalogUtil.getBackwardCompitablityStoreType(expr.getStorageType())); + createTableNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(expr.getStorageType())); } else { // otherwise, default type - createTableNode.setStorageType(BuiltinStorages.TEXT); + createTableNode.setDataFormat(BuiltinStorages.TEXT); } // Set default storage properties to table diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java index 62fa706b1a..4bf2734d48 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java @@ -38,10 +38,10 @@ public class TablePropertyUtil { * @param node CreateTableNode */ public static void setTableProperty(OverridableConf context, CreateTableNode node) { - String storeType = node.getStorageType(); + String dataFormat = node.getStorageType(); KeyValueSet property = node.getOptions(); - if (storeType.equalsIgnoreCase("TEXT") || storeType.equalsIgnoreCase("TEXT")) { + if (dataFormat.equalsIgnoreCase("TEXT") || dataFormat.equalsIgnoreCase("TEXT")) { setSessionToProperty(context, SessionVars.NULL_CHAR, property, StorageConstants.TEXT_NULL); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java index 1ce334c656..4030b69338 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java @@ -48,7 +48,7 @@ public void setTargetTable(TableDesc desc) { setUri(desc.getUri()); } setOptions(desc.getMeta().getOptions()); - setStorageType(desc.getMeta().getDataFormat()); + setDataFormat(desc.getMeta().getDataFormat()); if (desc.hasPartition()) { this.setPartitionMethod(desc.getPartitionMethod()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java index a28f01f4d7..30e3260aa8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java @@ -39,7 +39,7 @@ protected PersistentStoreNode(int pid, NodeType nodeType) { super(pid, nodeType); } - public void setStorageType(String storageType) { + public void setDataFormat(String storageType) { this.storageType = storageType; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index c75c3fd0fd..2051dfb0de 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -502,7 +502,7 @@ private static CreateTableNode convertCreateTable(Map node createTable.setOutSchema(convertSchema(protoNode.getOutSchema())); } createTable.setChild(nodeMap.get(persistentStoreProto.getChildSeq())); - createTable.setStorageType(persistentStoreProto.getStorageType()); + createTable.setDataFormat(persistentStoreProto.getStorageType()); createTable.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); createTable.setTableName(storeTableNodeSpec.getTableName()); @@ -540,7 +540,7 @@ private static InsertNode convertInsert(Map nodeMap, insertNode.setOutSchema(convertSchema(protoNode.getOutSchema())); } insertNode.setChild(nodeMap.get(persistentStoreProto.getChildSeq())); - insertNode.setStorageType(persistentStoreProto.getStorageType()); + insertNode.setDataFormat(persistentStoreProto.getStorageType()); insertNode.setOptions(new KeyValueSet(persistentStoreProto.getTableProperties())); if (storeTableNodeSpec.hasTableName()) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 3d0866aa8a..188ed33a7e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -842,7 +842,7 @@ public static boolean isFileStorageType(String storageType) { } } - public static String getStoreType(LogicalPlan plan) { + public static String getDataFormat(LogicalPlan plan) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); NodeType nodeType = rootNode.getChild().getType(); if (nodeType == NodeType.CREATE_TABLE) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 2c7feaca7a..c16736df73 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -190,7 +190,7 @@ private boolean assertRelationNoExistence(Context context, String tableName) { return true; } - private boolean assertSupportedStoreType(VerificationState state, String name) { + private boolean assertSupportedDataFormat(VerificationState state, String name) { Preconditions.checkNotNull(name); if (name.equalsIgnoreCase("RAW")) { @@ -271,7 +271,7 @@ public Expr visitCreateTable(Context context, Stack stack, CreateTable exp } if (expr.hasStorageType()) { - assertSupportedStoreType(context.state, expr.getStorageType()); + assertSupportedDataFormat(context.state, expr.getStorageType()); } return expr; @@ -298,7 +298,7 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws } if (expr.hasStorageType()) { - assertSupportedStoreType(context.state, expr.getStorageType()); + assertSupportedDataFormat(context.state, expr.getDataFormat()); } if (child != null && child.getType() == OpType.Projection) { diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index fa1deeb8e7..0cd0c32dcc 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -566,7 +566,7 @@ message DataChannelProto { repeated ColumnProto shuffleKeys = 7; optional int32 numOutputs = 9 [default = 1]; - optional string storeType = 10; + optional string dataFormat = 10; } message EnforcerProto { diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 81a572a8f5..14db7c0bc3 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -106,16 +106,16 @@ public static void shutdown() throws IOException { * Returns the proper Tablespace instance according to the dataFormat. * * @param tajoConf Tajo system property. - * @param storeType Storage type + * @param dataFormat Storage type * @return * @throws IOException */ - public static Tablespace getStorageManager(TajoConf tajoConf, String storeType) throws IOException { + public static Tablespace getStorageManager(TajoConf tajoConf, String dataFormat) throws IOException { FileSystem fileSystem = TajoConf.getWarehouseDir(tajoConf).getFileSystem(tajoConf); if (fileSystem != null) { - return getStorageManager(tajoConf, fileSystem.getUri(), storeType); + return getStorageManager(tajoConf, fileSystem.getUri(), dataFormat); } else { - return getStorageManager(tajoConf, null, storeType); + return getStorageManager(tajoConf, null, dataFormat); } } @@ -124,18 +124,18 @@ public static Tablespace getStorageManager(TajoConf tajoConf, String storeType) * * @param tajoConf Tajo system property. * @param uri Key that can identify each storage manager(may be a path) - * @param storeType Storage type + * @param dataFormat Storage type * @return * @throws IOException */ public static synchronized Tablespace getStorageManager( - TajoConf tajoConf, URI uri, String storeType) throws IOException { + TajoConf tajoConf, URI uri, String dataFormat) throws IOException { Preconditions.checkNotNull(tajoConf); Preconditions.checkNotNull(uri); - Preconditions.checkNotNull(storeType); + Preconditions.checkNotNull(dataFormat); String typeName; - if (storeType.equalsIgnoreCase("HBASE")) { + if (dataFormat.equalsIgnoreCase("HBASE")) { typeName = "hbase"; } else { typeName = "hdfs"; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 86ddfe2222..81fbbc0ce7 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -276,12 +276,12 @@ public Appender getAppender(OverridableConf queryContext, /** * Return the Scanner class for the DataFormat that is defined in storage-default.xml. * - * @param storeType store type + * @param dataFormat store type * @return The Scanner class * @throws java.io.IOException */ - public Class getScannerClass(String storeType) throws IOException { - String handlerName = storeType.toLowerCase(); + public Class getScannerClass(String dataFormat) throws IOException { + String handlerName = dataFormat.toLowerCase(); Class scannerClass = OldStorageManager.SCANNER_HANDLER_CACHE.get(handlerName); if (scannerClass == null) { scannerClass = conf.getClass( @@ -290,7 +290,7 @@ public Class getScannerClass(String storeType) throws IOExcep } if (scannerClass == null) { - throw new IOException("Unknown Storage Type: " + storeType); + throw new IOException("Unknown Storage Type: " + dataFormat); } return scannerClass; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java index 739b7036bc..d9405e2647 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java @@ -47,17 +47,17 @@ public class FragmentConvertor { */ private static final Class[] DEFAULT_FRAGMENT_PARAMS = { ByteString.class }; - public static Class getFragmentClass(Configuration conf, String storeType) + public static Class getFragmentClass(Configuration conf, String dataFormat) throws IOException { - Class fragmentClass = CACHED_FRAGMENT_CLASSES.get(storeType.toLowerCase()); + Class fragmentClass = CACHED_FRAGMENT_CLASSES.get(dataFormat.toLowerCase()); if (fragmentClass == null) { fragmentClass = conf.getClass( - String.format("tajo.storage.fragment.%s.class", storeType.toLowerCase()), null, Fragment.class); - CACHED_FRAGMENT_CLASSES.put(storeType.toLowerCase(), fragmentClass); + String.format("tajo.storage.fragment.%s.class", dataFormat.toLowerCase()), null, Fragment.class); + CACHED_FRAGMENT_CLASSES.put(dataFormat.toLowerCase(), fragmentClass); } if (fragmentClass == null) { - throw new IOException("No such a fragment for " + storeType.toLowerCase()); + throw new IOException("No such a fragment for " + dataFormat.toLowerCase()); } return fragmentClass; diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 67d0646e31..cc691196be 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -53,12 +53,12 @@ public class TestCompressionStorages { private TajoConf conf; private static String TEST_PATH = "target/test-data/TestCompressionStorages"; - private String storeType; + private String dataFormat; private Path testDir; private FileSystem fs; public TestCompressionStorages(String type) throws IOException { - this.storeType = type; + this.dataFormat = type; conf = new TajoConf(); testDir = CommonTestingUtil.getTestDir(TEST_PATH); @@ -76,44 +76,44 @@ public static Collection generateParameters() { @Test public void testDeflateCodecCompressionData() throws IOException { - storageCompressionTest(storeType, DeflateCodec.class); + storageCompressionTest(dataFormat, DeflateCodec.class); } @Test public void testGzipCodecCompressionData() throws IOException { - if (storeType.equalsIgnoreCase("RCFILE")) { + if (dataFormat.equalsIgnoreCase("RCFILE")) { if( ZlibFactory.isNativeZlibLoaded(conf)) { - storageCompressionTest(storeType, GzipCodec.class); + storageCompressionTest(dataFormat, GzipCodec.class); } - } else if (storeType.equalsIgnoreCase("SEQUENCEFILE")) { + } else if (dataFormat.equalsIgnoreCase("SEQUENCEFILE")) { if( ZlibFactory.isNativeZlibLoaded(conf)) { - storageCompressionTest(storeType, GzipCodec.class); + storageCompressionTest(dataFormat, GzipCodec.class); } } else { - storageCompressionTest(storeType, GzipCodec.class); + storageCompressionTest(dataFormat, GzipCodec.class); } } @Test public void testSnappyCodecCompressionData() throws IOException { if (SnappyCodec.isNativeCodeLoaded()) { - storageCompressionTest(storeType, SnappyCodec.class); + storageCompressionTest(dataFormat, SnappyCodec.class); } } @Test public void testLz4CodecCompressionData() throws IOException { if(NativeCodeLoader.isNativeCodeLoaded() && Lz4Codec.isNativeCodeLoaded()) - storageCompressionTest(storeType, Lz4Codec.class); + storageCompressionTest(dataFormat, Lz4Codec.class); } - private void storageCompressionTest(String storeType, Class codec) throws IOException { + private void storageCompressionTest(String dataFormat, Class codec) throws IOException { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("age", Type.FLOAT4); schema.addColumn("name", Type.TEXT); - TableMeta meta = CatalogUtil.newTableMeta(storeType); + TableMeta meta = CatalogUtil.newTableMeta(dataFormat); meta.putOption("compression.codec", codec.getCanonicalName()); meta.putOption("compression.type", SequenceFile.CompressionType.BLOCK.name()); meta.putOption("rcfile.serde", TextSerializerDeserializer.class.getName()); @@ -154,7 +154,7 @@ private void storageCompressionTest(String storeType, Class Date: Mon, 28 Sep 2015 14:16:48 -0700 Subject: [PATCH 5/5] Bump up the catalog schema version. --- .../src/main/resources/schemas/derby/derby.xml | 3 ++- .../src/main/resources/schemas/mariadb/mariadb.xml | 3 ++- .../src/main/resources/schemas/mysql/mysql.xml | 3 ++- .../src/main/resources/schemas/oracle/oracle.xml | 3 ++- .../src/main/resources/schemas/postgresql/postgresql.xml | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index bb16444887..ff0b20bb01 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -19,6 +19,7 @@ - + diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml index 82a81d5ec5..dbf82b6e3a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml @@ -19,6 +19,7 @@ - + diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml index 33c9b64763..59ceae460f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml @@ -19,6 +19,7 @@ - + diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml index 26e2e3f28a..b61778a74a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml @@ -19,6 +19,7 @@ - + - +
Table path<%=tableDesc.getUri()%>
Store type<%=tableDesc.getMeta().getStoreType()%>
Store type<%=tableDesc.getMeta().getDataFormat()%>
# rows<%=(tableDesc.hasStats() ? ("" + tableDesc.getStats().getNumRows()) : "-")%>
Volume<%=(tableDesc.hasStats() ? FileUtil.humanReadableByteCount(tableDesc.getStats().getNumBytes(),true) : "-")%>
Options<%=optionStr%>