From 8a6e0ab9d4f9fa451f5568534e0f924c8161d4e0 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 23 Nov 2015 20:45:21 -0800 Subject: [PATCH 1/2] TAJO-1986: Rename the name 'option' to property in TableMeta. --- .../org/apache/tajo/catalog/DDLBuilder.java | 6 +-- .../org/apache/tajo/catalog/TableMeta.java | 42 +++++++++---------- .../tajo/catalog/json/TableMetaAdapter.java | 2 +- .../apache/tajo/catalog/TestTableMeta.java | 2 +- .../tajo/catalog/store/HiveCatalogStore.java | 26 ++++++------ .../catalog/store/TestHiveCatalogStore.java | 24 +++++------ .../tajo/catalog/CatalogTestingUtil.java | 2 +- .../org/apache/tajo/catalog/TestCatalog.java | 6 +-- .../TestCatalogAgainstCaseSensitivity.java | 10 ++--- .../org/apache/tajo/QueryTestCaseBase.java | 2 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 4 +- .../apache/tajo/client/TestTajoClient.java | 2 +- .../tajo/engine/query/TestCTASQuery.java | 2 +- .../tajo/engine/query/TestCreateTable.java | 4 +- .../java/org/apache/tajo/benchmark/TPCH.java | 2 +- .../planner/physical/PhysicalPlanUtil.java | 12 +++--- .../tajo/master/exec/QueryExecutor.java | 4 +- .../org/apache/tajo/querymaster/Query.java | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 2 +- .../apache/tajo/plan/TablePropertyUtil.java | 4 +- .../plan/function/stream/CSVLineSerDe.java | 2 +- .../TextFieldSerializerDeserializer.java | 4 +- .../plan/function/stream/TextLineSerDe.java | 2 +- .../apache/tajo/plan/logical/InsertNode.java | 2 +- .../storage/hbase/AbstractHBaseAppender.java | 2 +- .../tajo/storage/hbase/HBaseScanner.java | 4 +- .../tajo/storage/hbase/HBaseTablespace.java | 32 +++++++------- .../tajo/storage/hbase/TestColumnMapping.java | 2 +- .../java/org/apache/tajo/storage/RowFile.java | 2 +- .../apache/tajo/storage/avro/AvroUtil.java | 8 ++-- .../storage/json/JsonLineDeserializer.java | 4 +- .../tajo/storage/json/JsonLineSerializer.java | 4 +- .../apache/tajo/storage/orc/ORCAppender.java | 10 ++--- .../apache/tajo/storage/orc/ORCScanner.java | 4 +- .../tajo/storage/parquet/ParquetAppender.java | 10 ++--- .../storage/rawfile/DirectRawFileWriter.java | 2 +- .../apache/tajo/storage/rcfile/RCFile.java | 12 +++--- .../sequencefile/SequenceFileAppender.java | 12 +++--- .../sequencefile/SequenceFileScanner.java | 6 +-- .../storage/text/CSVLineDeserializer.java | 4 +- .../tajo/storage/text/CSVLineSerDe.java | 2 +- .../tajo/storage/text/DelimitedTextFile.java | 10 ++--- .../text/TextFieldSerializerDeserializer.java | 4 +- .../tajo/storage/text/TextLineSerDe.java | 2 +- .../tajo/storage/TestCompressionStorages.java | 8 ++-- .../tajo/storage/TestDelimitedTextFile.java | 26 +++++------- .../apache/tajo/storage/TestLineReader.java | 2 +- .../apache/tajo/storage/TestMergeScanner.java | 4 +- .../org/apache/tajo/storage/TestStorages.java | 42 +++++++++---------- .../tajo/storage/avro/TestAvroUtil.java | 6 +-- 50 files changed, 195 insertions(+), 199 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index a597d14a60..e2250eb9ef 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -120,14 +120,14 @@ private static void buildUsingClause(StringBuilder sb, TableMeta meta) { } private static void buildWithClause(final StringBuilder sb, TableMeta meta) { - KeyValueSet options = meta.getOptions(); + KeyValueSet options = meta.getPropertySet(); if (options != null && options.size() > 0) { sb.append(" WITH ("); // sort table properties in an lexicographic order of the property keys. - Entry [] entries = meta.getOptions().getAllKeyValus().entrySet().toArray( - new Entry[meta.getOptions().size()]); + Entry [] entries = meta.getPropertySet().getAllKeyValus().entrySet().toArray( + new Entry[meta.getPropertySet().size()]); Arrays.sort(entries, new Comparator>() { @Override diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index cfbbf75553..6f37b8c3c9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -36,48 +36,48 @@ */ public class TableMeta implements ProtoObject, GsonObject, Cloneable { @Expose protected String dataFormat; - @Expose protected KeyValueSet options; + @Expose protected KeyValueSet propertySet; - public TableMeta(String dataFormat, KeyValueSet options) { + public TableMeta(String dataFormat, KeyValueSet propertySet) { this.dataFormat = dataFormat; - this.options = new KeyValueSet(options); + this.propertySet = new KeyValueSet(propertySet); } public TableMeta(TableProto proto) { this.dataFormat = proto.getDataFormat(); - this.options = new KeyValueSet(proto.getParams()); + this.propertySet = new KeyValueSet(proto.getParams()); } public String getDataFormat() { return this.dataFormat; } - public void setOptions(KeyValueSet options) { - this.options = options; + public void setPropertySet(KeyValueSet options) { + this.propertySet = options; } - public void putOption(String key, String val) { - options.set(key, val); + public KeyValueSet getPropertySet() { + return propertySet; } - public boolean containsOption(String key) { - return options.containsKey(key); + public void putProperty(String key, String val) { + propertySet.set(key, val); } - public String getOption(String key) { - return options.get(key); + public boolean containsProperty(String key) { + return propertySet.containsKey(key); } - public String getOption(String key, String defaultValue) { - return options.get(key, defaultValue); + public String getProperty(String key) { + return propertySet.get(key); } - public KeyValueSet getOptions() { - return options; + public String getProperty(String key, String defaultValue) { + return propertySet.get(key, defaultValue); } public Map toMap() { - return getOptions().getAllKeyValus(); + return getPropertySet().getAllKeyValus(); } public boolean equals(Object object) { @@ -85,7 +85,7 @@ public boolean equals(Object object) { TableMeta other = (TableMeta) object; boolean eq = this.getDataFormat().equals(other.getDataFormat()); - eq = eq && this.getOptions().equals(other.getOptions()); + eq = eq && this.getPropertySet().equals(other.getPropertySet()); return eq; } @@ -93,14 +93,14 @@ public boolean equals(Object object) { } public int hashCode() { - return Objects.hashCode(getDataFormat(), getOptions()); + return Objects.hashCode(getDataFormat(), getPropertySet()); } @Override public Object clone() throws CloneNotSupportedException { TableMeta meta = (TableMeta) super.clone(); meta.dataFormat = getDataFormat(); - meta.options = (KeyValueSet) (toMap() != null ? options.clone() : null); + meta.propertySet = (KeyValueSet) (toMap() != null ? propertySet.clone() : null); return meta; } @@ -116,7 +116,7 @@ public String toString() { public TableProto getProto() { TableProto.Builder builder = TableProto.newBuilder(); builder.setDataFormat(dataFormat); - builder.setParams(options.getProto()); + builder.setParams(propertySet.getProto()); return builder.build(); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java index 766a8a9a2d..6b2e931502 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/TableMetaAdapter.java @@ -46,7 +46,7 @@ public JsonElement serialize(TableMeta src, Type typeOfSrc, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.addProperty("store", src.getDataFormat()); - jsonObj.add("options", context.serialize(src.getOptions(), KeyValueSet.class)); + jsonObj.add("options", context.serialize(src.getPropertySet(), KeyValueSet.class)); return jsonObj; } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java index 3f1d5c5ac9..d85fc4870e 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableMeta.java @@ -98,7 +98,7 @@ public void testEqualsObject2() { TableMeta meta1 = CatalogUtil.newTableMeta(BuiltinStorages.TEXT); for (int i = 0; i < MAX_COUNT; i++) { - meta1.putOption("key"+i, "value"+i); + meta1.putProperty("key"+i, "value"+i); } PrimitiveProtos.KeyValueSetProto.Builder optionBuilder = PrimitiveProtos.KeyValueSetProto.newBuilder(); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index f8bc07e410..f025135b78 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -448,7 +448,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api.Table(); table.setDbName(databaseName); table.setTableName(tableName); - table.setParameters(new HashMap<>(tableDesc.getMeta().getOptions().getAllKeyValus())); + table.setParameters(new HashMap<>(tableDesc.getMeta().getPropertySet().getAllKeyValus())); // TODO: set owner //table.setOwner(); @@ -496,16 +496,16 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) sd.setInputFormat(descriptor.getInputFormat()); sd.setOutputFormat(descriptor.getOutputFormat()); - String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE); + String serde = tableDesc.getMeta().getProperty(StorageConstants.RCFILE_SERDE); if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { sd.getSerdeInfo().setSerializationLib(ColumnarSerDe.class.getName()); } else { sd.getSerdeInfo().setSerializationLib(LazyBinaryColumnarSerDe.class.getName()); } - if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) { + if (tableDesc.getMeta().getPropertySet().containsKey(StorageConstants.RCFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, - StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); + StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.RCFILE_NULL))); } } else if (tableDesc.getMeta().getDataFormat().equals(BuiltinStorages.TEXT)) { // TextFileStorageFormatDescriptor has deprecated class. so the class name set directly @@ -513,7 +513,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) sd.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getName()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); - String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.TEXT_DELIMITER, + String fieldDelimiter = tableDesc.getMeta().getProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); // User can use an unicode for filed delimiter such as \u0001, \001. @@ -527,9 +527,9 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) StringEscapeUtils.unescapeJava(fieldDelimiter)); table.getParameters().remove(StorageConstants.TEXT_DELIMITER); - if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) { + if (tableDesc.getMeta().containsProperty(StorageConstants.TEXT_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, - StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL))); + StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.TEXT_NULL))); table.getParameters().remove(StorageConstants.TEXT_NULL); } } else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { @@ -537,12 +537,12 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) sd.setInputFormat(descriptor.getInputFormat()); sd.setOutputFormat(descriptor.getOutputFormat()); - String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE); + String serde = tableDesc.getMeta().getProperty(StorageConstants.SEQUENCEFILE_SERDE); if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); - String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_DELIMITER, + String fieldDelimiter = tableDesc.getMeta().getProperty(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); // User can use an unicode for filed delimiter such as \u0001, \001. @@ -559,9 +559,9 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) sd.getSerdeInfo().setSerializationLib(LazyBinarySerDe.class.getName()); } - if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) { + if (tableDesc.getMeta().containsProperty(StorageConstants.SEQUENCEFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, - StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL))); + StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.SEQUENCEFILE_NULL))); table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); } } else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.PARQUET)) { @@ -570,9 +570,9 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) sd.setOutputFormat(descriptor.getOutputFormat()); sd.getSerdeInfo().setSerializationLib(descriptor.getSerde()); - if (tableDesc.getMeta().containsOption(ParquetOutputFormat.COMPRESSION)) { + if (tableDesc.getMeta().containsProperty(ParquetOutputFormat.COMPRESSION)) { table.putToParameters(ParquetOutputFormat.COMPRESSION, - tableDesc.getMeta().getOption(ParquetOutputFormat.COMPRESSION)); + tableDesc.getMeta().getProperty(ParquetOutputFormat.COMPRESSION)); } } else { throw new UnsupportedException(tableDesc.getMeta().getDataFormat() + " in HivecatalogStore"); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 7608a7b329..b4bc53f5fc 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -125,7 +125,7 @@ public void testTableUsingTextFile() throws Exception { } assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER), - table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); + table1.getMeta().getProperty(StorageConstants.TEXT_DELIMITER)); store.dropTable(DB_NAME, CUSTOMER); } @@ -159,7 +159,7 @@ public void testTableUsingRCFileWithBinarySerde() throws Exception { } assertEquals(StorageConstants.DEFAULT_BINARY_SERDE, - table1.getMeta().getOption(StorageConstants.RCFILE_SERDE)); + table1.getMeta().getProperty(StorageConstants.RCFILE_SERDE)); store.dropTable(DB_NAME, REGION); } @@ -192,7 +192,7 @@ public void testTableUsingRCFileWithTextSerde() throws Exception { assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); } - assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.RCFILE_SERDE)); + assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getProperty(StorageConstants.RCFILE_SERDE)); store.dropTable(DB_NAME, REGION); } @@ -232,16 +232,16 @@ public void testTableWithNullValue() throws Exception { assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); } - assertEquals(table.getMeta().getOption(StorageConstants.TEXT_DELIMITER), - table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); + assertEquals(table.getMeta().getProperty(StorageConstants.TEXT_DELIMITER), + table1.getMeta().getProperty(StorageConstants.TEXT_DELIMITER)); - assertEquals(table.getMeta().getOption(StorageConstants.TEXT_NULL), - table1.getMeta().getOption(StorageConstants.TEXT_NULL)); + assertEquals(table.getMeta().getProperty(StorageConstants.TEXT_NULL), + table1.getMeta().getProperty(StorageConstants.TEXT_NULL)); - assertEquals(table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER), + assertEquals(table1.getMeta().getProperty(StorageConstants.TEXT_DELIMITER), StringEscapeUtils.escapeJava("\u0002")); - assertEquals(table1.getMeta().getOption(StorageConstants.TEXT_NULL), + assertEquals(table1.getMeta().getProperty(StorageConstants.TEXT_NULL), StringEscapeUtils.escapeJava("\u0003")); store.dropTable(DB_NAME, SUPPLIER); @@ -577,7 +577,7 @@ public void testTableUsingSequenceFileWithBinarySerde() throws Exception { } assertEquals(StorageConstants.DEFAULT_BINARY_SERDE, - table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE)); + table1.getMeta().getProperty(StorageConstants.SEQUENCEFILE_SERDE)); store.dropTable(DB_NAME, REGION); } @@ -610,7 +610,7 @@ public void testTableUsingSequenceFileWithTextSerde() throws Exception { assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); } - assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE)); + assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getProperty(StorageConstants.SEQUENCEFILE_SERDE)); store.dropTable(DB_NAME, REGION); } @@ -683,7 +683,7 @@ public void testDataTypeCompatibility() throws Exception { } assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER), - table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); + table1.getMeta().getProperty(StorageConstants.TEXT_DELIMITER)); store.dropTable(DB_NAME, tableName); } } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java index ea89790807..a041adcc2b 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -208,7 +208,7 @@ public static TableDesc buildTableDesc(String databaseName, String tableName, St new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); desc.setStats(buildTableStats()); - desc.getMeta().setOptions(buildOptions()); + desc.getMeta().setPropertySet(buildOptions()); return desc; } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index ffd53614e1..2a23e94598 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -986,12 +986,12 @@ public void testAlterTableName () throws Exception { KeyValueSet options = new KeyValueSet(); options.set("timezone", "GMT+9"); // Seoul, Korea setPropertyDesc.setMeta(new TableMeta("TEXT", options)); - String prevTimeZone = setPropertyDesc.getMeta().getOption("timezone"); + String prevTimeZone = setPropertyDesc.getMeta().getProperty("timezone"); String newTimeZone = "GMT-7"; // Silicon Valley, California catalog.alterTable(createMockAlterTableSetProperty(newTimeZone)); setPropertyDesc = catalog.getTableDesc("default","mynewcooltable"); - assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getOption("timezone")); - assertEquals(newTimeZone, setPropertyDesc.getMeta().getOption("timezone")); + assertNotEquals(prevTimeZone, setPropertyDesc.getMeta().getProperty("timezone")); + assertEquals(newTimeZone, setPropertyDesc.getMeta().getProperty("timezone")); } private AlterTableDesc createMockAlterTableName(){ diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 99a7b43536..314dd8cc0b 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -423,11 +423,11 @@ public void testTableProperty() throws UndefinedTableException { String tableName = "testTable1"; TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); - assertEquals("ThisIsTest", tableDesc.getMeta().getOption("testString")); - assertEquals("true", tableDesc.getMeta().getOption("testBool")); - assertEquals("0.2", tableDesc.getMeta().getOption("testFloat")); - assertEquals("60", tableDesc.getMeta().getOption("testInt")); - assertEquals("800", tableDesc.getMeta().getOption("testLong")); + assertEquals("ThisIsTest", tableDesc.getMeta().getProperty("testString")); + assertEquals("true", tableDesc.getMeta().getProperty("testBool")); + assertEquals("0.2", tableDesc.getMeta().getProperty("testFloat")); + assertEquals("60", tableDesc.getMeta().getProperty("testInt")); + assertEquals("800", tableDesc.getMeta().getProperty("testLong")); } @Test diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 0639a76fb2..0b649fc08d 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -811,7 +811,7 @@ public void assertTablePropertyEquals(String tableName, String key, String expec throws UndefinedTableException { TableDesc tableDesc = getTableDesc(tableName); - assertEquals(expectedValue, tableDesc.getMeta().getOption(key)); + assertEquals(expectedValue, tableDesc.getMeta().getProperty(key)); } public String resultSetToString(ResultSet resultSet) throws SQLException { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java index bb8cf1bb1d..ec92884754 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java @@ -42,8 +42,8 @@ public class TestDDLBuilder { schema1.addColumn("addr", TajoDataTypes.Type.TEXT); meta1 = CatalogUtil.newTableMeta("TEXT"); - meta1.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); - meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName()); + meta1.putProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + meta1.putProperty(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName()); Schema expressionSchema = new Schema(); expressionSchema.addColumn("key", TajoDataTypes.Type.INT4); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index 2752ed4f3f..75a0ad0fe3 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -708,7 +708,7 @@ public void testNullCharSessionInCTAS() throws Exception { public void assertNullCharSessionVar(TableDesc resultDesc) throws Exception { TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - assertEquals(resultDesc.getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T"); + assertEquals(resultDesc.getMeta().getProperty(StorageConstants.TEXT_NULL), "\\\\T"); Path path = new Path(resultDesc.getUri()); FileSystem fs = path.getFileSystem(tajoConf); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index 667d193bac..8e0700b2c8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -251,7 +251,7 @@ public final void testCtasWithOptions() throws Exception { assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); - KeyValueSet options = desc.getMeta().getOptions(); + KeyValueSet options = desc.getMeta().getPropertySet(); assertNotNull(options); assertEquals(StringEscapeUtils.escapeJava("\u0001"), options.get(StorageConstants.TEXT_DELIMITER)); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 3450b03719..b7196bdfc9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -437,8 +437,8 @@ private boolean isClonedTable(String orignalTable, String newTable) throws Excep return false; } - KeyValueSet origOptions = origMeta.getOptions(); - KeyValueSet newOptions = newMeta.getOptions(); + KeyValueSet origOptions = origMeta.getPropertySet(); + KeyValueSet newOptions = newMeta.getPropertySet(); if(origOptions.equals(newOptions) == false) { fail("Meta options of input tables not equal"); return false; diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java index d807d2f03c..8472020ca0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java +++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/TPCH.java @@ -203,7 +203,7 @@ public void loadTables() throws TajoException { public void loadTable(String tableName) throws TajoException { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); - meta.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + meta.putProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); PartitionMethodDesc partitionMethodDesc = null; if (tableName.equals(CUSTOMER_PARTS)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 8d63410308..baa0bf6a4b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -211,11 +211,11 @@ public PhysicalExec visit(PhysicalExec exec, Stack stack, Class schema.size()) { throw new InvalidTablePropertyException(HBaseStorageConstants.META_COLUMNS_KEY, "mapping column pairs must be more than number of columns in the schema"); } - ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getOptions()); + ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getPropertySet()); int numRowKeys = 0; boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); for (int i = 0; i < isRowKeyMappings.length; i++) { @@ -223,15 +223,15 @@ private void createTable(URI uri, TableMeta tableMeta, Schema schema, private byte[][] getSplitKeys(TajoConf conf, String hbaseTableName, Schema schema, TableMeta meta) throws MissingTablePropertyException, InvalidTablePropertyException, IOException { - String splitRowKeys = meta.getOption(HBaseStorageConstants.META_SPLIT_ROW_KEYS_KEY, ""); - String splitRowKeysFile = meta.getOption(HBaseStorageConstants.META_SPLIT_ROW_KEYS_FILE_KEY, ""); + String splitRowKeys = meta.getProperty(HBaseStorageConstants.META_SPLIT_ROW_KEYS_KEY, ""); + String splitRowKeysFile = meta.getProperty(HBaseStorageConstants.META_SPLIT_ROW_KEYS_FILE_KEY, ""); if ((splitRowKeys == null || splitRowKeys.isEmpty()) && (splitRowKeysFile == null || splitRowKeysFile.isEmpty())) { return null; } - ColumnMapping columnMapping = new ColumnMapping(schema, meta.getOptions()); + ColumnMapping columnMapping = new ColumnMapping(schema, meta.getPropertySet()); boolean[] isBinaryColumns = columnMapping.getIsBinaryColumns(); boolean[] isRowKeys = columnMapping.getIsRowKeyMappings(); @@ -343,13 +343,13 @@ static String extractQuorum(URI uri) { public static HTableDescriptor parseHTableDescriptor(TableMeta tableMeta, Schema schema) throws MissingTablePropertyException, InvalidTablePropertyException { - String hbaseTableName = tableMeta.getOption(HBaseStorageConstants.META_TABLE_KEY, ""); + String hbaseTableName = tableMeta.getProperty(HBaseStorageConstants.META_TABLE_KEY, ""); if (hbaseTableName == null || hbaseTableName.trim().isEmpty()) { throw new MissingTablePropertyException(HBaseStorageConstants.META_TABLE_KEY, hbaseTableName); } TableName hTableName = TableName.valueOf(hbaseTableName); - ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getOptions()); + ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getPropertySet()); HTableDescriptor hTableDescriptor = new HTableDescriptor(hTableName); @@ -397,7 +397,7 @@ public URI getTableUri(String databaseName, String tableName) { private Column[] getIndexableColumns(TableDesc tableDesc) throws MissingTablePropertyException, InvalidTablePropertyException { - ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getOptions()); + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getPropertySet()); boolean[] isRowKeyMappings = columnMapping.getIsRowKeyMappings(); int[] rowKeyIndexes = columnMapping.getRowKeyFieldIndexes(); @@ -419,14 +419,14 @@ public List getSplits(String inputSourceId, @Nullable EvalNode filterCondition) throws IOException, TajoException { - ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getOptions()); + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getPropertySet()); List indexPredications = getIndexPredications(columnMapping, tableDesc, filterCondition); HTable htable = null; HBaseAdmin hAdmin = null; try { - htable = new HTable(hbaseConf, tableDesc.getMeta().getOption(HBaseStorageConstants.META_TABLE_KEY)); + htable = new HTable(hbaseConf, tableDesc.getMeta().getProperty(HBaseStorageConstants.META_TABLE_KEY)); org.apache.hadoop.hbase.util.Pair keys = htable.getStartEndKeys(); if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { @@ -915,7 +915,7 @@ public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId committer.commitJob(jobContext); // insert into table - String tableName = tableDesc.getMeta().getOption(HBaseStorageConstants.META_TABLE_KEY); + String tableName = tableDesc.getMeta().getProperty(HBaseStorageConstants.META_TABLE_KEY); HTable htable = new HTable(hbaseConf, tableName); try { @@ -944,7 +944,7 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc sortKeyIndexes[i] = inputSchema.getColumnId(sortSpecs[i].getSortKey().getQualifiedName()); } - ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getOptions()); + ColumnMapping columnMapping = new ColumnMapping(tableDesc.getSchema(), tableDesc.getMeta().getPropertySet()); HTable htable = new HTable(hbaseConf, columnMapping.getHbaseTableName()); try { @@ -1026,7 +1026,7 @@ public StorageProperty getProperty() { @Override public FormatProperty getFormatProperty(TableMeta meta) { - KeyValueSet tableProperty = meta.getOptions(); + KeyValueSet tableProperty = meta.getPropertySet(); if (tableProperty.isTrue(HBaseStorageConstants.INSERT_PUT_MODE) || tableProperty.isTrue(StorageConstants.INSERT_DIRECTLY)) { return PUT_MODE_PROPERTIES; @@ -1070,7 +1070,7 @@ public void rollbackTable(LogicalNode node) throws IOException, TajoException { @Override public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { - if (meta.getOptions().isTrue(HBaseStorageConstants.INSERT_PUT_MODE)) { + if (meta.getPropertySet().isTrue(HBaseStorageConstants.INSERT_PUT_MODE)) { throw new IOException("Staging phase is not supported in this storage."); } else { return TablespaceManager.getDefault().getStagingUri(context, queryId, meta); @@ -1079,7 +1079,7 @@ public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta public URI prepareStagingSpace(TajoConf conf, String queryId, OverridableConf context, TableMeta meta) throws IOException { - if (!meta.getOptions().isTrue(HBaseStorageConstants.INSERT_PUT_MODE)) { + if (!meta.getPropertySet().isTrue(HBaseStorageConstants.INSERT_PUT_MODE)) { return TablespaceManager.getDefault().prepareStagingSpace(conf, queryId, context, meta); } else { throw new IOException("Staging phase is not supported in this storage."); diff --git a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java index 5b1e2bd017..bfdc8ca5a3 100644 --- a/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java +++ b/tajo-storage/tajo-storage-hbase/src/test/java/org/apache/tajo/storage/hbase/TestColumnMapping.java @@ -43,7 +43,7 @@ public void testColumnKeyValueMapping() throws Exception { TableMeta tableMeta = new TableMeta("HBASE", keyValueSet); - ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getOptions()); + ColumnMapping columnMapping = new ColumnMapping(schema, tableMeta.getPropertySet()); List cfNames = columnMapping.getColumnFamilyNames(); assertEquals(2, cfNames.size()); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java index 87a112e7ae..a7c6b7146c 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java @@ -369,7 +369,7 @@ public void init() throws IOException { if (tableStatsEnabled) { this.stats = new TableStatistics(this.schema, columnStatsEnabled); this.shuffleType = PlannerUtil.getShuffleType( - meta.getOption(StorageConstants.SHUFFLE_TYPE, + meta.getProperty(StorageConstants.SHUFFLE_TYPE, PlannerUtil.getShuffleType(ShuffleType.NONE_SHUFFLE))); } } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java index 0d14c3d722..faeb4a04ab 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java @@ -35,17 +35,17 @@ public class AvroUtil { public static Schema getAvroSchema(TableMeta meta, Configuration conf) throws IOException { - boolean isSchemaLiteral = meta.containsOption(StorageConstants.AVRO_SCHEMA_LITERAL); - boolean isSchemaUrl = meta.containsOption(StorageConstants.AVRO_SCHEMA_URL); + boolean isSchemaLiteral = meta.containsProperty(StorageConstants.AVRO_SCHEMA_LITERAL); + boolean isSchemaUrl = meta.containsProperty(StorageConstants.AVRO_SCHEMA_URL); if (!isSchemaLiteral && !isSchemaUrl) { throw new RuntimeException("No Avro schema for table."); } if (isSchemaLiteral) { - String schema = meta.getOption(StorageConstants.AVRO_SCHEMA_LITERAL); + String schema = meta.getProperty(StorageConstants.AVRO_SCHEMA_LITERAL); return new Schema.Parser().parse(schema); } - String schemaURL = meta.getOption(StorageConstants.AVRO_SCHEMA_URL); + String schemaURL = meta.getProperty(StorageConstants.AVRO_SCHEMA_URL); if (schemaURL.toLowerCase().startsWith("http")) { return getAvroSchemaFromHttp(schemaURL); } else { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java index f3fdacb59c..335f12b596 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java @@ -60,8 +60,8 @@ public JsonLineDeserializer(Schema schema, TableMeta meta, Column [] projected) projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); - hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); - timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); + hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java index b42e14ad1d..1885c80ca3 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java @@ -55,8 +55,8 @@ public JsonLineSerializer(Schema schema, TableMeta meta) { projectedPaths = SchemaUtil.convertColumnsToPaths(schema.getAllColumns(), true); types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); - hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); - timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); + hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java index 7d00206481..7999d02487 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCAppender.java @@ -46,7 +46,7 @@ public ORCAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schem TableMeta meta, Path workDir) { super(conf, taskAttemptId, schema, meta, workDir); - timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); } @@ -54,11 +54,11 @@ public ORCAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schem public void init() throws IOException { writer = OrcFile.createWriter(workDir.getFileSystem(conf), path, conf, ObjectInspectorFactory.buildStructObjectInspector(schema), - Long.parseLong(meta.getOption(StorageConstants.ORC_STRIPE_SIZE, + Long.parseLong(meta.getProperty(StorageConstants.ORC_STRIPE_SIZE, StorageConstants.DEFAULT_ORC_STRIPE_SIZE)), getCompressionKind(), - Integer.parseInt(meta.getOption(StorageConstants.ORC_BUFFER_SIZE, + Integer.parseInt(meta.getProperty(StorageConstants.ORC_BUFFER_SIZE, StorageConstants.DEFAULT_ORC_BUFFER_SIZE)), - Integer.parseInt(meta.getOption(StorageConstants.ORC_ROW_INDEX_STRIDE, + Integer.parseInt(meta.getProperty(StorageConstants.ORC_ROW_INDEX_STRIDE, StorageConstants.DEFAULT_ORC_ROW_INDEX_STRIDE)), timezone); @@ -111,7 +111,7 @@ public long getEstimatedOutputSize() throws IOException { } private CompressionKind getCompressionKind() { - String kindstr = meta.getOption(StorageConstants.ORC_COMPRESSION, StorageConstants.DEFAULT_ORC_COMPRESSION_KIND); + String kindstr = meta.getProperty(StorageConstants.ORC_COMPRESSION, StorageConstants.DEFAULT_ORC_COMPRESSION_KIND); if (kindstr.equalsIgnoreCase(StorageConstants.ORC_COMPRESSION_KIND_ZIP)) { return CompressionKind.ZLIB; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java index b8b75eda8e..32a2aaa5c9 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java @@ -131,7 +131,7 @@ public void init() throws IOException { this.fragment.getPath().toString(), fis, fs.getFileStatus(path).getLen(), - Integer.parseInt(meta.getOption(StorageConstants.ORC_MAX_MERGE_DISTANCE, + Integer.parseInt(meta.getProperty(StorageConstants.ORC_MAX_MERGE_DISTANCE, StorageConstants.DEFAULT_ORC_MAX_MERGE_DISTANCE))); targetColInfo = new ColumnInfo[targets.length]; @@ -155,7 +155,7 @@ public void init() throws IOException { orcReader = new OrcReader(orcDataSource, new OrcMetadataReader()); - TimeZone timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, + TimeZone timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); // TODO: make OrcPredicate useful diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java index 07c9d0ead9..e7b523a912 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java @@ -58,15 +58,15 @@ public ParquetAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema s Path workDir) throws IOException { super(conf, taskAttemptId, schema, meta, workDir); this.blockSize = Integer.parseInt( - meta.getOption(ParquetOutputFormat.BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE)); + meta.getProperty(ParquetOutputFormat.BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE)); this.pageSize = Integer.parseInt( - meta.getOption(ParquetOutputFormat.PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE)); + meta.getProperty(ParquetOutputFormat.PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE)); this.compressionCodecName = CompressionCodecName.fromConf( - meta.getOption(ParquetOutputFormat.COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME)); + meta.getProperty(ParquetOutputFormat.COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME)); this.enableDictionary = Boolean.parseBoolean( - meta.getOption(ParquetOutputFormat.ENABLE_DICTIONARY, StorageConstants.PARQUET_DEFAULT_IS_DICTIONARY_ENABLED)); + meta.getProperty(ParquetOutputFormat.ENABLE_DICTIONARY, StorageConstants.PARQUET_DEFAULT_IS_DICTIONARY_ENABLED)); this.validating = Boolean.parseBoolean( - meta.getOption(ParquetOutputFormat.VALIDATION, StorageConstants.PARQUET_DEFAULT_IS_VALIDATION_ENABLED)); + meta.getProperty(ParquetOutputFormat.VALIDATION, StorageConstants.PARQUET_DEFAULT_IS_VALIDATION_ENABLED)); } /** diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java index 9cbb7a029a..8f4a6ae76a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rawfile/DirectRawFileWriter.java @@ -111,7 +111,7 @@ public void init() throws IOException { if (tableStatsEnabled) { this.stats = new TableStatistics(this.schema, columnStatsEnabled); if (ShuffleType.RANGE_SHUFFLE == PlannerUtil.getShuffleType( - meta.getOption(StorageConstants.SHUFFLE_TYPE, + meta.getProperty(StorageConstants.SHUFFLE_TYPE, PlannerUtil.getShuffleType(ShuffleType.NONE_SHUFFLE)))) { this.analyzeField = true; } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index 20519b7c50..26bbcaf84b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -726,8 +726,8 @@ public RCFileAppender(Configuration conf, final TaskAttemptId taskAttemptId, public void init() throws IOException { fs = path.getFileSystem(conf); - if (this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { - String codecClassname = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); + if (this.meta.containsProperty(StorageConstants.COMPRESSION_CODEC)) { + String codecClassname = this.meta.getProperty(StorageConstants.COMPRESSION_CODEC); try { Class codecClass = conf.getClassByName( codecClassname).asSubclass(CompressionCodec.class); @@ -738,7 +738,7 @@ public void init() throws IOException { } } - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.RCFILE_NULL, + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.RCFILE_NULL, NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); @@ -752,7 +752,7 @@ public void init() throws IOException { metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text("" + columnNumber)); - String serdeClass = this.meta.getOption(StorageConstants.RCFILE_SERDE, + String serdeClass = this.meta.getProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); try { serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); @@ -1177,7 +1177,7 @@ public void init() throws IOException { rowId = new LongWritable(); readBytes = 0; - String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.RCFILE_NULL, + String nullCharacters = StringEscapeUtils.unescapeJava(meta.getProperty(StorageConstants.RCFILE_NULL, NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); @@ -1357,7 +1357,7 @@ private void initHeader() throws IOException { if(text != null && !text.toString().isEmpty()){ serdeClass = text.toString(); } else{ - serdeClass = this.meta.getOption(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); + serdeClass = this.meta.getProperty(StorageConstants.RCFILE_SERDE, BinarySerializerDeserializer.class.getName()); } serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); serde.init(schema); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java index 8e0a88cb79..2a3e3028a5 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java @@ -83,10 +83,10 @@ public void init() throws IOException { this.fs = path.getFileSystem(conf); - this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_DELIMITER, + this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0); this.columnNum = schema.size(); - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL, + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.SEQUENCEFILE_NULL, NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); @@ -94,8 +94,8 @@ public void init() throws IOException { nullChars = nullCharacters.getBytes(); } - if(this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { - String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); + if(this.meta.containsProperty(StorageConstants.COMPRESSION_CODEC)) { + String codecName = this.meta.getProperty(StorageConstants.COMPRESSION_CODEC); codecFactory = new CompressionCodecFactory(conf); codec = codecFactory.getCodecByClassName(codecName); } else { @@ -105,7 +105,7 @@ public void init() throws IOException { } try { - String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, + String serdeClass = this.meta.getProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); serde.init(schema); @@ -125,7 +125,7 @@ public void init() throws IOException { valueClass = Text.class; } - String type = this.meta.getOption(StorageConstants.COMPRESSION_TYPE, CompressionType.NONE.name()); + String type = this.meta.getProperty(StorageConstants.COMPRESSION_TYPE, CompressionType.NONE.name()); if (type.equals(CompressionType.BLOCK.name())) { writer = SequenceFile.createWriter(fs, conf, path, keyClass, valueClass, CompressionType.BLOCK, codec); } else if (type.equals(CompressionType.RECORD.name())) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java index 5009701dd0..37cffdb8cb 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java @@ -96,7 +96,7 @@ public void init() throws IOException { reader = new SequenceFile.Reader(fs, fragment.getPath(), conf); - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL, + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getProperty(StorageConstants.SEQUENCEFILE_NULL, NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); @@ -104,7 +104,7 @@ public void init() throws IOException { nullChars = nullCharacters.getBytes(); } - String delim = meta.getOption(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + String delim = meta.getProperty(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0); this.start = fragment.getStartKey(); @@ -130,7 +130,7 @@ public void init() throws IOException { prepareProjection(targets); try { - String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); + String serdeClass = this.meta.getProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); serde.init(schema); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java index 8664f5e2c2..8d33f5b106 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java @@ -47,8 +47,8 @@ public CSVLineDeserializer(Schema schema, TableMeta meta, Column [] projected) { targetColumnIndexes = PlannerUtil.getTargetIds(schema, projected); // The quote char must be a single ASCII character. - hasQuoteChar = meta.containsOption(StorageConstants.QUOTE_CHAR); - quoteChar = meta.getOption(StorageConstants.QUOTE_CHAR, "\0").getBytes()[0]; + hasQuoteChar = meta.containsProperty(StorageConstants.QUOTE_CHAR); + quoteChar = meta.getProperty(StorageConstants.QUOTE_CHAR, "\0").getBytes()[0]; } @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java index 4ebdbe8ee1..b08ce15c63 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java @@ -37,7 +37,7 @@ public TextLineSerializer createSerializer(Schema schema, TableMeta meta) { } public static byte[] getFieldDelimiter(TableMeta meta) { - return StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.TEXT_DELIMITER, + return StringEscapeUtils.unescapeJava(meta.getProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER)).getBytes(Bytes.UTF8_CHARSET); } } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java index 53fbd57b1c..46d7f6ac9a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java @@ -78,7 +78,7 @@ public static TextLineSerDe getLineSerde(TableMeta meta) { String serDeClassName; // if there is no given serde class, it will use CSV line serder. - serDeClassName = meta.getOption(StorageConstants.TEXT_SERDE_CLASS, StorageConstants.DEFAULT_TEXT_SERDE_CLASS); + serDeClassName = meta.getProperty(StorageConstants.TEXT_SERDE_CLASS, StorageConstants.DEFAULT_TEXT_SERDE_CLASS); try { Class serdeClass; @@ -147,8 +147,8 @@ public void init() throws IOException { } os.reset(); - if (this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { - String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); + if (this.meta.containsProperty(StorageConstants.COMPRESSION_CODEC)) { + String codecName = this.meta.getProperty(StorageConstants.COMPRESSION_CODEC); codecFactory = new CompressionCodecFactory(conf); codec = codecFactory.getCodecByClassName(codecName); compressor = CodecPool.getCompressor(codec); @@ -294,7 +294,7 @@ public DelimitedTextFileScanner(Configuration conf, final Schema schema, final T endOffset = startOffset + fragment.getLength(); errorTorrenceMaxNum = - Integer.parseInt(meta.getOption(TEXT_ERROR_TOLERANCE_MAXNUM, DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM)); + Integer.parseInt(meta.getProperty(TEXT_ERROR_TOLERANCE_MAXNUM, DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM)); } @@ -329,7 +329,7 @@ public void init() throws IOException { } else { // skip header lines if it is defined // initialization for skipping header(max 20) - int headerLineNum = Math.min(Integer.parseInt(meta.getOption(StorageConstants.TEXT_SKIP_HEADER_LINE, "0")), 20); + int headerLineNum = Math.min(Integer.parseInt(meta.getProperty(StorageConstants.TEXT_SKIP_HEADER_LINE, "0")), 20); if (headerLineNum > 0) { LOG.info(String.format("Skip %d header lines", headerLineNum)); for (int i = 0; i < headerLineNum; i++) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java index d9d2639848..53b6bcda46 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java @@ -53,8 +53,8 @@ public class TextFieldSerializerDeserializer implements FieldSerializerDeseriali private Schema schema; public TextFieldSerializerDeserializer(TableMeta meta) { - hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); - timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); + hasTimezone = meta.containsProperty(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getProperty(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); } private static boolean isNull(ByteBuf val, ByteBuf nullBytes) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java index 3143ec208b..0717faedba 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java @@ -62,7 +62,7 @@ public static ByteBuf getNullChars(TableMeta meta) { public static byte[] getNullCharsAsBytes(TableMeta meta, String key, String defaultVal) { byte [] nullChars; - String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(key, defaultVal)); + String nullCharacters = StringEscapeUtils.unescapeJava(meta.getProperty(key, defaultVal)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index 9c30202c56..7836730ab4 100644 --- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -114,10 +114,10 @@ private void storageCompressionTest(String dataFormat, Class Date: Tue, 24 Nov 2015 02:04:49 -0800 Subject: [PATCH 2/2] Add change log to CHANGES. --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index 95158e189a..8000164a91 100644 --- a/CHANGES +++ b/CHANGES @@ -8,6 +8,8 @@ Release 0.12.0 - unreleased IMPROVEMENT + TAJO-1986: Rename the name 'option' to 'property' in TableMeta. (hyunsik) + TAJO-1984: Removed unused FileTablespace::getSplit methods. (hyunsik) TAJO-1271: Improve memory usage of Hash-shuffle. (jinho)