From 69f6678ad7d1ce9a12a0766abde94bc9def31588 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 15 May 2015 02:48:21 +0900 Subject: [PATCH] TAJO-1604: Merge TAJO-1442 to 0.10.1 --- .travis.yml | 2 +- tajo-catalog/pom.xml | 4 +- tajo-catalog/tajo-catalog-drivers/pom.xml | 34 +- .../tajo-catalog-drivers/tajo-hive/pom.xml | 351 +++++++ .../tajo/catalog/store/HiveCatalogStore.java | 877 ++++++++++++++++++ .../store/HiveCatalogStoreClientPool.java | 170 ++++ .../tajo/catalog/store/HiveCatalogUtil.java | 127 +++ .../catalog/store/TestHiveCatalogStore.java | 402 ++++++++ .../org/apache/tajo/catalog/TestCatalog.java | 4 +- tajo-core/pom.xml | 212 ----- .../org/apache/tajo/QueryTestCaseBase.java | 4 +- .../org/apache/tajo/TajoTestingCluster.java | 8 +- .../apache/tajo/cli/tools/TestTajoDump.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 54 +- .../engine/query/TestAlterTablespace.java | 2 +- .../tajo/engine/query/TestCTASQuery.java | 6 +- .../tajo/engine/query/TestCreateTable.java | 12 +- .../tajo/engine/query/TestInsertQuery.java | 40 +- .../tajo/engine/query/TestNetTypes.java | 14 +- .../tajo/engine/query/TestSelectQuery.java | 4 +- .../tajo/engine/query/TestSortQuery.java | 6 +- .../engine/query/TestTablePartitions.java | 22 +- .../org/apache/tajo/jdbc/TestResultSet.java | 4 +- .../tajo/jdbc/TestTajoDatabaseMetaData.java | 22 +- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 12 +- ..._table_various_types_for_hive_catalog.sql} | 0 tajo-dist/pom.xml | 11 +- tajo-dist/src/main/bin/tajo | 12 - .../src/main/conf/catalog-site.xml.template | 6 +- tajo-dist/src/main/conf/tajo-env.sh | 2 +- ...g_integration.rst => hive_integration.rst} | 24 +- tajo-docs/src/main/sphinx/index.rst | 2 +- .../rewrite/rules/FilterPushDownRule.java | 2 +- tajo-project/pom.xml | 37 + 34 files changed, 2100 insertions(+), 391 deletions(-) create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java create mode 100644 tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java rename tajo-core/src/test/resources/queries/TestCreateTable/{create_table_various_types_for_hcatalog.sql => create_table_various_types_for_hive_catalog.sql} (100%) rename tajo-docs/src/main/sphinx/{hcatalog_integration.rst => hive_integration.rst} (61%) diff --git a/.travis.yml b/.travis.yml index ab16f1dee3..4b3459a399 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,4 +37,4 @@ notifications: install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -Pparallel-test,hcatalog-0.12.0 -DLOG_LEVEL=WARN -Dmaven.fork.count=2 + mvn clean install -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 diff --git a/tajo-catalog/pom.xml b/tajo-catalog/pom.xml index 9cfdd5d3a0..864e1fed47 100644 --- a/tajo-catalog/pom.xml +++ b/tajo-catalog/pom.xml @@ -132,9 +132,9 @@ run cp -r ${basedir}/tajo-catalog-common/target/tajo-catalog-common-${project.version}*.jar . run cp -r ${basedir}/tajo-catalog-client/target/tajo-catalog-client-${project.version}*.jar . run cp -r ${basedir}/tajo-catalog-server/target/tajo-catalog-server-${project.version}*.jar . - if [ -f ${basedir}/tajo-catalog-drivers/tajo-hcatalog/target/tajo-hcatalog-${project.version}.jar ] + if [ -f ${basedir}/tajo-catalog-drivers/tajo-hive/target/tajo-hive-${project.version}.jar ] then - run cp -r ${basedir}/tajo-catalog-drivers/tajo-hcatalog/target/tajo-hcatalog-${project.version}*.jar . + run cp -r ${basedir}/tajo-catalog-drivers/tajo-hive/target/tajo-hive-${project.version}*.jar . fi echo echo "Tajo Catalog dist layout available at: ${project.build.directory}/tajo-catalog-${project.version}" diff --git a/tajo-catalog/tajo-catalog-drivers/pom.xml b/tajo-catalog/tajo-catalog-drivers/pom.xml index 89f3061126..5a0e327cc1 100644 --- a/tajo-catalog/tajo-catalog-drivers/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/pom.xml @@ -40,7 +40,7 @@ apache-rat-plugin - tajo-hcatalog/** + tajo-hive/** @@ -57,35 +57,9 @@ - - - hcatalog-0.12.0 - - false - - - tajo-hcatalog - - - - hcatalog-0.13.0 - - false - - - tajo-hcatalog - - - - hcatalog-0.13.1 - - false - - - tajo-hcatalog - - - + + tajo-hive + diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml new file mode 100644 index 0000000000..16cb17014c --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/pom.xml @@ -0,0 +1,351 @@ + + + + + + tajo-project + org.apache.tajo + 0.11.0-SNAPSHOT + ../../../tajo-project + + 4.0.0 + tajo-hive + jar + Tajo Catalog Drivers Hive + + UTF-8 + UTF-8 + 1.5.0 + 2.1.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + ${project.build.sourceEncoding} + + + + org.apache.rat + apache-rat-plugin + + + verify + + check + + + + + + derby.log + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + prepare-package + + copy-dependencies + + + runtime + ${project.build.directory}/lib + false + false + true + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + org.apache.tajo + tajo-common + + + org.apache.tajo + tajo-catalog-common + + + org.apache.tajo + tajo-catalog-client + + + org.apache.tajo + tajo-catalog-server + + + org.apache.tajo + tajo-rpc-protobuf + + + org.apache.tajo + tajo-storage-common + + + junit + junit + test + + + org.apache.thrift + libfb303 + 0.9.0 + provided + + + org.apache.thrift + libthrift + 0.9.0 + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + org.apache.hive + hive-exec + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-contrib + + + org.apache.hive + hive-hbase-handler + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shims + + + org.apache.hive + hive-testutils + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + com.jolbox + bonecp + + + com.google.protobuf + protobuf-java + + + org.apache.calcite + calcite-core + + + org.apache.calcite + calcite-avatica + + + + + org.apache.hive + hive-metastore + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shimss + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + com.jolbox + bonecp + + + + + org.apache.hive + hive-cli + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-service + + + org.apache.hive + hive-shims + + + com.jolbox + bonecp + + + jline + jline + + + + + com.twitter + parquet-hive-bundle + ${parquet.version} + + + + + + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + + src + + false + + + + + org.apache.maven.plugins + maven-source-plugin + + + + hadoop-java-sources + package + + jar-no-fork + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java new file mode 100644 index 0000000000..2fe5b352ee --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -0,0 +1,877 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.store; + +import com.google.common.collect.Lists; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; +import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.exception.*; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; +import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.exception.NotImplementedException; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.exception.InternalException; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.KeyValueSet; +import org.apache.thrift.TException; + +import java.io.IOException; +import java.util.*; + +import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType; + +public class HiveCatalogStore extends CatalogConstants implements CatalogStore { + protected final Log LOG = LogFactory.getLog(getClass()); + + private static String HIVE_WAREHOUSE_DIR_CONF_KEY = "hive.metastore.warehouse.dir"; + + protected Configuration conf; + private static final int CLIENT_POOL_SIZE = 2; + private final HiveCatalogStoreClientPool clientPool; + private final String defaultTableSpaceUri; + + public HiveCatalogStore(final Configuration conf) throws InternalException { + if (!(conf instanceof TajoConf)) { + throw new CatalogException("Invalid Configuration Type:" + conf.getClass().getSimpleName()); + } + this.conf = conf; + this.defaultTableSpaceUri = TajoConf.getWarehouseDir((TajoConf) conf).toString(); + this.clientPool = new HiveCatalogStoreClientPool(CLIENT_POOL_SIZE, conf); + } + + @Override + public boolean existTable(final String databaseName, final String tableName) throws CatalogException { + boolean exist = false; + org.apache.hadoop.hive.ql.metadata.Table table; + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + // get table + try { + client = clientPool.getClient(); + table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); + if (table != null) { + exist = true; + } + } catch (NoSuchObjectException nsoe) { + exist = false; + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + + return exist; + } + + @Override + public final CatalogProtos.TableDescProto getTable(String databaseName, final String tableName) throws CatalogException { + org.apache.hadoop.hive.ql.metadata.Table table = null; + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + Path path = null; + CatalogProtos.StoreType storeType = null; + org.apache.tajo.catalog.Schema schema = null; + KeyValueSet options = null; + TableStats stats = null; + PartitionMethodDesc partitions = null; + + ////////////////////////////////// + // set tajo table schema. + ////////////////////////////////// + try { + // get hive table schema + try { + client = clientPool.getClient(); + table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); + path = table.getPath(); + } catch (NoSuchObjectException nsoe) { + throw new CatalogException("Table not found. - tableName:" + tableName, nsoe); + } catch (Exception e) { + throw new CatalogException(e); + } + + // convert HiveCatalogStore field schema into tajo field schema. + schema = new org.apache.tajo.catalog.Schema(); + + List fieldSchemaList = table.getCols(); + boolean isPartitionKey = false; + for (FieldSchema eachField : fieldSchemaList) { + isPartitionKey = false; + + if (table.getPartitionKeys() != null) { + for (FieldSchema partitionKey : table.getPartitionKeys()) { + if (partitionKey.getName().equals(eachField.getName())) { + isPartitionKey = true; + } + } + } + + if (!isPartitionKey) { + String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + + CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); + TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType().toString()); + schema.addColumn(fieldName, dataType); + } + } + + // validate field schema. + HiveCatalogUtil.validateSchema(table); + + stats = new TableStats(); + options = new KeyValueSet(); + options.putAll(table.getParameters()); + options.remove("EXTERNAL"); + + Properties properties = table.getMetadata(); + if (properties != null) { + // set field delimiter + String fieldDelimiter = "", nullFormat = ""; + if (properties.getProperty(serdeConstants.FIELD_DELIM) != null) { + fieldDelimiter = properties.getProperty(serdeConstants.FIELD_DELIM); + } else { + // if hive table used default row format delimiter, Properties doesn't have it. + // So, Tajo must set as follows: + fieldDelimiter = "\u0001"; + } + + // set null format + if (properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT) != null) { + nullFormat = properties.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT); + } else { + nullFormat = "\\N"; + } + options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT); + + // set file output format + String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); + storeType = CatalogUtil.getStoreType(HiveCatalogUtil.getStoreType(fileOutputformat)); + + if (storeType.equals(CatalogProtos.StoreType.TEXTFILE)) { + options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); + options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava(nullFormat)); + } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) { + options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); + String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); + if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + } else if (ColumnarSerDe.class.getName().equals(serde)) { + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + } + } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) { + options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); + options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); + String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); + if (LazyBinarySerDe.class.getName().equals(serde)) { + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + } else if (LazySimpleSerDe.class.getName().equals(serde)) { + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + } + } + + // set data size + long totalSize = 0; + if (properties.getProperty("totalSize") != null) { + totalSize = Long.parseLong(properties.getProperty("totalSize")); + } else { + try { + FileSystem fs = path.getFileSystem(conf); + if (fs.exists(path)) { + totalSize = fs.getContentSummary(path).getLength(); + } + } catch (IOException ioe) { + throw new CatalogException("Fail to get path. - path:" + path.toString(), ioe); + } + } + stats.setNumBytes(totalSize); + } + + // set partition keys + List partitionKeys = table.getPartitionKeys(); + + if (null != partitionKeys) { + org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); + StringBuilder sb = new StringBuilder(); + if (partitionKeys.size() > 0) { + for (int i = 0; i < partitionKeys.size(); i++) { + FieldSchema fieldSchema = partitionKeys.get(i); + TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType().toString()); + String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); + expressionSchema.addColumn(new Column(fieldName, dataType)); + if (i > 0) { + sb.append(","); + } + sb.append(fieldSchema.getName()); + } + partitions = new PartitionMethodDesc( + databaseName, + tableName, + PartitionType.COLUMN, + sb.toString(), + expressionSchema); + } + } + } finally { + if(client != null) client.release(); + } + TableMeta meta = new TableMeta(storeType, options); + TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri()); + if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) { + tableDesc.setExternal(true); + } + if (stats != null) { + tableDesc.setStats(stats); + } + if (partitions != null) { + tableDesc.setPartitionMethod(partitions); + } + return tableDesc.getProto(); + } + + + private TajoDataTypes.Type getDataType(final String typeStr) { + try { + return Enum.valueOf(TajoDataTypes.Type.class, typeStr); + } catch (IllegalArgumentException iae) { + LOG.error("Cannot find a matched type against from '" + typeStr + "'"); + return null; + } + } + + @Override + public final List getAllTableNames(String databaseName) throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + client = clientPool.getClient(); + return client.getHiveClient().getAllTables(databaseName); + } catch (TException e) { + throw new CatalogException(e); + } finally { + if(client != null) client.release(); + } + } + + @Override + public void createTablespace(String spaceName, String spaceUri) throws CatalogException { + // SKIP + } + + @Override + public boolean existTablespace(String spaceName) throws CatalogException { + // SKIP + return spaceName.equals(TajoConstants.DEFAULT_TABLESPACE_NAME); + } + + @Override + public void dropTablespace(String spaceName) throws CatalogException { + // SKIP + } + + @Override + public Collection getAllTablespaceNames() throws CatalogException { + return Lists.newArrayList(TajoConstants.DEFAULT_TABLESPACE_NAME); + } + + @Override + public TablespaceProto getTablespace(String spaceName) throws CatalogException { + if (spaceName.equals(TajoConstants.DEFAULT_TABLESPACE_NAME)) { + TablespaceProto.Builder builder = TablespaceProto.newBuilder(); + builder.setSpaceName(TajoConstants.DEFAULT_TABLESPACE_NAME); + builder.setUri(defaultTableSpaceUri); + return builder.build(); + } else { + throw new CatalogException("tablespace concept is not supported in HiveCatalogStore"); + } + } + + @Override + public void updateTableStats(CatalogProtos.UpdateTableStatsProto statsProto) throws + CatalogException { + // TODO - not implemented yet + } + + @Override + public void alterTablespace(CatalogProtos.AlterTablespaceProto alterProto) throws CatalogException { + throw new CatalogException("tablespace concept is not supported in HiveCatalogStore"); + } + + @Override + public void createDatabase(String databaseName, String tablespaceName) throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + Database database = new Database( + databaseName, + "", + defaultTableSpaceUri + "/" + databaseName, + new HashMap()); + client = clientPool.getClient(); + client.getHiveClient().createDatabase(database); + } catch (AlreadyExistsException e) { + throw new AlreadyExistsDatabaseException(databaseName); + } catch (Throwable t) { + throw new CatalogException(t); + } finally { + if (client != null) { + client.release(); + } + } + } + + @Override + public boolean existDatabase(String databaseName) throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + client = clientPool.getClient(); + List databaseNames = client.getHiveClient().getAllDatabases(); + return databaseNames.contains(databaseName); + } catch (Throwable t) { + throw new CatalogException(t); + } finally { + if (client != null) { + client.release(); + } + } + } + + @Override + public void dropDatabase(String databaseName) throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + client = clientPool.getClient(); + client.getHiveClient().dropDatabase(databaseName); + } catch (NoSuchObjectException e) { + throw new NoSuchDatabaseException(databaseName); + } catch (Throwable t) { + throw new CatalogException(databaseName); + } finally { + if (client != null) { + client.release(); + } + } + } + + @Override + public Collection getAllDatabaseNames() throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + client = clientPool.getClient(); + return client.getHiveClient().getAllDatabases(); + } catch (TException e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + } + + @Override + public final void createTable(final CatalogProtos.TableDescProto tableDescProto) throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + TableDesc tableDesc = new TableDesc(tableDescProto); + String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); + String databaseName = splitted[0]; + String tableName = splitted[1]; + + try { + client = clientPool.getClient(); + + org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api.Table(); + table.setDbName(databaseName); + table.setTableName(tableName); + table.setParameters(new HashMap(tableDesc.getMeta().getOptions().getAllKeyValus())); + // TODO: set owner + //table.setOwner(); + + StorageDescriptor sd = new StorageDescriptor(); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setParameters(new HashMap()); + sd.getSerdeInfo().setName(table.getTableName()); + + // if tajo set location method, thrift client make exception as follows: + // Caused by: MetaException(message:java.lang.NullPointerException) + // If you want to modify table path, you have to modify on Hive cli. + if (tableDesc.isExternal()) { + table.setTableType(TableType.EXTERNAL_TABLE.name()); + table.putToParameters("EXTERNAL", "TRUE"); + + Path tablePath = new Path(tableDesc.getPath()); + FileSystem fs = tablePath.getFileSystem(conf); + if (fs.isFile(tablePath)) { + LOG.warn("A table path is a file, but HiveCatalogStore does not allow a file path."); + sd.setLocation(tablePath.getParent().toString()); + } else { + sd.setLocation(tablePath.toString()); + } + } + + // set column information + List columns = tableDesc.getSchema().getColumns(); + ArrayList cols = new ArrayList(columns.size()); + + for (Column eachField : columns) { + cols.add(new FieldSchema(eachField.getSimpleName(), + HiveCatalogUtil.getHiveFieldType(eachField.getDataType()), "")); + } + sd.setCols(cols); + + // set partition keys + if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { + List partitionKeys = new ArrayList(); + for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) { + partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(), + HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), "")); + } + table.setPartitionKeys(partitionKeys); + } + + if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.RCFILE)) { + String serde = tableDesc.getMeta().getOption(StorageConstants.RCFILE_SERDE); + sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); + sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); + if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { + sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); + } else { + sd.getSerdeInfo().setSerializationLib( + org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName()); + } + + if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) { + table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, + StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); + } + } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV) + || tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) { + sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); + sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName()); + + String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.TEXT_DELIMITER, + StorageConstants.DEFAULT_FIELD_DELIMITER); + + // User can use an unicode for filed delimiter such as \u0001, \001. + // In this case, java console will convert this value into "\\u001". + // And hive will un-espace this value again. + // As a result, user can use right field delimiter. + // So, we have to un-escape this value. + sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT, + StringEscapeUtils.unescapeJava(fieldDelimiter)); + sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, + StringEscapeUtils.unescapeJava(fieldDelimiter)); + table.getParameters().remove(StorageConstants.TEXT_DELIMITER); + + if (tableDesc.getMeta().containsOption(StorageConstants.TEXT_NULL)) { + table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, + StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.TEXT_NULL))); + table.getParameters().remove(StorageConstants.TEXT_NULL); + } + } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) { + String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE); + sd.setInputFormat(org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName()); + sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat.class.getName()); + + if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { + sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + + String fieldDelimiter = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_DELIMITER, + StorageConstants.DEFAULT_FIELD_DELIMITER); + + // User can use an unicode for filed delimiter such as \u0001, \001. + // In this case, java console will convert this value into "\\u001". + // And hive will un-espace this value again. + // As a result, user can use right field delimiter. + // So, we have to un-escape this value. + sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT, + StringEscapeUtils.unescapeJava(fieldDelimiter)); + sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, + StringEscapeUtils.unescapeJava(fieldDelimiter)); + table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER); + } else { + sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName()); + } + + if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) { + table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, + StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL))); + table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); + } + } else { + if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.PARQUET)) { + sd.setInputFormat(parquet.hive.DeprecatedParquetInputFormat.class.getName()); + sd.setOutputFormat(parquet.hive.DeprecatedParquetOutputFormat.class.getName()); + sd.getSerdeInfo().setSerializationLib(parquet.hive.serde.ParquetHiveSerDe.class.getName()); + } else { + throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name())); + } + } + + sd.setSortCols(new ArrayList()); + + table.setSd(sd); + client.getHiveClient().createTable(table); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if(client != null) client.release(); + } + } + + @Override + public final void dropTable(String databaseName, final String tableName) throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + client = clientPool.getClient(); + client.getHiveClient().dropTable(databaseName, tableName, false, false); + } catch (NoSuchObjectException nsoe) { + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + } + + + @Override + public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescProto) throws CatalogException { + final String[] split = CatalogUtil.splitFQTableName(alterTableDescProto.getTableName()); + + if (split.length == 1) { + throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" + + alterTableDescProto.getTableName() + "\"."); + } + + final String databaseName = split[0]; + final String tableName = split[1]; + + + switch (alterTableDescProto.getAlterTableType()) { + case RENAME_TABLE: + if (existTable(databaseName,alterTableDescProto.getNewTableName().toLowerCase())) { + throw new AlreadyExistsTableException(alterTableDescProto.getNewTableName()); + } + renameTable(databaseName, tableName, alterTableDescProto.getNewTableName().toLowerCase()); + break; + case RENAME_COLUMN: + if (existColumn(databaseName,tableName, alterTableDescProto.getAlterColumnName().getNewColumnName())) { + throw new ColumnNameAlreadyExistException(alterTableDescProto.getAlterColumnName().getNewColumnName()); + } + renameColumn(databaseName, tableName, alterTableDescProto.getAlterColumnName()); + break; + case ADD_COLUMN: + if (existColumn(databaseName,tableName, alterTableDescProto.getAddColumn().getName())) { + throw new ColumnNameAlreadyExistException(alterTableDescProto.getAddColumn().getName()); + } + addNewColumn(databaseName, tableName, alterTableDescProto.getAddColumn()); + break; + default: + //TODO + } + } + + + private void renameTable(String databaseName, String tableName, String newTableName) { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + try { + client = clientPool.getClient(); + Table newTable = client.getHiveClient().getTable(databaseName, tableName); + newTable.setTableName(newTableName); + client.getHiveClient().alter_table(databaseName, tableName, newTable); + + } catch (NoSuchObjectException nsoe) { + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + } + + private void renameColumn(String databaseName, String tableName, CatalogProtos.AlterColumnProto alterColumnProto) { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + try { + + client = clientPool.getClient(); + Table table = client.getHiveClient().getTable(databaseName, tableName); + List columns = table.getSd().getCols(); + + for (final FieldSchema currentColumn : columns) { + if (currentColumn.getName().equalsIgnoreCase(alterColumnProto.getOldColumnName())) { + currentColumn.setName(alterColumnProto.getNewColumnName()); + } + } + client.getHiveClient().alter_table(databaseName, tableName, table); + + } catch (NoSuchObjectException nsoe) { + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + } + + + private void addNewColumn(String databaseName, String tableName, CatalogProtos.ColumnProto columnProto) { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + try { + + client = clientPool.getClient(); + Table table = client.getHiveClient().getTable(databaseName, tableName); + List columns = table.getSd().getCols(); + columns.add(new FieldSchema(columnProto.getName(), + HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), "")); + client.getHiveClient().alter_table(databaseName, tableName, table); + + + } catch (NoSuchObjectException nsoe) { + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + } + + @Override + public void addPartitionMethod(CatalogProtos.PartitionMethodProto partitionMethodProto) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName, String tableName) + throws CatalogException { + return null; // TODO - not implemented yet + } + + @Override + public boolean existPartitionMethod(String databaseName, String tableName) throws CatalogException { + return false; // TODO - not implemented yet + } + + @Override + public void dropPartitionMethod(String databaseName, String tableName) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public void addPartitions(CatalogProtos.PartitionsProto partitionsProto) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public void addPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto partitionDescProto) throws CatalogException { + + } + + @Override + public CatalogProtos.PartitionsProto getPartitions(String tableName) throws CatalogException { + return null; // TODO - not implemented yet + } + + @Override + public CatalogProtos.PartitionDescProto getPartition(String partitionName) throws CatalogException { + return null; // TODO - not implemented yet + } + + @Override + public void delPartition(String partitionName) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public void dropPartitions(String tableName) throws CatalogException { + + } + + + @Override + public final void addFunction(final FunctionDesc func) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public final void deleteFunction(final FunctionDesc func) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public final void existFunction(final FunctionDesc func) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public final List getAllFunctionNames() throws CatalogException { + // TODO - not implemented yet + return null; + } + + @Override + public void dropIndex(String databaseName, String indexName) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public boolean existIndexByName(String databaseName, String indexName) throws CatalogException { + // TODO - not implemented yet + return false; + } + + @Override + public CatalogProtos.IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException { + // TODO - not implemented yet + return null; + } + + @Override + public void createIndex(CatalogProtos.IndexDescProto proto) throws CatalogException { + // TODO - not implemented yet + } + + @Override + public CatalogProtos.IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException { + // TODO - not implemented yet + return null; + } + + @Override + public CatalogProtos.IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) + throws CatalogException { + // TODO - not implemented yet + return null; + } + + @Override + public boolean existIndexByColumn(String databaseName, String tableName, String columnName) throws CatalogException { + // TODO - not implemented yet + return false; + } + + @Override + public final void close() { + clientPool.close(); + } + + private boolean existColumn(final String databaseName ,final String tableName , final String columnName) throws CatalogException { + boolean exist = false; + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + + client = clientPool.getClient(); + Table table = client.getHiveClient().getTable(databaseName, tableName); + List columns = table.getSd().getCols(); + + for (final FieldSchema currentColumn : columns) { + if (currentColumn.getName().equalsIgnoreCase(columnName)) { + exist = true; + } + } + client.getHiveClient().alter_table(databaseName, tableName, table); + + } catch (NoSuchObjectException nsoe) { + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } + + return exist; + } + + @Override + public List getAllColumns() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getAllDatabases() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getAllIndexes() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getAllPartitions() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getAllTableOptions() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getAllTableStats() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getAllTables() throws CatalogException { + throw new UnsupportedOperationException(); + } + + @Override + public List getTablespaces() throws CatalogException { + throw new UnsupportedOperationException(); + } +} diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java new file mode 100644 index 0000000000..9053c5661b --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java @@ -0,0 +1,170 @@ +// Copyright 2012 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package org.apache.tajo.catalog.store; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.*; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.log4j.Logger; + +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Manages a pool of HiveMetaStoreClient connections. If the connection pool is empty + * a new client is created and added to the pool. There is no size limit. + */ +public class HiveCatalogStoreClientPool { + private static final Logger LOG = Logger.getLogger(HiveCatalogStoreClientPool.class); + private final ConcurrentLinkedQueue clientPool = + new ConcurrentLinkedQueue(); + private AtomicBoolean poolClosed = new AtomicBoolean(false); + private HiveConf hiveConf; + + /** + * A wrapper around the HiveMetaStoreClient that manages interactions with the + * connection pool. + */ + public class HiveCatalogStoreClient { + private final IMetaStoreClient hiveClient; + public AtomicBoolean isInUse = new AtomicBoolean(false); + + private HiveCatalogStoreClient(HiveConf hiveConf) { + try { + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook(Table table) throws MetaException { + /* metadata hook implementation, or null if this + * storage handler does not need any metadata notifications + */ + return null; + } + }; + + this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName()); + clientPool.add(this); + LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")"); + } catch (Exception e) { + // Turn in to an unchecked exception + throw new IllegalStateException(e); + } + } + + /** + * Returns the internal HiveMetaStoreClient object. + */ + public IMetaStoreClient getHiveClient() { + return hiveClient; + } + + /** + * Returns this client back to the connection pool. If the connection pool has been + * closed, just close the Hive client connection. + */ + public synchronized void release() { + if(!this.isInUse.getAndSet(false)){ + return; + } + // Ensure the connection isn't returned to the pool if the pool has been closed. + // This lock is needed to ensure proper behavior when a thread reads poolClosed + // is false, but a call to pool.close() comes in immediately afterward. + if (poolClosed.get()) { + this.getHiveClient().close(); + } else { + clientPool.add(this); + } + } + + // Marks this client as in use + private void markInUse() { + isInUse.set(true); + } + } + + public HiveCatalogStoreClientPool(int initialSize) { + this(initialSize, new HiveConf(HiveCatalogStoreClientPool.class)); + } + + public HiveCatalogStoreClientPool(int initialSize, HiveConf hiveConf) { + this.hiveConf = hiveConf; + addClients(initialSize); + } + + public HiveCatalogStoreClientPool(int initialSize, Configuration conf) { + this.hiveConf = new HiveConf(); + setParameters(conf); + addClients(initialSize); + } + + public void setParameters(Configuration conf) { + for( Iterator> iter = conf.iterator(); iter.hasNext();) { + Map.Entry entry = iter.next(); + this.hiveConf.set(entry.getKey(), entry.getValue()); + } + } + + /** + * Add numClients to the client pool. + */ + public void addClients(int numClients) { + for (int i = 0; i < numClients; ++i) { + clientPool.add(new HiveCatalogStoreClient(hiveConf)); + } + } + + /** + * Gets a client from the pool. If the pool is empty a new client is created. + */ + public synchronized HiveCatalogStoreClient getClient() { + // The MetaStoreClient c'tor relies on knowing the Hadoop version by asking + // org.apache.hadoop.util.VersionInfo. The VersionInfo class relies on opening + // the 'common-version-info.properties' file as a resource from hadoop-common*.jar + // using the Thread's context classloader. If necessary, set the Thread's context + // classloader, otherwise VersionInfo will fail in it's c'tor. + if (Thread.currentThread().getContextClassLoader() == null) { + Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader()); + } + + HiveCatalogStoreClient client = clientPool.poll(); + // The pool was empty so create a new client and return that. + if (client == null) { + client = new HiveCatalogStoreClient(hiveConf); + } + client.markInUse(); + + return client; + } + + /** + * Removes all items from the connection pool and closes all Hive Meta Store client + * connections. Can be called multiple times. + */ + public void close() { + // Ensure no more items get added to the pool once close is called. + if (poolClosed.getAndSet(true)) { + return; + } + + HiveCatalogStoreClient client = null; + while ((client = clientPool.poll()) != null) { + client.getHiveClient().close(); + } + } +} diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java new file mode 100644 index 0000000000..59910b84f9 --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.catalog.store; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; +import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.tajo.catalog.exception.CatalogException; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.thrift.TException; +import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; + +public class HiveCatalogUtil { + public static void validateSchema(Table tblSchema) throws CatalogException { + for (FieldSchema fieldSchema : tblSchema.getCols()) { + String fieldType = fieldSchema.getType(); + if (fieldType.equalsIgnoreCase("ARRAY") || fieldType.equalsIgnoreCase("STRUCT") + || fieldType.equalsIgnoreCase("MAP")) { + throw new CatalogException("Unsupported field type :" + fieldType.toUpperCase()); + } + } + } + + public static TajoDataTypes.Type getTajoFieldType(String fieldType) { + Preconditions.checkNotNull(fieldType); + + if(fieldType.equalsIgnoreCase(serdeConstants.INT_TYPE_NAME)) { + return TajoDataTypes.Type.INT4; + } else if(fieldType.equalsIgnoreCase(serdeConstants.TINYINT_TYPE_NAME)) { + return TajoDataTypes.Type.INT1; + } else if(fieldType.equalsIgnoreCase(serdeConstants.SMALLINT_TYPE_NAME)) { + return TajoDataTypes.Type.INT2; + } else if(fieldType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)) { + return TajoDataTypes.Type.INT8; + } else if(fieldType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) { + return TajoDataTypes.Type.BOOLEAN; + } else if(fieldType.equalsIgnoreCase(serdeConstants.FLOAT_TYPE_NAME)) { + return TajoDataTypes.Type.FLOAT4; + } else if(fieldType.equalsIgnoreCase(serdeConstants.DOUBLE_TYPE_NAME)) { + return TajoDataTypes.Type.FLOAT8; + } else if(fieldType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) { + return TajoDataTypes.Type.TEXT; + } else if(fieldType.equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME)) { + return TajoDataTypes.Type.BLOB; + } else if(fieldType.equalsIgnoreCase(serdeConstants.TIMESTAMP_TYPE_NAME)) { + return TajoDataTypes.Type.TIMESTAMP; + } else if(fieldType.equalsIgnoreCase(serdeConstants.DATE_TYPE_NAME)) { + return TajoDataTypes.Type.DATE; + } else { + throw new CatalogException("Cannot find a matched type against from '" + fieldType + "'"); + } + } + + public static String getHiveFieldType(TajoDataTypes.DataType dataType) { + Preconditions.checkNotNull(dataType); + + switch (dataType.getType()) { + case CHAR: return serdeConstants.CHAR_TYPE_NAME; + case BOOLEAN: return serdeConstants.BOOLEAN_TYPE_NAME; + case INT1: return serdeConstants.TINYINT_TYPE_NAME; + case INT2: return serdeConstants.SMALLINT_TYPE_NAME; + case INT4: return serdeConstants.INT_TYPE_NAME; + case INT8: return serdeConstants.BIGINT_TYPE_NAME; + case FLOAT4: return serdeConstants.FLOAT_TYPE_NAME; + case FLOAT8: return serdeConstants.DOUBLE_TYPE_NAME; + case TEXT: return serdeConstants.STRING_TYPE_NAME; + case VARCHAR: return serdeConstants.VARCHAR_TYPE_NAME; + case NCHAR: return serdeConstants.VARCHAR_TYPE_NAME; + case NVARCHAR: return serdeConstants.VARCHAR_TYPE_NAME; + case BINARY: return serdeConstants.BINARY_TYPE_NAME; + case VARBINARY: return serdeConstants.BINARY_TYPE_NAME; + case BLOB: return serdeConstants.BINARY_TYPE_NAME; + case DATE: return serdeConstants.DATE_TYPE_NAME; + case TIMESTAMP: return serdeConstants.TIMESTAMP_TYPE_NAME; + default: + throw new CatalogException(dataType + " is not supported."); + } + } + + public static String getStoreType(String fileFormat) { + Preconditions.checkNotNull(fileFormat); + + String[] fileFormatArrary = fileFormat.split("\\."); + if(fileFormatArrary.length < 1) { + throw new CatalogException("Hive file output format is wrong. - file output format:" + fileFormat); + } + + String outputFormatClass = fileFormatArrary[fileFormatArrary.length-1]; + if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) { + return CatalogUtil.TEXTFILE_NAME; + } else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) { + return CatalogProtos.StoreType.SEQUENCEFILE.name(); + } else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) { + return CatalogProtos.StoreType.RCFILE.name(); + } else if(outputFormatClass.equals(DeprecatedParquetOutputFormat.class.getSimpleName())) { + return CatalogProtos.StoreType.PARQUET.name(); + } else { + throw new CatalogException("Not supported file output format. - file output format:" + fileFormat); + } + } + + public static Table getTable(IMetaStoreClient client, String dbName, String tableName) throws TException { + return new Table(client.getTable(dbName, tableName)); + } +} diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java new file mode 100644 index 0000000000..c4793c3393 --- /dev/null +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -0,0 +1,402 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.store; + + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.KeyValueSet; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * TestHiveCatalogStore. Test case for + * {@link org.apache.tajo.catalog.store.HiveCatalogStore} + */ + +public class TestHiveCatalogStore { + private static final String DB_NAME = "test_hive"; + private static final String CUSTOMER = "customer"; + private static final String NATION = "nation"; + private static final String REGION = "region"; + private static final String SUPPLIER = "supplier"; + + private static HiveCatalogStore store; + private static Path warehousePath; + + @BeforeClass + public static void setUp() throws Exception { + Path testPath = CommonTestingUtil.getTestDir(); + warehousePath = new Path(testPath, "warehouse"); + + //create local hiveMeta + HiveConf conf = new HiveConf(); + String jdbcUri = "jdbc:derby:;databaseName="+testPath.toUri().getPath()+"metastore_db;create=true"; + conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousePath.toUri().toString()); + conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, jdbcUri); + conf.set(TajoConf.ConfVars.WAREHOUSE_DIR.varname, warehousePath.toUri().toString()); + + // create local HiveCatalogStore. + TajoConf tajoConf = new TajoConf(conf); + store = new HiveCatalogStore(tajoConf); + store.createDatabase(DB_NAME, null); + } + + @AfterClass + public static void tearDown() throws IOException { + store.close(); + } + + @Test + public void testTableUsingTextFile() throws Exception { + TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_name", TajoDataTypes.Type.TEXT); + schema.addColumn("c_address", TajoDataTypes.Type.TEXT); + schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); + schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); + schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); + schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, CUSTOMER)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(StringEscapeUtils.escapeJava(StorageConstants.DEFAULT_FIELD_DELIMITER), + table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); + store.dropTable(DB_NAME, CUSTOMER); + } + + @Test + public void testTableUsingRCFileWithBinarySerde() throws Exception { + KeyValueSet options = new KeyValueSet(); + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("r_name", TajoDataTypes.Type.TEXT); + schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, REGION)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(StorageConstants.DEFAULT_BINARY_SERDE, + table1.getMeta().getOption(StorageConstants.RCFILE_SERDE)); + store.dropTable(DB_NAME, REGION); + } + + @Test + public void testTableUsingRCFileWithTextSerde() throws Exception { + KeyValueSet options = new KeyValueSet(); + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("r_name", TajoDataTypes.Type.TEXT); + schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, REGION)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.RCFILE_SERDE)); + store.dropTable(DB_NAME, REGION); + } + + @Test + public void testTableWithNullValue() throws Exception { + KeyValueSet options = new KeyValueSet(); + options.set(StorageConstants.TEXT_DELIMITER, StringEscapeUtils.escapeJava("\u0002")); + options.set(StorageConstants.TEXT_NULL, StringEscapeUtils.escapeJava("\u0003")); + TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4); + schema.addColumn("s_name", TajoDataTypes.Type.TEXT); + schema.addColumn("s_address", TajoDataTypes.Type.TEXT); + schema.addColumn("s_nationkey", TajoDataTypes.Type.INT4); + schema.addColumn("s_phone", TajoDataTypes.Type.TEXT); + schema.addColumn("s_acctbal", TajoDataTypes.Type.FLOAT8); + schema.addColumn("s_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, + new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri()); + + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, SUPPLIER)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, SUPPLIER)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(table.getMeta().getOption(StorageConstants.TEXT_DELIMITER), + table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER)); + + assertEquals(table.getMeta().getOption(StorageConstants.TEXT_NULL), + table1.getMeta().getOption(StorageConstants.TEXT_NULL)); + + assertEquals(table1.getMeta().getOption(StorageConstants.TEXT_DELIMITER), + StringEscapeUtils.escapeJava("\u0002")); + + assertEquals(table1.getMeta().getOption(StorageConstants.TEXT_NULL), + StringEscapeUtils.escapeJava("\u0003")); + + store.dropTable(DB_NAME, SUPPLIER); + + } + + @Test + public void testAddTableByPartition() throws Exception { + TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("n_name", TajoDataTypes.Type.TEXT); + schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); + + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta, + new Path(warehousePath, new Path(DB_NAME, NATION)).toUri()); + + org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); + expressionSchema.addColumn("n_nationkey", TajoDataTypes.Type.INT4); + + PartitionMethodDesc partitions = new PartitionMethodDesc( + DB_NAME, + NATION, + CatalogProtos.PartitionType.COLUMN, expressionSchema.getColumn(0).getQualifiedName(), expressionSchema); + table.setPartitionMethod(partitions); + + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, NATION)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, NATION)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + + Schema partitionSchema = table.getPartitionMethod().getExpressionSchema(); + Schema partitionSchema1 = table1.getPartitionMethod().getExpressionSchema(); + assertEquals(partitionSchema.size(), partitionSchema1.size()); + for (int i = 0; i < partitionSchema.size(); i++) { + assertEquals(partitionSchema.getColumn(i).getSimpleName(), partitionSchema1.getColumn(i).getSimpleName()); + } + + store.dropTable(DB_NAME, NATION); + } + + + @Test + public void testGetAllTableNames() throws Exception{ + TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("n_name", TajoDataTypes.Type.TEXT); + schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); + + String[] tableNames = new String[]{"table1", "table2", "table3"}; + + for(String tableName : tableNames){ + TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema, meta, + new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); + store.createTable(table.getProto()); + } + + List tables = store.getAllTableNames("default"); + assertEquals(tableNames.length, tables.size()); + + for(String tableName : tableNames){ + assertTrue(tables.contains(tableName)); + } + + for(String tableName : tableNames){ + store.dropTable("default", tableName); + } + } + + @Test + public void testDeleteTable() throws Exception { + TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, new KeyValueSet()); + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("n_name", TajoDataTypes.Type.TEXT); + schema.addColumn("n_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("n_comment", TajoDataTypes.Type.TEXT); + + String tableName = "table1"; + TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath.toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, tableName)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, tableName)); + FileSystem fs = FileSystem.getLocal(new Configuration()); + assertTrue(fs.exists(new Path(table1.getPath()))); + + store.dropTable(DB_NAME, tableName); + assertFalse(store.existTable(DB_NAME, tableName)); + fs.close(); + } + + @Test + public void testTableUsingSequenceFileWithBinarySerde() throws Exception { + KeyValueSet options = new KeyValueSet(); + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("r_name", TajoDataTypes.Type.TEXT); + schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, REGION)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(StorageConstants.DEFAULT_BINARY_SERDE, + table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE)); + store.dropTable(DB_NAME, REGION); + } + + @Test + public void testTableUsingSequenceFileWithTextSerde() throws Exception { + KeyValueSet options = new KeyValueSet(); + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("r_regionkey", TajoDataTypes.Type.INT4); + schema.addColumn("r_name", TajoDataTypes.Type.TEXT); + schema.addColumn("r_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, REGION)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, REGION)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + assertEquals(StorageConstants.DEFAULT_TEXT_SERDE, table1.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE)); + store.dropTable(DB_NAME, REGION); + } + + + @Test + public void testTableUsingParquet() throws Exception { + TableMeta meta = new TableMeta(CatalogProtos.StoreType.PARQUET, new KeyValueSet()); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_name", TajoDataTypes.Type.TEXT); + schema.addColumn("c_address", TajoDataTypes.Type.TEXT); + schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); + schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); + schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); + schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, CUSTOMER)); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getPath(), table1.getPath()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + store.dropTable(DB_NAME, CUSTOMER); + } +} diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 305742f9bb..d905f2eec6 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -68,12 +68,12 @@ public class TestCatalog { @BeforeClass public static void setUp() throws Exception { - final String HCATALOG_CLASS_NAME = "org.apache.tajo.catalog.store.HCatalogStore"; + final String HIVE_CATALOG_CLASS_NAME = "org.apache.tajo.catalog.store.HiveCatalogStore"; String driverClass = System.getProperty(CatalogConstants.STORE_CLASS); // here, we don't choose HCatalogStore due to some dependency problems. - if (driverClass == null || driverClass.equals(HCATALOG_CLASS_NAME)) { + if (driverClass == null || driverClass.equals(HIVE_CATALOG_CLASS_NAME)) { driverClass = DerbyStore.class.getCanonicalName(); } String catalogURI = System.getProperty(CatalogConstants.CATALOG_URI); diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 42b143b1ad..8dd67554f3 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -500,218 +500,6 @@ - - hcatalog-0.12.0 - - false - - - - org.apache.thrift - libfb303 - 0.9.0 - test - - - org.apache.thrift - libthrift - 0.9.0 - test - - - org.apache.tajo - tajo-hcatalog - ${tajo.version} - test - - - com.google.protobuf - protobuf-java - - - - - org.apache.hive - hive-exec - 0.12.0 - provided - - - org.apache.hive - hive-common - - - org.apache.hive - hive-contrib - - - org.apache.hive - hive-hbase-handler - - - org.apache.hive - hive-metastore - - - org.apache.hive - hive-serde - - - org.apache.hive - hive-shims - - - org.apache.hive - hive-testutils - - - org.apache.thrift - libfb303 - - - org.apache.thrift - libthrift - - - com.jolbox - bonecp - - - org.apache.hbase - hbase - - - com.google.protobuf - protobuf-java - - - - - org.apache.hive - hive-metastore - 0.12.0 - provided - - - org.apache.hive - hive-common - - - org.apache.hive - hive-serde - - - org.apache.hive - hive-shimss - - - org.apache.thrift - libfb303 - - - org.apache.thrift - libthrift - - - com.jolbox - bonecp - - - com.google.protobuf - protobuf-java - - - - - org.apache.hive - hive-cli - 0.12.0 - provided - - - org.apache.hive - hive-common - - - org.apache.hive - hive-exec - - - org.apache.hive - hive-metastore - - - org.apache.hive - hive-serde - - - org.apache.hive - hive-service - - - org.apache.hive - hive-shims - - - com.jolbox - bonecp - - - com.google.protobuf - protobuf-java - - - jline - jline - - - - - org.apache.hive.hcatalog - hcatalog-core - 0.12.0 - provided - - - org.apache.hive - hive-cli - - - org.apache.hive - hive-common - - - org.apache.hive - hive-exec - - - org.apache.hive - hive-metastore - - - org.apache.hive - hive-serde - - - org.apache.hive - hive-service - - - org.apache.hive - hive-shims - - - com.jolbox - bonecp - - - com.google.protobuf - protobuf-java - - - - - parallel-test diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index ddfa7a67fb..3bc99c2229 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -219,8 +219,8 @@ public void printTestName() { public QueryTestCaseBase() { // hive 0.12 does not support quoted identifier. - // So, we use lower case database names when Tajo uses HCatalogStore. - if (testingCluster.isHCatalogStoreRunning()) { + // So, we use lower case database names when Tajo uses HiveCatalogStore. + if (testingCluster.isHiveCatalogStoreRunning()) { this.currentDatabase = getClass().getSimpleName().toLowerCase(); } else { this.currentDatabase = getClass().getSimpleName(); diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 17348e1b25..f8d5c39d20 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -93,7 +93,7 @@ public class TajoTestingCluster { /** * True If HCatalogStore is used. Otherwise, it is FALSE. */ - public Boolean isHCatalogStoreUse = false; + public Boolean isHiveCatalogStoreUse = false; private static final String LOG_LEVEL; @@ -328,8 +328,8 @@ public MiniCatalogServer getMiniCatalogCluster() { return this.catalogServer; } - public boolean isHCatalogStoreRunning() { - return isHCatalogStoreUse; + public boolean isHiveCatalogStoreRunning() { + return isHiveCatalogStoreUse; } //////////////////////////////////////////////////////// @@ -411,7 +411,7 @@ private void setupCatalogForTesting(TajoConf c, File testBuildDir) throws IOExce if (!fs.exists(defaultDatabasePath)) { fs.mkdirs(defaultDatabasePath); } - isHCatalogStoreUse = true; + isHiveCatalogStoreUse = true; } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe); } diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index d6631f6c59..dc27a5ae88 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -29,7 +29,7 @@ public class TestTajoDump extends QueryTestCaseBase { @Test public void testDump1() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { executeString("CREATE TABLE \"" + getCurrentDatabase() + "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 487c497a11..e895ef1ff7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -100,7 +100,7 @@ private void assertOutputResult(String expectedResultFile, String actual) throws } private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues) - throws Exception { + throws Exception { FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile); assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile)); @@ -140,7 +140,7 @@ public void testParseParam() throws Exception { @Test public void testParseConf() throws Exception { String[] args = new String[]{"--conf", "tajo.cli.print.pause=false", - "--conf", "tajo.executor.join.inner.in-memory-table-num=256"}; + "--conf", "tajo.executor.join.inner.in-memory-table-num=256"}; CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(TajoCli.options, args); @@ -186,7 +186,7 @@ public void testLocalQueryWithoutFrom() throws Exception { public void testConnectDatabase() throws Exception { String databaseName; - if (cluster.isHCatalogStoreRunning()) { + if (cluster.isHiveCatalogStoreRunning()) { databaseName = "TEST_CONNECTION_DATABASE".toLowerCase(); } else { databaseName = "TEST_CONNECTION_DATABASE"; @@ -208,7 +208,7 @@ public void testConnectDatabase() throws Exception { @Test public void testDescTable() throws Exception { String tableName; - if (cluster.isHCatalogStoreRunning()) { + if (cluster.isHiveCatalogStoreRunning()) { tableName = "TEST_DESC_TABLE".toLowerCase(); } else { tableName = "TEST_DESC_TABLE"; @@ -225,24 +225,24 @@ public void testDescTable() throws Exception { String consoleResult = new String(out.toByteArray()); FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration()); - if (!cluster.isHCatalogStoreRunning()) { + if (!cluster.isHiveCatalogStoreRunning()) { assertOutputResult("testDescTable.result", consoleResult, new String[]{"${table.path}"}, - new String[]{fs.getUri() + "/tajo/warehouse/default/" + tableName}); + new String[]{fs.getUri() + "/tajo/warehouse/default/" + tableName}); } } @Test public void testSelectResultWithNullFalse() throws Exception { String sql = - "select\n" + - " c_custkey,\n" + - " orders.o_orderkey,\n" + - " orders.o_orderstatus \n" + - "from\n" + - " orders full outer join customer on c_custkey = o_orderkey\n" + - "order by\n" + - " c_custkey,\n" + - " orders.o_orderkey;\n"; + "select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); tajoCli.executeScript(sql); @@ -253,15 +253,15 @@ public void testSelectResultWithNullFalse() throws Exception { private void verifySelectResultWithNullTrue() throws Exception { String sql = - "select\n" + - " c_custkey,\n" + - " orders.o_orderkey,\n" + - " orders.o_orderstatus \n" + - "from\n" + - " orders full outer join customer on c_custkey = o_orderkey\n" + - "order by\n" + - " c_custkey,\n" + - " orders.o_orderkey;\n"; + "select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); @@ -291,8 +291,8 @@ private void verifyStopWhenError() throws Exception { assertSessionVar(tajoCli, SessionVars.ON_ERROR_STOP.keyname(), "true"); tajoCli.executeScript("select count(*) from lineitem; " + - "select count(*) from lineitem2; " + - "select count(*) from orders"); + "select count(*) from lineitem2; " + + "select count(*) from orders"); String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); @@ -415,7 +415,7 @@ public void testNonForwardQueryPause() throws Exception { } @Test - public void testAlterTableAddPartition() throws Exception { + public void testAlterTableAddPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTablespace.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTablespace.java index 47e98a9f9c..b0ec92d2e5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTablespace.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTablespace.java @@ -31,7 +31,7 @@ public class TestAlterTablespace extends QueryTestCaseBase { @Test public final void testAlterLocation() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { ////////////////////////////////////////////////////////////////////////////// // Create two table spaces ////////////////////////////////////////////////////////////////////////////// diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index e93d214c83..063c5e3cd4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -78,7 +78,7 @@ public final void testCtasWithoutTableDefinition() throws Exception { assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0"))); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -121,7 +121,7 @@ public final void testCtasWithColumnedPartition() throws Exception { assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0"))); - if (!cluster.isHCatalogStoreRunning()) { + if (!cluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -233,7 +233,7 @@ public final void testCtasWithManagedTable() throws Exception { ResultSet res = executeFile("CtasWithManagedTable.sql"); res.close(); - if (testingCluster.isHCatalogStoreRunning()) { + if (testingCluster.isHiveCatalogStoreRunning()) { assertTrue(client.existTable("managed_table1")); TableDesc desc = client.getTableDesc("managed_table1"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 21b3910eb2..ca1f520e8d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -41,8 +41,8 @@ public class TestCreateTable extends QueryTestCaseBase { @Test public final void testVariousTypes() throws Exception { List createdNames; - if (testingCluster.isHCatalogStoreRunning()) { - createdNames = executeDDL("create_table_various_types_for_hcatalog.sql", null); + if (testingCluster.isHiveCatalogStoreRunning()) { + createdNames = executeDDL("create_table_various_types_for_hive_catalog.sql", null); } else { createdNames = executeDDL("create_table_various_types.sql", null); } @@ -135,7 +135,7 @@ public final void testCreatedTableJustCreatedAndVerifyPath() throws Exception { @Test public final void testCreatedTableWithQuotedIdentifierAndVerifyPath() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertPathOfCreatedTable("D6", "OldTable", "NewMgmtTable", "CREATE TABLE \"D6\".\"OldTable\" (age integer);"); } } @@ -174,7 +174,7 @@ public final void testDropTableIfExists() throws Exception { @Test public final void testDelimitedIdentifierWithNonAsciiCharacters() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { ResultSet res = null; try { List tableNames = executeDDL("quoted_identifier_non_ascii_ddl.sql", "table1", "\"테이블1\""); @@ -207,7 +207,7 @@ public final void testDelimitedIdentifierWithNonAsciiCharacters() throws Excepti @Test public final void testDelimitedIdentifierWithMixedCharacters() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { ResultSet res = null; try { @@ -443,7 +443,7 @@ private boolean isClonedTable(String orignalTable, String newTable) throws Excep @Test public final void testCreateTableLike1() throws Exception { // Hcatalog does not support varchar type in hive-0.12.0 - if (testingCluster.isHCatalogStoreRunning()) { + if (testingCluster.isHiveCatalogStoreRunning()) { // Basic create table with default database executeString("CREATE TABLE table1 (c1 int, c2 text);").close(); executeString("CREATE TABLE table2 LIKE table1"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index 4c09acde13..b4334f688a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -55,7 +55,7 @@ public final void testInsertOverwrite() throws Exception { res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -75,7 +75,7 @@ public final void testInsertInto() throws Exception { res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -310,7 +310,7 @@ public final void testInsertOverwriteSmallerColumns() throws Exception { res = executeFile("testInsertOverwriteSmallerColumns.sql"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } assertEquals(originalDesc.getSchema(), desc.getSchema()); @@ -330,7 +330,7 @@ public final void testInsertOverwriteWithTargetColumns() throws Exception { res = executeFile("testInsertOverwriteWithTargetColumns.sql"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -384,7 +384,7 @@ public final void testInsertOverwriteWithAsterisk() throws Exception { res = executeString("insert overwrite into full_table_csv select * from default.lineitem where l_orderkey = 3"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "full_table_csv"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } executeString("DROP TABLE full_table_csv PURGE"); @@ -401,7 +401,7 @@ public final void testInsertOverwriteWithAsteriskAndMore() throws Exception { res = executeFile("load_to_lineitem_year_month.sql"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "lineitem_year_month"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -423,7 +423,7 @@ public final void testInsertOverwriteIntoSelect() throws Exception { CatalogService catalog = testingCluster.getMaster().getCatalog(); assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); TableDesc orderKeys = catalog.getTableDesc(getCurrentDatabase(), tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, orderKeys.getStats().getNumRows().intValue()); } @@ -434,7 +434,7 @@ public final void testInsertOverwriteIntoSelect() throws Exception { assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); orderKeys = catalog.getTableDesc(getCurrentDatabase(), tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(2, orderKeys.getStats().getNumRows().intValue()); } executeString("DROP TABLE " + tableName + " PURGE"); @@ -452,7 +452,7 @@ public final void testInsertOverwriteCapitalTableName() throws Exception { res = executeString("insert overwrite into " + tableName + " select * from default.lineitem where l_orderkey = 3"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } executeString("DROP TABLE " + tableName + " PURGE"); @@ -479,7 +479,7 @@ public final void testInsertOverwriteWithCompression() throws Exception { res = executeQuery(); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } @@ -496,7 +496,7 @@ public final void testInsertOverwriteWithCompression() throws Exception { @Test public final void testInsertOverwriteLocationWithCompression() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { ResultSet res = executeQuery(); res.close(); FileSystem fs = FileSystem.get(testingCluster.getConfiguration()); @@ -514,7 +514,7 @@ public final void testInsertOverwriteLocationWithCompression() throws Exception @Test public final void testInsertOverwriteWithAsteriskUsingParquet() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { ResultSet res = executeFile("full_table_parquet_ddl.sql"); res.close(); @@ -525,7 +525,7 @@ public final void testInsertOverwriteWithAsteriskUsingParquet() throws Exception "insert overwrite into full_table_parquet select * from default.lineitem where l_orderkey = 3"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "full_table_parquet"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } @@ -541,7 +541,7 @@ public final void testInsertOverwriteWithAsteriskUsingParquet() throws Exception @Test public final void testInsertOverwriteIntoParquet() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { executeString("create table parquet_table " + "(l_orderkey int4, l_shipdate text, l_shipdate_function text) using parquet").close(); @@ -553,7 +553,7 @@ public final void testInsertOverwriteIntoParquet() throws Exception { "select l_orderkey, l_shipdate, substr(l_shipdate, 1, 10) from default.lineitem").close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "parquet_table"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -576,7 +576,7 @@ public final void testInsertOverwriteIntoParquet() throws Exception { @Test public final void testInsertOverwriteIntoPartitionedParquet() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { executeString("create table parquet_table " + "(l_orderkey int4, l_shipdate_function text) using parquet partition by column (l_shipdate text)").close(); @@ -588,7 +588,7 @@ public final void testInsertOverwriteIntoPartitionedParquet() throws Exception { "select l_orderkey, substr(l_shipdate, 1, 10), l_shipdate from default.lineitem").close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "parquet_table"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -621,7 +621,7 @@ public final void testInsertOverwriteWithDatabase() throws Exception { res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } executeString("DROP TABLE table1 PURGE"); @@ -641,7 +641,7 @@ public final void testInsertOverwriteTableWithNonFromQuery() throws Exception { res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(1, desc.getStats().getNumRows().intValue()); } @@ -668,7 +668,7 @@ public final void testInsertOverwriteTableWithNonFromQuery2() throws Exception { res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(1, desc.getStats().getNumRows().intValue()); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java index 182bf5bad7..9e2a7bd6b4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java @@ -28,7 +28,7 @@ public class TestNetTypes extends QueryTestCaseBase { @Before public final void setUp() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { executeDDL("table1_ddl.sql", "table1"); executeDDL("table2_ddl.sql", "table2"); } @@ -37,7 +37,7 @@ public final void setUp() throws Exception { @Test public final void testSelect() throws Exception { // Skip all tests when HCatalogStore is used. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // select name, addr from table1; ResultSet res = executeQuery(); assertResultSet(res); @@ -48,7 +48,7 @@ public final void testSelect() throws Exception { @Test public final void testGroupby() throws Exception { // Skip all tests when HCatalogStore is used. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // select name, addr, count(1) from table1 group by name, addr; ResultSet res = executeQuery(); assertResultSet(res); @@ -59,7 +59,7 @@ public final void testGroupby() throws Exception { @Test public final void testGroupby2() throws Exception { // Skip all tests when HCatalogStore is used. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // select addr, count(*) from table1 group by addr; ResultSet res = executeQuery(); assertResultSet(res); @@ -70,7 +70,7 @@ public final void testGroupby2() throws Exception { @Test public final void testSort() throws Exception { // Skip all tests when HCatalogStore is used. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // select * from table1 order by addr; ResultSet res = executeQuery(); assertResultSet(res); @@ -81,7 +81,7 @@ public final void testSort() throws Exception { @Test public final void testSort2() throws Exception { // Skip all tests when HCatalogStore is used. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // select addr from table2 order by addr; ResultSet res = executeQuery(); assertResultSet(res); @@ -92,7 +92,7 @@ public final void testSort2() throws Exception { @Test public final void testJoin() throws Exception { // Skip all tests when HCatalogStore is used. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // select * from table1 as t1, table2 as t2 where t1.addr = t2.addr; ResultSet res = executeQuery(); assertResultSet(res); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index f7b1382db7..60a84c181e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -352,7 +352,7 @@ public final void testCreateAfterSelect() throws Exception { CatalogService catalog = cluster.getMaster().getCatalog(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "orderkeys")); TableDesc orderKeys = catalog.getTableDesc(DEFAULT_DATABASE_NAME, "orderkeys"); - if (!cluster.isHCatalogStoreRunning()) { + if (!cluster.isHiveCatalogStoreRunning()) { assertEquals(5, orderKeys.getStats().getNumRows().intValue()); } } @@ -374,7 +374,7 @@ public final void testSelectWithJson() throws Exception { @Test public final void testDatabaseRef() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { executeString("CREATE DATABASE \"TestSelectQuery\"").close(); executeString("CREATE TABLE \"TestSelectQuery\".\"LineItem\" AS SELECT * FROM default.lineitem" ).close(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index ff911778ff..1aee96164e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -159,10 +159,10 @@ public final void testSortAfterGroupbyWithAlias() throws Exception { @Test public final void testSortWithDate() throws Exception { - // skip this test if catalog uses HCatalogStore. - // It is because HCatalogStore does not support Time data type. + // skip this test if catalog uses HiveCatalogStore. + // It is because HiveCatalogStore does not support Time data type. - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // create external table table1 (col1 timestamp, col2 date, col3 time) ... executeDDL("create_table_with_date_ddl.sql", "table1"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index b48720a301..bb15bfcad4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -209,7 +209,7 @@ private void assertPartitionDirectories(TableDesc desc) throws IOException { assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0"))); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } } @@ -324,7 +324,7 @@ public final void testColumnPartitionedTableByThreeColumns() throws Exception { assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -388,7 +388,7 @@ public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Ex assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -441,7 +441,7 @@ public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Ex assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } String expected = "N\n" + @@ -503,7 +503,7 @@ public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Ex assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0"))); assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0"))); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // TODO: If there is existing another partition directory, we must add its rows number to result row numbers. // assertEquals(6, desc.getStats().getNumRows().intValue()); } @@ -550,7 +550,7 @@ public final void testColumnPartitionedTableByOneColumnsWithCompression() throws "insert overwrite into " + tableName + " select l_partkey, l_quantity, l_orderkey from lineitem"); res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -587,7 +587,7 @@ public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws " select l_quantity, l_returnflag, l_orderkey, l_partkey from lineitem"); res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -632,7 +632,7 @@ public final void testColumnPartitionedTableByThreeColumnsWithCompression() thro " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -715,7 +715,7 @@ public final void testColumnPartitionedTableNoMatchedPartition() throws Exceptio " select l_returnflag , l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -820,7 +820,7 @@ public final void testColumnPartitionedTableWithSmallerExpressions3() throws Exc res.close(); TableDesc desc = catalog.getTableDesc("testinsertquery1", "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } @@ -829,7 +829,7 @@ public final void testColumnPartitionedTableWithSmallerExpressions3() throws Exc res.close(); desc = catalog.getTableDesc("testinsertquery2", "table1"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 524c6e8930..98e5ff984f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -139,8 +139,8 @@ public void testMemoryResultSet() throws Exception { @Test public void testDateTimeType() throws Exception { - // Hcatalog does not support date type, time type in hive-0.12.0 - if(util.isHCatalogStoreRunning()) return; + // HiveCatalog does not support date type, time type in hive-0.12.0 + if(util.isHiveCatalogStoreRunning()) return; ResultSet res = null; TajoClient client = util.newTajoClient(); diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java index dfb0269f3b..8ee6755e11 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java @@ -62,7 +62,7 @@ public void testSetAndGetCatalogAndSchema() throws Exception { assertDatabaseExists("jdbc_test1"); pstmt.close(); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertDatabaseNotExists("Jdbc_Test2"); pstmt = conn.prepareStatement("CREATE DATABASE \"Jdbc_Test2\""); pstmt.executeUpdate(); @@ -72,7 +72,7 @@ public void testSetAndGetCatalogAndSchema() throws Exception { conn.setCatalog("jdbc_test1"); assertEquals("jdbc_test1", conn.getCatalog()); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { conn.setCatalog("Jdbc_Test2"); assertEquals("Jdbc_Test2", conn.getCatalog()); } @@ -95,7 +95,7 @@ public void testSetAndGetCatalogAndSchema() throws Exception { pstmt = conn.prepareStatement("DROP DATABASE jdbc_test1"); pstmt.executeUpdate(); pstmt.close(); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { pstmt = conn.prepareStatement("DROP DATABASE \"Jdbc_Test2\""); pstmt.executeUpdate(); pstmt.close(); @@ -126,7 +126,7 @@ public void testGetCatalogsAndTables() throws Exception { pstmt.executeUpdate(); pstmt.close(); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { // create database "jdbc_test2" and its tables assertDatabaseNotExists("Jdbc_Test4"); pstmt = defaultConnect.prepareStatement("CREATE DATABASE \"Jdbc_Test4\""); @@ -147,13 +147,13 @@ public void testGetCatalogsAndTables() throws Exception { List newDatabases = getListFromResultSet(dbmd.getCatalogs(), "TABLE_CAT"); newDatabases.removeAll(existingDatabases); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(2, newDatabases.size()); } else { assertEquals(1, newDatabases.size()); } assertTrue(newDatabases.contains("jdbc_test3")); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { assertTrue(newDatabases.contains("Jdbc_Test4")); } @@ -162,7 +162,7 @@ public void testGetCatalogsAndTables() throws Exception { assertResultSet(res, "getTables1.result"); res.close(); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { res = defaultConnect.getMetaData().getTables("Jdbc_Test4", null, null, null); assertResultSet(res, "getTables2.result"); res.close(); @@ -182,7 +182,7 @@ public void testGetCatalogsAndTables() throws Exception { executeString("DROP TABLE jdbc_test3.table2"); executeString("DROP DATABASE jdbc_test3"); - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { String jdbcTest2ConnUri = TestTajoJdbc.buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), "Jdbc_Test4"); Connection jdbcTest2Conn = DriverManager.getConnection(jdbcTest2ConnUri); @@ -268,7 +268,7 @@ private static String getTestColName(String dbName, String tableName, int i) { @Test public void testGetColumnsWithPattern() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { String connUri = TestTajoJdbc.buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), TajoConstants.DEFAULT_DATABASE_NAME); Connection conn = DriverManager.getConnection(connUri); @@ -397,7 +397,7 @@ public void testGetColumnsWithPattern() throws Exception { @Test public void testEmptyMetaInfo() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { String connUri = TestTajoJdbc.buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), TajoConstants.DEFAULT_DATABASE_NAME); Connection conn = DriverManager.getConnection(connUri); @@ -464,7 +464,7 @@ public void testEmptyMetaInfo() throws Exception { @Test public void testGetTypeInfo() throws Exception { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { String connUri = TestTajoJdbc.buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), TajoConstants.DEFAULT_DATABASE_NAME); Connection conn = DriverManager.getConnection(connUri); diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index 36bbd942df..0dbb8e546e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -553,10 +553,10 @@ public void testSortWithDateTime() throws Exception { Connection conn = null; int result; - // skip this test if catalog uses HCatalogStore. - // It is because HCatalogStore does not support Time data type. + // skip this test if catalog uses HiveCatalogStore. + // It is because HiveCatalogStore does not support Time data type. try { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { executeDDL("create_table_with_date_ddl.sql", "table1"); String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), @@ -604,10 +604,10 @@ public void testAlterTableAddPartition() throws Exception { int result; String errorMessage = null; - // skip this test if catalog uses HCatalogStore. - // It is because HCatalogStore does not support Time data type. + // skip this test if catalog uses HiveCatalogStore. + // It is because HiveCatalogStore does not support Time data type. try { - if (!testingCluster.isHCatalogStoreRunning()) { + if (!testingCluster.isHiveCatalogStoreRunning()) { String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), DEFAULT_DATABASE_NAME); conn = DriverManager.getConnection(connUri); diff --git a/tajo-core/src/test/resources/queries/TestCreateTable/create_table_various_types_for_hcatalog.sql b/tajo-core/src/test/resources/queries/TestCreateTable/create_table_various_types_for_hive_catalog.sql similarity index 100% rename from tajo-core/src/test/resources/queries/TestCreateTable/create_table_various_types_for_hcatalog.sql rename to tajo-core/src/test/resources/queries/TestCreateTable/create_table_various_types_for_hive_catalog.sql diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index aed7b4be43..0cc61aafbc 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -60,7 +60,12 @@ org.apache.tajo - tajo-rpc + tajo-rpc-protobuf + provided + + + org.apache.tajo + tajo-ws-rs provided @@ -146,9 +151,9 @@ run mkdir -p share/jdbc-dist run cp -r $ROOT/tajo-jdbc/target/tajo-jdbc-${project.version}-jar-with-dependencies.jar ./share/jdbc-dist/tajo-jdbc-${project.version}.jar - if [ -f $ROOT/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/target/lib/parquet-hive-bundle-*.jar ] + if [ -f $ROOT/tajo-catalog/tajo-catalog-drivers/tajo-hive/target/lib/parquet-hive-bundle-*.jar ] then - run cp -r $ROOT/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/target/lib/parquet-hive-bundle-*.jar lib/ + run cp -r $ROOT/tajo-catalog/tajo-catalog-drivers/tajo-hive/target/lib/parquet-hive-bundle-*.jar lib/ echo echo "Tajo installed parquet-hive-bundle library at: ${project.build.directory}/tajo-${project.version}" echo diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index cc611d537f..4383f3c474 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -295,9 +295,6 @@ if [ -d ${HIVE_LIB} ]; then CLASSPATH=${CLASSPATH}:$f; done - for f in $HIVE_HOME/hcatalog/share/hcatalog/*hcatalog-core-*.jar; do - CLASSPATH=${CLASSPATH}:$f; - done fi if [ "${HIVE_JDBC_DRIVER_DIR}" != "" ]; then @@ -390,15 +387,6 @@ elif [ "$COMMAND" = "worker" ] ; then elif [ "$COMMAND" = "pullserver" ] ; then CLASS='org.apache.tajo.pullserver.TajoPullServer' TAJO_OPTS="$TAJO_OPTS $JAVA_PULLSERVER_HEAP_MAX $TAJO_PULLSERVER_OPTS" -elif [ "$COMMAND" = "querymaster" ] ; then - CLASS='org.apache.tajo.worker.TajoWorker' - TAJO_OPTS="$TAJO_OPTS $JAVA_QUERYMASTER_HEAP_MAX $TAJO_QUERYMASTER_OPTS" - TAJO_DAEMON_MODE='standby-qm' -elif [ "$COMMAND" = "taskrunner" ] ; then - CLASS='org.apache.tajo.worker.TajoWorker' - TAJO_OPTS="$TAJO_OPTS $TAJO_WORKER_OPTS" - TAJO_OPTS="$TAJO_OPTS $JAVA_WORKER_HEAP_MAX $TAJO_WORKER_OPTS" - TAJO_DAEMON_MODE='standby-tr' elif [ "$COMMAND" = "catalog" ] ; then CLASS='org.apache.tajo.catalog.CatalogServer' TAJO_OPTS="$TAJO_OPTS $TAJO_CATALOG_OPTS" diff --git a/tajo-dist/src/main/conf/catalog-site.xml.template b/tajo-dist/src/main/conf/catalog-site.xml.template index dddcc8971b..365de6b280 100644 --- a/tajo-dist/src/main/conf/catalog-site.xml.template +++ b/tajo-dist/src/main/conf/catalog-site.xml.template @@ -86,12 +86,12 @@ --> - - + + diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh index 353d87d27e..59076e01c4 100755 --- a/tajo-dist/src/main/conf/tajo-env.sh +++ b/tajo-dist/src/main/conf/tajo-env.sh @@ -72,7 +72,7 @@ # Tajo cluster mode. the default mode is standby mode. export TAJO_WORKER_STANDBY_MODE=true -# It must be required to use HCatalogStore +# It must be required to use HiveCatalogStore # export HIVE_HOME= # export HIVE_JDBC_DRIVER_DIR= diff --git a/tajo-docs/src/main/sphinx/hcatalog_integration.rst b/tajo-docs/src/main/sphinx/hive_integration.rst similarity index 61% rename from tajo-docs/src/main/sphinx/hcatalog_integration.rst rename to tajo-docs/src/main/sphinx/hive_integration.rst index d81975df93..4c1d8d4d99 100644 --- a/tajo-docs/src/main/sphinx/hcatalog_integration.rst +++ b/tajo-docs/src/main/sphinx/hive_integration.rst @@ -1,28 +1,18 @@ ************************************* -HCatalog Integration +Hive Integration ************************************* -Apache Tajo™ catalog supports HCatalogStore driver to integrate with Apache Hive™. +Apache Tajo™ catalog supports HiveCatalogStore to integrate with Apache Hive™. This integration allows Tajo to access all tables used in Apache Hive. Depending on your purpose, you can execute either SQL queries or HiveQL queries on the same tables managed in Apache Hive. In order to use this feature, you need to build Tajo with a specified maven profile and then add some configs into ``conf/tajo-env.sh`` and ``conf/catalog-site.xml``. -This section describes how to setup HCatalog integration. -This instruction would take no more than ten minutes. +This section describes how to setup HiveMetaStore integration. +This instruction would take no more than five minutes. -First, you need to compile the source code with hcatalog profile. -Currently, Tajo supports hcatalog-0.11.0 and hcatalog-0.12.0 profile. -So, if you want to use Hive 0.11.0, you need to set ``-Phcatalog-0.11.0`` as the maven profile :: - - $ mvn clean package -DskipTests -Pdist -Dtar -Phcatalog-0.11.0 - -Or, if you want to use Hive 0.12.0, you need to set ``-Phcatalog-0.12.0`` as the maven profile :: - - $ mvn clean package -DskipTests -Pdist -Dtar -Phcatalog-0.12.0 - -Then, you need to set your Hive home directory to the environment variable ``HIVE_HOME`` in conf/tajo-env.sh as follows: :: +You need to set your Hive home directory to the environment variable ``HIVE_HOME`` in conf/tajo-env.sh as follows: :: export HIVE_HOME=/path/to/your/hive/directory @@ -31,11 +21,11 @@ Next, you should set the path of MySQL JDBC driver jar file to the environment v export HIVE_JDBC_DRIVER_DIR==/path/to/your/mysql_jdbc_driver/mysql-connector-java-x.x.x-bin.jar -Finally, you should specify HCatalogStore as Tajo catalog driver class in ``conf/catalog-site.xml`` as follows: :: +Finally, you should specify HiveCatalogStore as Tajo catalog driver class in ``conf/catalog-site.xml`` as follows: :: tajo.catalog.store.class - org.apache.tajo.catalog.store.HCatalogStore + org.apache.tajo.catalog.store.HiveCatalogStore .. note:: diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst index 0ab50b64a6..730bed44d3 100644 --- a/tajo-docs/src/main/sphinx/index.rst +++ b/tajo-docs/src/main/sphinx/index.rst @@ -39,7 +39,7 @@ Table of Contents: table_partitioning index_overview backup_and_restore - hcatalog_integration + hive_integration hbase_integration swift_integration jdbc_driver diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index d7cd82e29a..12d96b6e4d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -857,7 +857,7 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, } Column column = columns.iterator().next(); - // If catalog runs with HCatalog, partition column is a qualified name + // If catalog runs with HiveCatalog, partition column is a qualified name // Else partition column is a simple name boolean isPartitionColumn = false; if (hasQualifiedName) { diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index d9685ee670..6fa995df92 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -37,7 +37,9 @@ 2.5.0 0.11.0-SNAPSHOT 0.98.7-hadoop2 + 1.1.0 4.0.25.Final + 2.6 ${project.parent.relativePath}/.. src/main/hadoop-${hadoop.version} @@ -787,6 +789,21 @@ tajo-rpc ${tajo.version} + + org.apache.tajo + tajo-rpc-common + ${tajo.version} + + + org.apache.tajo + tajo-rpc-protobuf + ${tajo.version} + + + org.apache.tajo + tajo-ws-rs + ${tajo.version} + org.apache.tajo tajo-algebra @@ -995,6 +1012,11 @@ commons-lang 2.6 + + commons-codec + commons-codec + 1.10 + com.google.guava guava @@ -1063,6 +1085,21 @@ jcip-annotations 1.0-1 + + org.glassfish.jersey.core + jersey-common + ${jersey.version} + + + org.glassfish.jersey.core + jersey-server + ${jersey.version} + + + javax.ws.rs + javax.ws.rs-api + 2.0.1 + net.minidev json-smart