From 7c98709f0fcb06dfb675acae3d6489a6126f55b5 Mon Sep 17 00:00:00 2001 From: jinossy Date: Wed, 6 Aug 2014 17:43:35 +0900 Subject: [PATCH 1/3] TAJO-995: HiveMetaStoreClient wrapper should retry the connection --- .../tajo/catalog/store/HCatalogStore.java | 11 +++++----- .../store/HCatalogStoreClientPool.java | 20 +++++++++++++++---- .../tajo/catalog/store/HCatalogUtil.java | 6 ++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 6f483481fe..d27f19c51c 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -48,6 +48,7 @@ import org.apache.tajo.exception.InternalException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; +import org.apache.thrift.TException; import java.io.IOException; import java.util.*; @@ -77,13 +78,13 @@ public HCatalogStore(final Configuration conf) throws InternalException { @Override public boolean existTable(final String databaseName, final String tableName) throws CatalogException { boolean exist = false; - org.apache.hadoop.hive.ql.metadata.Table table = null; + org.apache.hadoop.hive.ql.metadata.Table table; HCatalogStoreClientPool.HCatalogStoreClient client = null; // get table try { client = clientPool.getClient(); - table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName); + table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); if (table != null) { exist = true; } @@ -118,7 +119,7 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St // get hive table schema try { client = clientPool.getClient(); - table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName); + table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); path = table.getPath(); } catch (NoSuchObjectException nsoe) { throw new CatalogException("Table not found. - tableName:" + tableName, nsoe); @@ -291,7 +292,7 @@ public final List getAllTableNames(String databaseName) throws CatalogEx try { client = clientPool.getClient(); return client.getHiveClient().getAllTables(databaseName); - } catch (MetaException e) { + } catch (TException e) { throw new CatalogException(e); } finally { if(client != null) client.release(); @@ -401,7 +402,7 @@ public Collection getAllDatabaseNames() throws CatalogException { try { client = clientPool.getClient(); return client.getHiveClient().getAllDatabases(); - } catch (MetaException e) { + } catch (TException e) { throw new CatalogException(e); } finally { if (client != null) { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java index 2fff67c722..8ccb100b01 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java @@ -16,7 +16,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.*; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.log4j.Logger; import java.util.Iterator; @@ -41,12 +43,22 @@ public class HCatalogStoreClientPool { * connection pool. */ public class HCatalogStoreClient { - private final HiveMetaStoreClient hiveClient; + private final IMetaStoreClient hiveClient; public AtomicBoolean isInUse = new AtomicBoolean(false); private HCatalogStoreClient(HiveConf hiveConf) { try { - this.hiveClient = new HiveMetaStoreClient(hiveConf); + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook(Table table) throws MetaException { + /* metadata hook implementation, or null if this + * storage handler does not need any metadata notifications + */ + return null; + } + }; + + this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName()); clientPool.add(this); LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")"); } catch (Exception e) { @@ -58,7 +70,7 @@ private HCatalogStoreClient(HiveConf hiveConf) { /** * Returns the internal HiveMetaStoreClient object. */ - public HiveMetaStoreClient getHiveClient() { + public IMetaStoreClient getHiveClient() { return hiveClient; } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java index 9e60768ad6..54fdb9db84 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java @@ -20,9 +20,11 @@ import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hcatalog.common.HCatException; import org.apache.hcatalog.data.schema.HCatFieldSchema; @@ -30,6 +32,7 @@ import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; +import org.apache.thrift.TException; import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; public class HCatalogUtil { @@ -137,4 +140,7 @@ public static String getStoreType(String fileFormat) { } } + public static Table getTable(IMetaStoreClient client, String dbName, String tableName) throws TException { + return new Table(client.getTable(dbName, tableName)); + } } From 9147f43f12931bdb80fb09e31fbeb266045e2e0a Mon Sep 17 00:00:00 2001 From: Jaehwa Jung Date: Thu, 4 Sep 2014 14:56:40 +0900 Subject: [PATCH 2/3] TAJO-1027: Upgrade Hive to 0.13.0 and 0.13.1 --- tajo-catalog/tajo-catalog-drivers/pom.xml | 18 + .../tajo-hcatalog/pom.xml | 345 ++++++++++++++++++ tajo-dist/src/main/bin/tajo | 2 +- .../configuration/catalog_configuration.rst | 2 +- 4 files changed, 365 insertions(+), 2 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/pom.xml b/tajo-catalog/tajo-catalog-drivers/pom.xml index 2cb05eec6a..001329ddf4 100644 --- a/tajo-catalog/tajo-catalog-drivers/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/pom.xml @@ -69,6 +69,24 @@ tajo-hcatalog + + hcatalog-0.13.0 + + false + + + tajo-hcatalog + + + + hcatalog-0.13.1 + + false + + + tajo-hcatalog + + diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml index 02ea16be0a..0aed5b5560 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml @@ -286,12 +286,322 @@ org.apache.hive.hcatalog hcatalog-core ${hive.version} + + + org.apache.hive + hive-cli + + + org.apache.hive + hive-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-service + + + org.apache.hive + hive-shims + + + com.jolbox + bonecp + + + + + com.twitter + parquet-hive-bundle + ${parquet.version} + + + + + hcatalog-0.13.0 + + false + + + 0.13.0 + 1.5.0 + 2.1.0 + + + + org.apache.hive + hive-exec + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-contrib + + + org.apache.hive + hive-hbase-handler + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shims + + + org.apache.hive + hive-testutils + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + com.jolbox + bonecp + + + com.google.protobuf + protobuf-java + + + + + org.apache.hive + hive-metastore + ${hive.version} provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shimss + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + com.jolbox + bonecp + + + + + org.apache.hive + hive-cli + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-service + + + org.apache.hive + hive-shims + + + com.jolbox + bonecp + + + jline + jline + + + + + org.apache.hive.hcatalog + hive-hcatalog-core + ${hive.version} org.apache.hive hive-cli + + org.apache.hive + hive-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-metastore + + + com.google.guava + guava + + + org.codehaus.jackson + jackson-mapper-asl + + + + + com.twitter + parquet-hive-bundle + ${parquet.version} + + + + + hcatalog-0.13.1 + + false + + + 0.13.1 + 1.5.0 + 2.1.0 + + + + org.apache.hive + hive-exec + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-contrib + + + org.apache.hive + hive-hbase-handler + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shims + + + org.apache.hive + hive-testutils + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + com.jolbox + bonecp + + + com.google.protobuf + protobuf-java + + + + + org.apache.hive + hive-metastore + ${hive.version} + provided + + + org.apache.hive + hive-common + + + org.apache.hive + hive-serde + + + org.apache.hive + hive-shimss + + + org.apache.thrift + libfb303 + + + org.apache.thrift + libthrift + + + com.jolbox + bonecp + + + + + org.apache.hive + hive-cli + ${hive.version} + provided + org.apache.hive hive-common @@ -320,6 +630,41 @@ com.jolbox bonecp + + jline + jline + + + + + org.apache.hive.hcatalog + hive-hcatalog-core + ${hive.version} + + + org.apache.hive + hive-cli + + + org.apache.hive + hive-common + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-metastore + + + com.google.guava + guava + + + org.codehaus.jackson + jackson-mapper-asl + diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index 3fc2ae0560..4236d9b4c7 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -248,7 +248,7 @@ if [ -d ${HIVE_LIB} ]; then CLASSPATH=${CLASSPATH}:$f; done - for f in $HIVE_HOME/hcatalog/share/hcatalog/hcatalog-core-*.jar; do + for f in $HIVE_HOME/hcatalog/share/hcatalog/*hcatalog-core-*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi diff --git a/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst b/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst index 9b2404a8a0..cdf7168825 100644 --- a/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst +++ b/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst @@ -107,7 +107,7 @@ First, you must compile source code and get a binary archive as follows: $ mvn clean install -DskipTests -Pdist -Dtar -Phcatalog-0.1x.0 $ ls tajo-dist/target/tajo-x.y.z-SNAPSHOT.tar.gz -Currently Tajo supports only hive 0.12.0. If you enables HCatalogStore, you set the maven profile as ``-Phcatalog-0.12.0``. +Currently Tajo supports hive 0.12.0, hive 0.13.0, hive 0.13.1. If you enables HCatalogStore, you set the maven profile as ``-Phcatalog-0.12.0``. Second, you must set your hive home directory to HIVE_HOME variable in ``conf/tajo-env.sh`` with it as follows: From fca2e743d2c5978b297a804b42de4d04fd4dc578 Mon Sep 17 00:00:00 2001 From: Jaehwa Jung Date: Thu, 4 Sep 2014 15:09:19 +0900 Subject: [PATCH 3/3] Added dependency scope for hive-hcatalog-core. --- tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml index 0aed5b5560..203d56a47f 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml @@ -467,6 +467,7 @@ org.apache.hive.hcatalog hive-hcatalog-core ${hive.version} + provided org.apache.hive @@ -640,6 +641,7 @@ org.apache.hive.hcatalog hive-hcatalog-core ${hive.version} + provided org.apache.hive