From 3458323cd1ae1db0d6ea0a77b621775d4e379d9d Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Mon, 28 Aug 2017 14:18:53 -0700 Subject: [PATCH 01/10] Moved EventMessage and friends. --- standalone-metastore/pom.xml | 6 +++ .../messaging/AddForeignKeyMessage.java | 2 +- .../AddNotNullConstraintMessage.java | 2 +- .../messaging/AddPartitionMessage.java | 2 +- .../messaging/AddPrimaryKeyMessage.java | 2 +- .../messaging/AddUniqueConstraintMessage.java | 2 +- .../messaging/AlterIndexMessage.java | 2 +- .../messaging/AlterPartitionMessage.java | 2 +- .../messaging/AlterTableMessage.java | 2 +- .../messaging/CreateDatabaseMessage.java | 2 +- .../messaging/CreateFunctionMessage.java | 2 +- .../messaging/CreateIndexMessage.java | 2 +- .../messaging/CreateTableMessage.java | 2 +- .../messaging/DropConstraintMessage.java | 2 +- .../messaging/DropDatabaseMessage.java | 2 +- .../messaging/DropFunctionMessage.java | 2 +- .../metastore/messaging/DropIndexMessage.java | 2 +- .../messaging/DropPartitionMessage.java | 2 +- .../metastore/messaging/DropTableMessage.java | 2 +- .../metastore/messaging/EventMessage.java | 4 +- .../metastore/messaging/InsertMessage.java | 2 +- .../messaging/MessageDeserializer.java | 2 +- .../metastore/messaging/MessageFactory.java | 42 +++++++++---------- .../metastore/messaging/PartitionFiles.java | 2 +- .../hive/metastore/utils/JavaUtils.java | 14 +++++++ 25 files changed, 63 insertions(+), 45 deletions(-) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java (87%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java (99%) diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 9cfc95ef5b66..1f5f5f57deca 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -225,6 +225,12 @@ javax.jdo ${datanucleus-jdo.version} + + org.skyscreamer + jsonassert + 1.4.0 + test + diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java index 2eb14a1d61c7..db8a7d79fb9c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddForeignKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java index 28ee610af883..07fbe644a22f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddNotNullConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java index 774c020ff5c0..3262b52beaf6 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java index 0e899ad75307..37f2b46c7ce1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddPrimaryKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java index 8072d84f63e0..5c71f22aa775 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AddUniqueConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java index e72a94bfbe04..78259ba7c373 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java index 077c9f7e9ccf..aaa7ef5299d3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java index 58f01fe12677..30e28629a07b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java index 761429884cff..328c1180e150 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java index d94a3f06c3ac..54781436987b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java index 3ce0d62c3ac2..f10710023d9b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java index b75caa635163..49732ff97d1a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/CreateTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java index 6e691e9cc45b..95c9f9f423b4 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java index fa6da38f884e..a450d4769984 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java index 2b45d40f8909..c418feb76456 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java index 5997f92ba9d8..210b5920f366 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java index d254ad93f6d5..fa7c9dbe634b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java index 7aed21b488b0..b8e0e78a9df4 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/DropTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java index 3b37cb3e8ba6..7b22facb1a42 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,7 +28,7 @@ public abstract class EventMessage { /** * Enumeration of all supported types of Metastore operations. */ - public static enum EventType { + public enum EventType { CREATE_DATABASE(MessageFactory.CREATE_DATABASE_EVENT), DROP_DATABASE(MessageFactory.DROP_DATABASE_EVENT), diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java index 01fc0f2cdebb..c47009716570 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/InsertMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java index 50a2728aa40c..810dc64acdc0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java index e639e482d09e..46fd33678975 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,18 +19,20 @@ package org.apache.hadoop.hive.metastore.messaging; -import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.utils.JavaUtils; import java.util.Iterator; import java.util.List; @@ -63,21 +65,18 @@ public abstract class MessageFactory { private static MessageFactory instance = null; - protected static final HiveConf hiveConf = new HiveConf(); + protected static final Configuration conf = MetastoreConf.newMetastoreConf(); + /* + // TODO MS-SPLIT I'm 99% certain we don't need this, as MetastoreConf.newMetastoreConf already + adds this resource. static { - hiveConf.addResource("hive-site.xml"); + conf.addResource("hive-site.xml"); } + */ - // This parameter is retained for legacy reasons, in case someone implemented custom - // factories. This, however, should not be the case, since this api was intended to - // be internal-only, and we should manage the jms and json implementations without - // needing this parameter. Marking as deprecated, for removal by 2.4 - see corresponding - // note on the getDeserializer(String,String) method - @Deprecated - private static final String CONF_LABEL_HCAT_MESSAGE_FACTORY_IMPL_PREFIX = "hcatalog.message.factory.impl."; - - protected static final String MS_SERVER_URL = hiveConf.get(HiveConf.ConfVars.METASTOREURIS.name(), ""); - protected static final String MS_SERVICE_PRINCIPAL = hiveConf.get(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.name(), ""); + protected static final String MS_SERVER_URL = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS, ""); + protected static final String MS_SERVICE_PRINCIPAL = + MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL, ""); /** * Getter for MessageFactory instance. @@ -85,17 +84,17 @@ public abstract class MessageFactory { public static MessageFactory getInstance() { if (instance == null) { instance = - getInstance(hiveConf.get(HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY.varname)); + getInstance(MetastoreConf.getVar(conf, ConfVars.EVENT_MESSAGE_FACTORY)); } return instance; } private static MessageFactory getInstance(String className) { try { - return (MessageFactory)ReflectionUtils.newInstance(JavaUtils.loadClass(className), hiveConf); + return JavaUtils.newInstance(JavaUtils.getClass(className, MessageFactory.class)); } - catch (ClassNotFoundException classNotFound) { - throw new IllegalStateException("Could not construct MessageFactory implementation: ", classNotFound); + catch (MetaException e) { + throw new IllegalStateException("Could not construct MessageFactory implementation: ", e); } } @@ -107,8 +106,7 @@ private static MessageFactory getInstance(String className) { */ public static MessageDeserializer getDeserializer(String format, String version) { - return getInstance(hiveConf.get(CONF_LABEL_HCAT_MESSAGE_FACTORY_IMPL_PREFIX + format, - HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY.varname)).getDeserializer(); + return getInstance(MetastoreConf.getVar(conf, ConfVars.EVENT_MESSAGE_FACTORY)).getDeserializer(); // Note : The reason this method exists outside the no-arg getDeserializer method is in // case there is a user-implemented MessageFactory that's used, and some the messages // are in an older format and the rest in another. Then, what MessageFactory is default diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java index 4fd7f8c57b9e..308eaf1f6445 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/PartitionFiles.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java index 593dee3996f3..b08d9fd71f6c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java @@ -87,6 +87,20 @@ public static T newInstance(Class theClass, Class[] parameterTypes, } } + /** + * Create an object of the given class using a no-args constructor + * @param theClass class to return new object of + * @param the type of the class to be returned + * @return an object of the requested type + */ + public static T newInstance(Class theClass) { + try { + return theClass.newInstance(); + } catch (InstantiationException|IllegalAccessException e) { + throw new RuntimeException("Unable to instantiate " + theClass.getName(), e); + } + } + /** * @return name of current host */ From 0c330f65e7d9da0f7382430d94e7a27cd80ff323 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Mon, 28 Aug 2017 14:39:27 -0700 Subject: [PATCH 02/10] Moved JSON message implementations. --- .../metastore/messaging/json/JSONAddForeignKeyMessage.java | 6 +++--- .../messaging/json/JSONAddNotNullConstraintMessage.java | 6 +++--- .../metastore/messaging/json/JSONAddPartitionMessage.java | 6 +++--- .../metastore/messaging/json/JSONAddPrimaryKeyMessage.java | 6 +++--- .../messaging/json/JSONAddUniqueConstraintMessage.java | 6 +++--- .../metastore/messaging/json/JSONAlterIndexMessage.java | 2 +- .../metastore/messaging/json/JSONAlterPartitionMessage.java | 4 +--- .../metastore/messaging/json/JSONAlterTableMessage.java | 2 +- .../metastore/messaging/json/JSONCreateDatabaseMessage.java | 2 +- .../metastore/messaging/json/JSONCreateFunctionMessage.java | 2 +- .../metastore/messaging/json/JSONCreateIndexMessage.java | 2 +- .../metastore/messaging/json/JSONCreateTableMessage.java | 2 +- .../metastore/messaging/json/JSONDropConstraintMessage.java | 2 +- .../metastore/messaging/json/JSONDropDatabaseMessage.java | 2 +- .../metastore/messaging/json/JSONDropFunctionMessage.java | 3 +-- .../hive/metastore/messaging/json/JSONDropIndexMessage.java | 3 +-- .../metastore/messaging/json/JSONDropPartitionMessage.java | 2 +- .../hive/metastore/messaging/json/JSONDropTableMessage.java | 2 +- .../hive/metastore/messaging/json/JSONInsertMessage.java | 2 +- .../metastore/messaging/json/JSONMessageDeserializer.java | 2 +- .../hive/metastore/messaging/json/JSONMessageFactory.java | 4 ++-- .../messaging/json/JSONMessageDeserializerTest.java | 2 +- 22 files changed, 33 insertions(+), 37 deletions(-) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java (95%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java (95%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java (97%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java (95%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java (95%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java (98%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java (98%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java (98%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java (99%) rename {metastore/src/test => standalone-metastore/src/test/java}/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java (99%) diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java index 9c04c15191eb..9c912bf11ada 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddForeignKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -51,7 +51,7 @@ public JSONAddForeignKeyMessage(String server, String servicePrincipal, List(); + this.foreignKeyListJson = new ArrayList<>(); try { for (SQLForeignKey pk : fks) { foreignKeyListJson.add(JSONMessageFactory.createForeignKeyObjJson(pk)); @@ -83,7 +83,7 @@ public Long getTimestamp() { @Override public List getForeignKeys() throws Exception { - List fks = new ArrayList(); + List fks = new ArrayList<>(); for (String pkJson : foreignKeyListJson) { fks.add((SQLForeignKey)JSONMessageFactory.getTObj(pkJson, SQLForeignKey.class)); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java index 95ff23adb109..79312e4290b9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddNotNullConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,7 +46,7 @@ public JSONAddNotNullConstraintMessage(String server, String servicePrincipal, L this.server = server; this.servicePrincipal = servicePrincipal; this.timestamp = timestamp; - this.notNullConstraintListJson = new ArrayList(); + this.notNullConstraintListJson = new ArrayList<>(); try { for (SQLNotNullConstraint nn : nns) { notNullConstraintListJson.add(JSONMessageFactory.createNotNullConstraintObjJson(nn)); @@ -78,7 +78,7 @@ public Long getTimestamp() { @Override public List getNotNullConstraints() throws Exception { - List nns = new ArrayList(); + List nns = new ArrayList<>(); for (String nnJson : notNullConstraintListJson) { nns.add((SQLNotNullConstraint)JSONMessageFactory.getTObj(nnJson, SQLNotNullConstraint.class)); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java similarity index 97% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java index db3431eac939..c232ccef3a4d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -75,8 +75,8 @@ public JSONAddPartitionMessage(String server, String servicePrincipal, Table tab this.table = tableObj.getTableName(); this.tableType = tableObj.getTableType(); this.timestamp = timestamp; - partitions = new ArrayList>(); - partitionListJson = new ArrayList(); + partitions = new ArrayList<>(); + partitionListJson = new ArrayList<>(); Partition partitionObj; try { this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java index 2551cbf0064b..f4f4d5df999e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddPrimaryKeyMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -51,7 +51,7 @@ public JSONAddPrimaryKeyMessage(String server, String servicePrincipal, List(); + this.primaryKeyListJson = new ArrayList<>(); try { for (SQLPrimaryKey pk : pks) { primaryKeyListJson.add(JSONMessageFactory.createPrimaryKeyObjJson(pk)); @@ -83,7 +83,7 @@ public Long getTimestamp() { @Override public List getPrimaryKeys() throws Exception { - List pks = new ArrayList(); + List pks = new ArrayList<>(); for (String pkJson : primaryKeyListJson) { pks.add((SQLPrimaryKey)JSONMessageFactory.getTObj(pkJson, SQLPrimaryKey.class)); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java index 37f6b8137231..5a3cf96e6b91 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAddUniqueConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -48,7 +48,7 @@ public JSONAddUniqueConstraintMessage(String server, String servicePrincipal, Li this.server = server; this.servicePrincipal = servicePrincipal; this.timestamp = timestamp; - this.uniqueConstraintListJson = new ArrayList(); + this.uniqueConstraintListJson = new ArrayList<>(); try { for (SQLUniqueConstraint uk : uks) { uniqueConstraintListJson.add(JSONMessageFactory.createUniqueConstraintObjJson(uk)); @@ -80,7 +80,7 @@ public Long getTimestamp() { @Override public List getUniqueConstraints() throws Exception { - List uks = new ArrayList(); + List uks = new ArrayList<>(); for (String pkJson : uniqueConstraintListJson) { uks.add((SQLUniqueConstraint)JSONMessageFactory.getTObj(pkJson, SQLUniqueConstraint.class)); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java index 5dffdd104324..d1657b466957 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java index 2991e08836a1..68cbd95a144f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,8 +24,6 @@ import org.apache.thrift.TException; import org.codehaus.jackson.annotate.JsonProperty; -import java.util.ArrayList; -import java.util.List; import java.util.Map; /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java index 1df9c8b15988..ce8c0fa18bd2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java index f8717b24c1b9..f442e99923f8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java index 3646d85e837e..81f5d25fa58d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java index a2e2fc02e3b8..a40e5243fcfe 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java index 9dd7a744c197..d894af9b5867 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java index 4d3422b94b3a..9e5d582cefda 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java index be17e6d5ac12..cd8bcdc81284 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java index b9ee4c4dfdeb..ca32d161a715 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; -import org.apache.thrift.TException; import org.codehaus.jackson.annotate.JsonProperty; /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java similarity index 98% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java index e99a31f68b85..fb719c2dc272 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropIndexMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.messaging.DropIndexMessage; -import org.apache.thrift.TException; import org.codehaus.jackson.annotate.JsonProperty; /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java index 576806cf62d0..46897116935d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java index 4334f732fd01..591c9dd08863 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java index 1369fd2f3b03..39372bdbfa68 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java index 7f588a04c9c2..15fa4aa039e0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java index c7877ee7cc9e..916a8e8f358b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -205,7 +205,7 @@ private long now() { } static Map getPartitionKeyValues(Table table, Partition partition) { - Map partitionKeys = new LinkedHashMap(); + Map partitionKeys = new LinkedHashMap<>(); for (int i = 0; i < table.getPartitionKeysSize(); ++i) partitionKeys.put(table.getPartitionKeys().get(i).getName(), partition.getValues().get(i)); return partitionKeys; diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java similarity index 99% rename from metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java index c278338c1955..90329c8d4005 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From b34f893b5057e4ec2d877ada7b7662600dfa3e8b Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Mon, 28 Aug 2017 16:56:09 -0700 Subject: [PATCH 03/10] Moved RetryingHMSHandler --- .../hadoop/hive/metastore/HiveMetaStore.java | 10 +++++ .../hadoop/hive/metastore/IHMSHandler.java | 6 +++ .../hive/metastore/RetryingHMSHandler.java | 39 ++++++++--------- .../TestRetriesInRetryingHMSHandler.java | 43 +++++++++---------- 4 files changed, 56 insertions(+), 42 deletions(-) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java (87%) rename {metastore/src/test => standalone-metastore/src/test/java}/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java (67%) diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 23033fac81ce..c53baac76966 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -410,10 +410,20 @@ public static String getThreadLocalIpAddress() { return threadLocalIpAddress.get(); } + /** + * Use {@link #getThreadId()} instead. + * @return thread id + */ + @Deprecated public static Integer get() { return threadLocalId.get(); } + @Override + public int getThreadId() { + return threadLocalId.get(); + } + public HMSHandler(String name) throws MetaException { this(name, new HiveConf(HMSHandler.class), true); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java index 633b3c7a171d..eab290542f6e 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java @@ -25,4 +25,10 @@ public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable { void init() throws MetaException; + + /** + * Get the id of the thread of this handler. + * @return thread id + */ + int getThreadId(); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java index affb38f882f0..559ff34f5702 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -26,13 +26,14 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.metrics.PerfLogger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.datanucleus.exceptions.NucleusException; @@ -58,21 +59,21 @@ public Result(Object result, int numRetries) { private final MetaStoreInit.MetaStoreInitData metaStoreInitData = new MetaStoreInit.MetaStoreInitData(); - private final HiveConf origConf; // base configuration + private final Configuration origConf; // base configuration private final Configuration activeConf; // active configuration - private RetryingHMSHandler(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) throws MetaException { - this.origConf = hiveConf; + private RetryingHMSHandler(Configuration origConf, IHMSHandler baseHandler, boolean local) throws MetaException { + this.origConf = origConf; this.baseHandler = baseHandler; if (local) { - baseHandler.setConf(hiveConf); // tests expect configuration changes applied directly to metastore + baseHandler.setConf(origConf); // tests expect configuration changes applied directly to metastore } activeConf = baseHandler.getConf(); // This has to be called before initializing the instance of HMSHandler // Using the hook on startup ensures that the hook always has priority // over settings in *.xml. The thread local conf needs to be used because at this point // it has already been initialized using hiveConf. - MetaStoreInit.updateConnectionURL(hiveConf, getActiveConf(), null, metaStoreInitData); + MetaStoreInit.updateConnectionURL(origConf, getActiveConf(), null, metaStoreInitData); try { //invoking init method of baseHandler this way since it adds the retry logic //in case of transient failures in init method @@ -86,10 +87,10 @@ private RetryingHMSHandler(HiveConf hiveConf, IHMSHandler baseHandler, boolean l } } - public static IHMSHandler getProxy(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) + public static IHMSHandler getProxy(Configuration conf, IHMSHandler baseHandler, boolean local) throws MetaException { - RetryingHMSHandler handler = new RetryingHMSHandler(hiveConf, baseHandler, local); + RetryingHMSHandler handler = new RetryingHMSHandler(conf, baseHandler, local); return (IHMSHandler) Proxy.newProxyInstance( RetryingHMSHandler.class.getClassLoader(), @@ -99,7 +100,7 @@ public static IHMSHandler getProxy(HiveConf hiveConf, IHMSHandler baseHandler, b @Override public Object invoke(final Object proxy, final Method method, final Object[] args) throws Throwable { int retryCount = -1; - int threadId = HiveMetaStore.HMSHandler.get(); + int threadId = baseHandler.getThreadId(); boolean error = true; PerfLogger perfLogger = PerfLogger.getPerfLogger(false); perfLogger.PerfLogBegin(CLASS_NAME, method.getName()); @@ -119,14 +120,12 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg public Result invokeInternal(final Object proxy, final Method method, final Object[] args) throws Throwable { boolean gotNewConnectUrl = false; - boolean reloadConf = HiveConf.getBoolVar(origConf, - HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF); - long retryInterval = HiveConf.getTimeVar(origConf, - HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS); - int retryLimit = HiveConf.getIntVar(origConf, - HiveConf.ConfVars.HMSHANDLERATTEMPTS); - long timeout = HiveConf.getTimeVar(origConf, - HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS); + boolean reloadConf = MetastoreConf.getBoolVar(origConf, ConfVars.HMSHANDLERFORCERELOADCONF); + long retryInterval = MetastoreConf.getTimeVar(origConf, + ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS); + int retryLimit = MetastoreConf.getIntVar(origConf, ConfVars.HMSHANDLERATTEMPTS); + long timeout = MetastoreConf.getTimeVar(origConf, + ConfVars.CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS); Deadline.registerIfNot(timeout); diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java similarity index 67% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java index 0d2a9cb18016..8d44bf80814c 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,26 +24,25 @@ import javax.jdo.JDOException; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; public class TestRetriesInRetryingHMSHandler { - private static HiveConf hiveConf; + private static Configuration conf; private static final int RETRY_ATTEMPTS = 3; @BeforeClass public static void setup() throws IOException { - hiveConf = new HiveConf(); - int port = MetaStoreTestUtils.findFreePort(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); - hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, RETRY_ATTEMPTS); - hiveConf.setTimeVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, 10, TimeUnit.MILLISECONDS); - hiveConf.setBoolVar(HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF, false); + conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setLongVar(conf, ConfVars.HMSHANDLERATTEMPTS, RETRY_ATTEMPTS); + MetastoreConf.setTimeVar(conf, ConfVars.HMSHANDLERINTERVAL, 10, TimeUnit.MILLISECONDS); + MetastoreConf.setBoolVar(conf, ConfVars.HMSHANDLERFORCERELOADCONF, false); } /* @@ -52,13 +51,13 @@ public static void setup() throws IOException { */ @Test public void testRetryInit() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); Mockito .doThrow(JDOException.class) .doNothing() .when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(2)).init(); } @@ -67,10 +66,10 @@ public void testRetryInit() throws MetaException { */ @Test public void testNoRetryInit() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); Mockito.doNothing().when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(1)).init(); } @@ -80,10 +79,10 @@ public void testNoRetryInit() throws MetaException { */ @Test(expected = MetaException.class) public void testRetriesLimit() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); Mockito.doThrow(JDOException.class).when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(RETRY_ATTEMPTS)).init(); } @@ -93,8 +92,8 @@ public void testRetriesLimit() throws MetaException { */ @Test public void testWrappedMetaExceptionRetry() throws MetaException { - IHMSHandler mockBaseHandler = Mockito.mock(HiveMetaStore.HMSHandler.class); - Mockito.when(mockBaseHandler.getConf()).thenReturn(hiveConf); + IHMSHandler mockBaseHandler = Mockito.mock(IHMSHandler.class); + Mockito.when(mockBaseHandler.getConf()).thenReturn(conf); //JDOException wrapped in MetaException wrapped in InvocationException MetaException me = new MetaException("Dummy exception"); me.initCause(new JDOException()); @@ -103,7 +102,7 @@ public void testWrappedMetaExceptionRetry() throws MetaException { .doThrow(me) .doNothing() .when(mockBaseHandler).init(); - RetryingHMSHandler.getProxy(hiveConf, mockBaseHandler, false); + RetryingHMSHandler.getProxy(conf, mockBaseHandler, false); Mockito.verify(mockBaseHandler, Mockito.times(2)).init(); } } From 7430867403cd1e54e8b58138e9398f9924e0c04a Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Wed, 30 Aug 2017 10:35:31 -0700 Subject: [PATCH 04/10] Moved events. As a part of this I had to change events and listeners to take IHMSHandler interface rather than a direct reference to HMSHandler. I believe this is an improvement as it gives a clean separation between the interface and the implementation. But it will break existing listeners. As part of this I had to add a few methods to IHMSHandler. We may need to continue to add methods here to support existing external listeners. --- .../listener/NotificationListener.java | 20 ++++----- .../metastore/TestMetaStoreEventListener.java | 4 +- ...mmyHiveMetastoreAuthorizationProvider.java | 4 +- .../InjectableDummyAuthenticator.java | 4 +- .../hive/metastore/FileMetadataManager.java | 7 ++- .../hadoop/hive/metastore/HiveMetaStore.java | 35 +++------------ .../hive/ql/parse/MetaDataExportListener.java | 16 +++---- .../HadoopDefaultMetastoreAuthenticator.java | 4 +- .../HiveMetastoreAuthenticationProvider.java | 4 +- ...ultHiveMetastoreAuthorizationProvider.java | 4 +- .../HiveAuthorizationProviderBase.java | 6 +-- .../HiveMetastoreAuthorizationProvider.java | 4 +- .../MetaStoreAuthzAPIAuthorizerEmbedOnly.java | 4 +- .../StorageBasedAuthorizationProvider.java | 4 +- .../hive/metastore/api/ClientCapability.java | 8 +--- .../hadoop/hive/metastore/IHMSHandler.java | 45 +++++++++++++++++++ .../hive/metastore/conf/MetastoreConf.java | 12 +++++ .../metastore/events/AddForeignKeyEvent.java | 10 +++-- .../hive/metastore/events/AddIndexEvent.java | 10 ++--- .../events/AddNotNullConstraintEvent.java | 11 +++-- .../metastore/events/AddPartitionEvent.java | 16 ++++--- .../metastore/events/AddPrimaryKeyEvent.java | 10 +++-- .../events/AddUniqueConstraintEvent.java | 11 +++-- .../metastore/events/AlterIndexEvent.java | 11 +++-- .../metastore/events/AlterPartitionEvent.java | 10 ++--- .../metastore/events/AlterTableEvent.java | 11 ++--- .../metastore/events/ConfigChangeEvent.java | 12 ++--- .../metastore/events/CreateDatabaseEvent.java | 10 ++--- .../metastore/events/CreateFunctionEvent.java | 10 ++--- .../metastore/events/CreateTableEvent.java | 10 ++--- .../metastore/events/DropConstraintEvent.java | 10 +++-- .../metastore/events/DropDatabaseEvent.java | 10 ++--- .../metastore/events/DropFunctionEvent.java | 10 ++--- .../hive/metastore/events/DropIndexEvent.java | 10 ++--- .../metastore/events/DropPartitionEvent.java | 10 ++--- .../hive/metastore/events/DropTableEvent.java | 10 ++--- .../metastore/events/EventCleanerTask.java | 8 ++-- .../hive/metastore/events/InsertEvent.java | 41 +++++++++++------ .../hive/metastore/events/ListenerEvent.java | 14 +++--- .../events/LoadPartitionDoneEvent.java | 10 ++--- .../metastore/events/PreAddIndexEvent.java | 10 ++--- .../events/PreAddPartitionEvent.java | 14 +++--- .../metastore/events/PreAlterIndexEvent.java | 10 ++--- .../events/PreAlterPartitionEvent.java | 10 ++--- .../metastore/events/PreAlterTableEvent.java | 10 ++--- .../events/PreAuthorizationCallEvent.java | 10 ++--- .../events/PreCreateDatabaseEvent.java | 10 ++--- .../metastore/events/PreCreateTableEvent.java | 10 ++--- .../events/PreDropDatabaseEvent.java | 10 ++--- .../metastore/events/PreDropIndexEvent.java | 10 ++--- .../events/PreDropPartitionEvent.java | 12 ++--- .../metastore/events/PreDropTableEvent.java | 10 ++--- .../metastore/events/PreEventContext.java | 17 ++++--- .../events/PreLoadPartitionDoneEvent.java | 12 ++--- .../events/PreReadDatabaseEvent.java | 10 ++--- .../metastore/events/PreReadTableEvent.java | 10 ++--- 56 files changed, 349 insertions(+), 286 deletions(-) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java (82%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java (79%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java (87%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java (82%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java (84%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java (87%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java (85%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java (84%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java (83%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java (79%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java (86%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java (86%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java (90%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java (70%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java (94%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java (84%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java (88%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java (87%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java (82%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java (79%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java (84%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java (83%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java (80%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java (85%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java (81%) diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java index f7c2f446cff5..e7289a7c1ef8 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java @@ -36,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -54,6 +55,7 @@ import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.hive.hcatalog.messaging.MessageFactory; +import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,7 +184,7 @@ public void onCreateDatabase(CreateDatabaseEvent dbEvent) // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_ADD_DATABASE" if (dbEvent.getStatus()) { - String topicName = getTopicPrefix(dbEvent.getHandler().getHiveConf()); + String topicName = getTopicPrefix(dbEvent.getHandler().getConf()); send(messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()), topicName); } } @@ -193,7 +195,7 @@ public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_DROP_DATABASE" if (dbEvent.getStatus()) { - String topicName = getTopicPrefix(dbEvent.getHandler().getHiveConf()); + String topicName = getTopicPrefix(dbEvent.getHandler().getConf()); send(messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()), topicName); } } @@ -205,8 +207,8 @@ public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { // as "HCAT_EVENT = HCAT_ADD_TABLE" if (tableEvent.getStatus()) { Table tbl = tableEvent.getTable(); - HMSHandler handler = tableEvent.getHandler(); - HiveConf conf = handler.getHiveConf(); + IHMSHandler handler = tableEvent.getHandler(); + Configuration conf = handler.getConf(); Table newTbl; try { newTbl = handler.get_table_core(tbl.getDbName(), tbl.getTableName()) @@ -216,11 +218,7 @@ public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { getTopicPrefix(conf) + "." + newTbl.getDbName().toLowerCase() + "." + newTbl.getTableName().toLowerCase()); handler.alter_table(newTbl.getDbName(), newTbl.getTableName(), newTbl); - } catch (InvalidOperationException e) { - MetaException me = new MetaException(e.toString()); - me.initCause(e); - throw me; - } catch (NoSuchObjectException e) { + } catch (TException e) { MetaException me = new MetaException(e.toString()); me.initCause(e); throw me; @@ -258,7 +256,7 @@ public void onAlterTable(AlterTableEvent tableEvent) throws MetaException { } // I think this is wrong, the alter table statement should come on the table topic not the // DB topic - Alan. - String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + + String topicName = getTopicPrefix(tableEvent.getHandler().getConf()) + "." + after.getDbName().toLowerCase(); send(messageFactory.buildAlterTableMessage(before, after), topicName); } @@ -288,7 +286,7 @@ public void onDropTable(DropTableEvent tableEvent) throws MetaException { Table table = tableEvent.getTable(); // I think this is wrong, the drop table statement should come on the table topic not the // DB topic - Alan. - String topicName = getTopicPrefix(tableEvent.getHandler().getHiveConf()) + "." + table.getDbName().toLowerCase(); + String topicName = getTopicPrefix(tableEvent.getHandler().getConf()) + "." + table.getDbName().toLowerCase(); send(messageFactory.buildDropTableMessage(table), topicName); } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 42e476a8e50f..61e3ae696ea1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -509,13 +509,13 @@ public void testMetaConfSameHandler() throws Exception { closingClient.setMetaConf(metaConfKey, "[test pattern modified]"); ConfigChangeEvent event = (ConfigChangeEvent) DummyListener.getLastEvent(); int beforeCloseNotificationEventCounts = DummyListener.notifyList.size(); - HiveMetaStore.HMSHandler beforeHandler = event.getHandler(); + IHMSHandler beforeHandler = event.getHandler(); closingClient.close(); Thread.sleep(5 * 1000); event = (ConfigChangeEvent) DummyListener.getLastEvent(); int afterCloseNotificationEventCounts = DummyListener.notifyList.size(); - HiveMetaStore.HMSHandler afterHandler = event.getHandler(); + IHMSHandler afterHandler = event.getHandler(); // Meta-conf cleanup should trigger an event to listener assertNotSame(beforeCloseNotificationEventCounts, afterCloseNotificationEventCounts); // Both the handlers should be same diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java index 93637d43211d..6662fcd92f4c 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/DummyHiveMetastoreAuthorizationProvider.java @@ -24,7 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -201,7 +201,7 @@ private String debugPrivPrint(Privilege[] privileges) { } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { debugLog("DHMAP.setMetaStoreHandler"); } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java index 322834e85ef1..d4cf026906e5 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/InjectableDummyAuthenticator.java @@ -20,7 +20,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; @@ -91,7 +91,7 @@ public void setConf(Configuration config) { } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { hmap.setMetaStoreHandler(handler); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java index 67aa1440deb3..9115480fb299 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.io.HdfsUtils; -import org.apache.hadoop.hive.metastore.HiveMetaStore.ThreadLocalRawStore; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.shims.HadoopShims; @@ -58,7 +57,7 @@ public class FileMetadataManager { private static final Log LOG = LogFactory.getLog(FileMetadataManager.class); private static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); - private final ThreadLocalRawStore tlms; + private final RawStore tlms; private final ExecutorService threadPool; private final HiveConf conf; @@ -85,7 +84,7 @@ public Void call() throws Exception { } } - public FileMetadataManager(ThreadLocalRawStore tlms, HiveConf conf) { + public FileMetadataManager(RawStore tlms, HiveConf conf) { this.tlms = tlms; this.conf = conf; int numThreads = HiveConf.getIntVar(conf, ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS); @@ -128,7 +127,7 @@ private void cacheMetadata(FileMetadataExprType type, String location) } LOG.info("Caching file metadata for " + file + " (file ID " + fileId + ")"); file = HdfsUtils.getFileIdPath(fs, file, fileId); - tlms.getMS().getFileMetadataHandler(type).cacheFileMetadata(fileId, fs, file); + tlms.getFileMetadataHandler(type).cacheFileMetadata(fileId, fs, file); } } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index c53baac76966..f067899fbc57 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -219,17 +219,7 @@ public TTransport getTransport(TTransport trans) { } } - /** - * An ugly interface because everything about this file is ugly. RawStore is threadlocal so this - * thread-local disease propagates everywhere, and FileMetadataManager cannot just get a RawStore - * or handlers to use; it will need to have this method to make thread-local handlers and a - * thread-local RawStore. - */ - public interface ThreadLocalRawStore { - RawStore getMS() throws MetaException; - } - - public static class HMSHandler extends FacebookBase implements IHMSHandler, ThreadLocalRawStore { + public static class HMSHandler extends FacebookBase implements IHMSHandler { public static final Logger LOG = HiveMetaStore.LOG; private final HiveConf hiveConf; // stores datastore (jpox) properties, // right now they come from jpox.properties @@ -551,7 +541,7 @@ public void init() throws MetaException { cleaner.schedule(new DumpDirCleanerTask(hiveConf), cleanFreq, cleanFreq); } expressionProxy = PartFilterExprUtil.createExpressionProxy(hiveConf); - fileMetadataManager = new FileMetadataManager((ThreadLocalRawStore)this, hiveConf); + fileMetadataManager = new FileMetadataManager(this.getMS(), hiveConf); } private static String addPrefix(String s) { @@ -594,6 +584,7 @@ private Map getModifiedConf() { return modifiedConf; } + @Override public Warehouse getWh() { return wh; } @@ -1041,14 +1032,7 @@ public Database get_database(final String name) throws NoSuchObjectException, Me return db; } - /** - * Equivalent to get_database, but does not write to audit logs, or fire pre-event listners. - * Meant to be used for internal hive classes that don't use the thrift interface. - * @param name - * @return - * @throws NoSuchObjectException - * @throws MetaException - */ + @Override public Database get_database_core(final String name) throws NoSuchObjectException, MetaException { Database db = null; @@ -2414,16 +2398,7 @@ public List get_table_meta(String dbnames, String tblNames, List columns, } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { // no-op - HMSHander not needed by this impl } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java index 32d8b3eb663d..56fc9e46d253 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java @@ -27,6 +27,7 @@ import javax.security.auth.login.LoginException; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -36,7 +37,6 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; @@ -271,7 +271,7 @@ public void authorize(Table table, Partition part, List columns, } @Override - public void setMetaStoreHandler(HMSHandler handler) { + public void setMetaStoreHandler(IHMSHandler handler) { hive_db.setHandler(handler); this.wh = handler.getWh(); this.isRunFromMetaStore = true; diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java index 8fc83110d127..8b0d71c99d2f 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java @@ -1,4 +1,4 @@ -/** +/* * Autogenerated by Thrift Compiler (0.9.3) * * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING @@ -7,17 +7,13 @@ package org.apache.hadoop.hive.metastore.api; -import java.util.Map; -import java.util.HashMap; -import org.apache.thrift.TEnum; - public enum ClientCapability implements org.apache.thrift.TEnum { TEST_CAPABILITY(1), INSERT_ONLY_TABLES(2); private final int value; - private ClientCapability(int value) { + ClientCapability(int value) { this.value = value; } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java index eab290542f6e..53c631b1a9a3 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java @@ -18,10 +18,19 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +/** + * An interface wrapper for HMSHandler. This interface contains methods that need to be + * called by internal classes but that are not part of the thrift interface. + */ +@InterfaceAudience.Private public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable { void init() throws MetaException; @@ -31,4 +40,40 @@ public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable { * @return thread id */ int getThreadId(); + + /** + * Get a reference to the underlying RawStore. + * @return the RawStore instance. + * @throws MetaException if the creation of a new RawStore object is necessary but fails. + */ + RawStore getMS() throws MetaException; + + /** + * Get a reference to Hive's warehouse object (the class that does all the physical operations). + * @return Warehouse instance. + */ + Warehouse getWh(); + + /** + * Equivalent to get_database, but does not write to audit logs, or fire pre-event listners. + * Meant to be used for internal hive classes that don't use the thrift interface. + * @param name database name + * @return database object + * @throws NoSuchObjectException If the database does not exist. + * @throws MetaException If another error occurs. + */ + Database get_database_core(final String name) throws NoSuchObjectException, MetaException; + + /** + * Equivalent of get_table, but does not log audits and fire pre-event listener. + * Meant to be used for calls made by other hive classes, that are not using the + * thrift interface. + * @param dbname database name + * @param name table name + * @return Table object + * @throws NoSuchObjectException If the table does not exist. + * @throws MetaException If another error occurs. + */ + public Table get_table_core(final String dbname, final String name) throws MetaException, + NoSuchObjectException; } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 5933318e64b2..31dae375d0c4 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -485,6 +485,18 @@ public enum ConfVars { "javax.jdo.PersistenceManagerFactoryClass", "org.datanucleus.api.jdo.JDOPersistenceManagerFactory", "class implementing the jdo persistence"), + // Parameters for exporting metadata on table drop (requires the use of the) + // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener + METADATA_EXPORT_LOCATION("metastore.metadata.export.location", "hive.metadata.export.location", + "", + "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + + "it is the location to which the metadata will be exported. The default is an empty string, which results in the \n" + + "metadata being exported to the current user's home directory on HDFS."), + MOVE_EXPORTED_METADATA_TO_TRASH("metastore.metadata.move.exported.metadata.to.trash", + "hive.metadata.move.exported.metadata.to.trash", true, + "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + + "this setting determines if the metadata that is exported will subsequently be moved to the user's trash directory \n" + + "alongside the dropped table data. This ensures that the metadata will be cleaned up along with the dropped table data."), METRICS_ENABLED("metastore.metrics.enabled", "hive.metastore.metrics.enabled", false, "Enable metrics on the metastore."), METRICS_JSON_FILE_INTERVAL("metastore.metrics.file.frequency", diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java similarity index 82% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java index 1dc95889d13e..f9c951c59fde 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddForeignKeyEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,13 +20,17 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +@InterfaceStability.Stable +@InterfaceAudience.Public public class AddForeignKeyEvent extends ListenerEvent { private final List fks; - public AddForeignKeyEvent(List fks, boolean status, HMSHandler handler) { + public AddForeignKeyEvent(List fks, boolean status, IHMSHandler handler) { super(status, handler); this.fks = fks; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java similarity index 79% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java index 57fe5d1a1575..0d6cb5649c88 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class AddIndexEvent extends ListenerEvent { private final Index index; - public AddIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + public AddIndexEvent(Index index, boolean status, IHMSHandler handler) { super(status, handler); this.index = index; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java index c01083cd889a..6b4cf0b5eab3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddNotNullConstraintEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,13 +20,18 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +@InterfaceAudience.Public +@InterfaceStability.Stable public class AddNotNullConstraintEvent extends ListenerEvent { private final List nns; - public AddNotNullConstraintEvent(List nns, boolean status, HMSHandler handler) { + public AddNotNullConstraintEvent(List nns, boolean status, + IHMSHandler handler) { super(status, handler); this.nns = nns; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java index 85f5fb84dee1..d4542d7a5b3b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; @@ -37,21 +37,23 @@ public class AddPartitionEvent extends ListenerEvent { private final List partitions; private PartitionSpecProxy partitionSpecProxy; - public AddPartitionEvent(Table table, List partitions, boolean status, HMSHandler handler) { + public AddPartitionEvent(Table table, List partitions, boolean status, + IHMSHandler handler) { super(status, handler); this.table = table; this.partitions = partitions; this.partitionSpecProxy = null; } - public AddPartitionEvent(Table table, Partition partition, boolean status, HMSHandler handler) { + public AddPartitionEvent(Table table, Partition partition, boolean status, IHMSHandler handler) { this(table, Arrays.asList(partition), status, handler); } /** * Alternative constructor to use PartitionSpec APIs. */ - public AddPartitionEvent(Table table, PartitionSpecProxy partitionSpec, boolean status, HMSHandler handler) { + public AddPartitionEvent(Table table, PartitionSpecProxy partitionSpec, boolean status, + IHMSHandler handler) { super(status, handler); this.table = table; this.partitions = null; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java similarity index 82% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java index cb0f562c4169..de7978cfbd4a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddPrimaryKeyEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,14 +20,18 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +@InterfaceAudience.Public +@InterfaceStability.Stable public class AddPrimaryKeyEvent extends ListenerEvent { private final List pks; - public AddPrimaryKeyEvent(List pks, boolean status, HMSHandler handler) { + public AddPrimaryKeyEvent(List pks, boolean status, IHMSHandler handler) { super(status, handler); this.pks = pks; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java index c02a3099f3cd..b677ee64276c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AddUniqueConstraintEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,13 +20,18 @@ import java.util.List; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +@InterfaceAudience.Public +@InterfaceStability.Stable public class AddUniqueConstraintEvent extends ListenerEvent { private final List uks; - public AddUniqueConstraintEvent(List uks, boolean status, HMSHandler handler) { + public AddUniqueConstraintEvent(List uks, boolean status, + IHMSHandler handler) { super(status, handler); this.uks = uks; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java index 99e51f603991..2c72b836356e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -30,8 +30,7 @@ public class AlterIndexEvent extends ListenerEvent { private final Index newIndex; private final Index oldIndex; - public AlterIndexEvent(Index oldIndex, Index newIndex, boolean status, - HiveMetaStore.HMSHandler handler) { + public AlterIndexEvent(Index oldIndex, Index newIndex, boolean status, IHMSHandler handler) { super(status, handler); this.oldIndex = oldIndex; this.newIndex = newIndex; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java index 4025f9f5047a..09cad85334c1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -34,7 +34,7 @@ public class AlterPartitionEvent extends ListenerEvent { private final boolean isTruncateOp; public AlterPartitionEvent(Partition oldPart, Partition newPart, Table table, boolean isTruncateOp, - boolean status, HMSHandler handler) { + boolean status, IHMSHandler handler) { super(status, handler); this.oldPart = oldPart; this.newPart = newPart; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java similarity index 85% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java index eb9d94fcc327..2e3f6f00ee89 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -32,7 +32,8 @@ public class AlterTableEvent extends ListenerEvent { private final Table oldTable; private final boolean isTruncateOp; - public AlterTableEvent (Table oldTable, Table newTable, boolean isTruncateOp, boolean status, HMSHandler handler) { + public AlterTableEvent (Table oldTable, Table newTable, boolean isTruncateOp, boolean status, + IHMSHandler handler) { super (status, handler); this.oldTable = oldTable; this.newTable = newTable; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java index 9d23dc8c13fb..090d175f1fb9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ConfigChangeEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; @InterfaceAudience.Public @InterfaceStability.Stable @@ -30,8 +30,8 @@ public class ConfigChangeEvent extends ListenerEvent { private final String oldValue; private final String newValue; - public ConfigChangeEvent(HiveMetaStore.HMSHandler handler, String key, - String oldValue, String newValue) { + public ConfigChangeEvent(IHMSHandler handler, String key, + String oldValue, String newValue) { super(true, handler); this.key = key; this.oldValue = oldValue; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java index e853aec13841..e2c3ee30852c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class CreateDatabaseEvent extends ListenerEvent { private final Database db; - public CreateDatabaseEvent (Database db, boolean status, HMSHandler handler) { + public CreateDatabaseEvent (Database db, boolean status, IHMSHandler handler) { super (status, handler); this.db = db; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java index 0c6ab17d6d45..c1403ec09bf5 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateFunctionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Function; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class CreateFunctionEvent extends ListenerEvent { private final Function function; - public CreateFunctionEvent (Function function, boolean status, HMSHandler handler) { + public CreateFunctionEvent (Function function, boolean status, IHMSHandler handler) { super (status, handler); this.function = function; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java index 09a40bb98ebb..4f5e887ac27f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class CreateTableEvent extends ListenerEvent { private final Table table; - public CreateTableEvent (Table table, boolean status, HMSHandler handler) { + public CreateTableEvent (Table table, boolean status, IHMSHandler handler) { super (status, handler); this.table = table; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java similarity index 83% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java index 53968633b833..4c5918f1c36b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,15 +18,19 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +@InterfaceAudience.Public +@InterfaceStability.Stable public class DropConstraintEvent extends ListenerEvent { private final String dbName; private final String tableName; private final String constraintName; public DropConstraintEvent(String dbName, String tableName, String constraintName, - boolean status, HMSHandler handler) { + boolean status, IHMSHandler handler) { super(status, handler); this.dbName = dbName; this.tableName = tableName; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java index 92e95a6f21da..94fe26420637 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class DropDatabaseEvent extends ListenerEvent { private final Database db; - public DropDatabaseEvent(Database db, boolean status, HMSHandler handler) { + public DropDatabaseEvent(Database db, boolean status, IHMSHandler handler) { super (status, handler); this.db = db; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java index 52f04ca46ea0..e8728a5904a9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropFunctionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Function; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class DropFunctionEvent extends ListenerEvent { private final Function function; - public DropFunctionEvent(Function function, boolean status, HMSHandler handler) { + public DropFunctionEvent(Function function, boolean status, IHMSHandler handler) { super(status, handler); this.function = function; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java similarity index 79% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java index 9b7505535330..4d6983469bc1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class DropIndexEvent extends ListenerEvent { private final Index index; - public DropIndexEvent(Index index, boolean status, HiveMetaStore.HMSHandler handler) { + public DropIndexEvent(Index index, boolean status, IHMSHandler handler) { super(status, handler); this.index = index; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java similarity index 86% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java index d8f2ea968c14..dce9df3a1126 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -36,7 +36,7 @@ public class DropPartitionEvent extends ListenerEvent { private final boolean deleteData; public DropPartitionEvent (Table table, - Partition partition, boolean status, boolean deleteData, HMSHandler handler) { + Partition partition, boolean status, boolean deleteData, IHMSHandler handler) { super (status, handler); this.table = table; this.partitions = Collections.singletonList(partition); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java similarity index 86% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java index 2f2ea9dc0136..9152232aaeb0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -30,7 +30,7 @@ public class DropTableEvent extends ListenerEvent { private final Table table; private final boolean deleteData; - public DropTableEvent(Table table, boolean status, boolean deleteData, HMSHandler handler) { + public DropTableEvent(Table table, boolean status, boolean deleteData, IHMSHandler handler) { super(status, handler); this.table = table; // In HiveMetaStore, the deleteData flag indicates whether DFS data should be diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java similarity index 90% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java index 7f99f18828f0..230c0d3b16a9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/EventCleanerTask.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,17 +20,17 @@ import java.util.TimerTask; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.RawStore; public class EventCleanerTask extends TimerTask{ public static final Logger LOG = LoggerFactory.getLogger(EventCleanerTask.class); - private final HMSHandler handler; + private final IHMSHandler handler; - public EventCleanerTask(HMSHandler handler) { + public EventCleanerTask(IHMSHandler handler) { super(); this.handler = handler; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java similarity index 70% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java index d451122a59da..b963f78c066c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/InsertEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,16 +18,20 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import com.google.common.collect.Lists; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.ClientCapabilities; +import org.apache.hadoop.hive.metastore.api.ClientCapability; import org.apache.hadoop.hive.metastore.api.GetTableRequest; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.thrift.TException; import java.util.ArrayList; import java.util.List; @@ -40,7 +44,7 @@ public class InsertEvent extends ListenerEvent { private final Partition ptnObj; private final boolean replace; private final List files; - private List fileChecksums = new ArrayList(); + private List fileChecksums = new ArrayList<>(); /** * @@ -52,17 +56,28 @@ public class InsertEvent extends ListenerEvent { * @param handler handler that is firing the event */ public InsertEvent(String db, String table, List partVals, - InsertEventRequestData insertData, boolean status, HMSHandler handler) throws MetaException, + InsertEventRequestData insertData, boolean status, IHMSHandler handler) throws MetaException, NoSuchObjectException { super(status, handler); GetTableRequest req = new GetTableRequest(db, table); - req.setCapabilities(HiveMetaStoreClient.TEST_VERSION); - this.tableObj = handler.get_table_req(req).getTable(); - if (partVals != null) { - this.ptnObj = handler.get_partition(db, table, partVals); - } else { - this.ptnObj = null; + // TODO MS-SPLIT Switch this back once HiveMetaStoreClient is moved. + //req.setCapabilities(HiveMetaStoreClient.TEST_VERSION); + req.setCapabilities(new ClientCapabilities( + Lists.newArrayList(ClientCapability.TEST_CAPABILITY))); + try { + this.tableObj = handler.get_table_req(req).getTable(); + if (partVals != null) { + this.ptnObj = handler.get_partition(db, table, partVals); + } else { + this.ptnObj = null; + } + } catch (NoSuchObjectException e) { + // This is to mimic previous behavior where NoSuchObjectException was thrown through this + // method. + throw e; + } catch (TException e) { + throw MetaStoreUtils.newMetaException(e); } // If replace flag is not set by caller, then by default set it to true to maintain backward compatibility diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java similarity index 94% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java index cd66f10ba0cb..f4f46d8ea4ae 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/ListenerEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import javax.annotation.concurrent.NotThreadSafe; @@ -42,7 +42,7 @@ public abstract class ListenerEvent { * status of the event, whether event was successful or not. */ private final boolean status; - private final HMSHandler handler; + private final IHMSHandler handler; /** * Key/value parameters used by listeners to store notifications results @@ -67,7 +67,7 @@ public abstract class ListenerEvent { // Properties passed by the client, to be used in execution hooks. private EnvironmentContext environmentContext = null; - public ListenerEvent(boolean status, HMSHandler handler) { + public ListenerEvent(boolean status, IHMSHandler handler) { super(); this.status = status; this.handler = handler; @@ -102,7 +102,7 @@ public EnvironmentContext getEnvironmentContext() { /** * @return the handler */ - public HMSHandler getHandler() { + public IHMSHandler getHandler() { return handler; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java index b56b3bb3d43b..3128a8f6a410 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,9 +20,9 @@ import java.util.Map; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -34,7 +34,7 @@ public class LoadPartitionDoneEvent extends ListenerEvent { private final Map partSpec; public LoadPartitionDoneEvent(boolean status, Table table, - Map partSpec, HMSHandler handler) { + Map partSpec, IHMSHandler handler) { super(status, handler); this.table = table; this.partSpec = partSpec; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java index 94a8836e7fb3..d9a53f85b712 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class PreAddIndexEvent extends PreEventContext { private final Index table; - public PreAddIndexEvent(Index table, HiveMetaStore.HMSHandler handler) { + public PreAddIndexEvent(Index table, IHMSHandler handler) { super(PreEventType.ADD_INDEX, handler); this.table = table; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java similarity index 88% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java index 99105f3e56df..b5c4607d14a8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAddPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; @@ -37,21 +37,21 @@ public class PreAddPartitionEvent extends PreEventContext { private final List partitions; private PartitionSpecProxy partitionSpecProxy; - public PreAddPartitionEvent (Table table, List partitions, HMSHandler handler) { + public PreAddPartitionEvent (Table table, List partitions, IHMSHandler handler) { super(PreEventType.ADD_PARTITION, handler); this.table = table; this.partitions = partitions; this.partitionSpecProxy = null; } - public PreAddPartitionEvent(Table table, Partition partition, HMSHandler handler) { + public PreAddPartitionEvent(Table table, Partition partition, IHMSHandler handler) { this(table, Arrays.asList(partition), handler); } /** * Alternative constructor, using */ - public PreAddPartitionEvent(Table table, PartitionSpecProxy partitionSpecProxy, HMSHandler handler) { + public PreAddPartitionEvent(Table table, PartitionSpecProxy partitionSpecProxy, IHMSHandler handler) { this(table, (List)null, handler); this.partitionSpecProxy = partitionSpecProxy; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java index a2b7662b2d92..2e9d9d77dd44 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -30,7 +30,7 @@ public class PreAlterIndexEvent extends PreEventContext { private final Index newIndex; private final Index oldIndex; - public PreAlterIndexEvent(Index oldIndex, Index newIndex, HiveMetaStore.HMSHandler handler) { + public PreAlterIndexEvent(Index oldIndex, Index newIndex, IHMSHandler handler) { super(PreEventType.ALTER_INDEX, handler); this.oldIndex = oldIndex; this.newIndex = newIndex; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java index 5f2969917988..a269563e6021 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,9 +20,9 @@ import java.util.List; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; @InterfaceAudience.Public @@ -35,7 +35,7 @@ public class PreAlterPartitionEvent extends PreEventContext { private final Partition newPart; public PreAlterPartitionEvent(String dbName, String tableName, List oldPartVals, - Partition newPart, HMSHandler handler) { + Partition newPart, IHMSHandler handler) { super(PreEventType.ALTER_PARTITION, handler); this.dbName = dbName; this.tableName = tableName; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java similarity index 82% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java index 4e82ad1aec48..31db0e9cb496 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -31,7 +31,7 @@ public class PreAlterTableEvent extends PreEventContext { private final Table newTable; private final Table oldTable; - public PreAlterTableEvent (Table oldTable, Table newTable, HMSHandler handler) { + public PreAlterTableEvent (Table oldTable, Table newTable, IHMSHandler handler) { super (PreEventType.ALTER_TABLE, handler); this.oldTable = oldTable; this.newTable = newTable; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java similarity index 79% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java index 54f6dee12a80..1205fc935fc3 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAuthorizationCallEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,15 +18,15 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; @InterfaceAudience.Public @InterfaceStability.Stable public class PreAuthorizationCallEvent extends PreEventContext { - public PreAuthorizationCallEvent (HMSHandler handler) { + public PreAuthorizationCallEvent (IHMSHandler handler) { super(PreEventType.AUTHORIZATION_API_CALL, handler); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java index a5f87e11caed..86ff1a077b92 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class PreCreateDatabaseEvent extends PreEventContext { private final Database db; - public PreCreateDatabaseEvent (Database db, HMSHandler handler) { + public PreCreateDatabaseEvent (Database db, IHMSHandler handler) { super (PreEventType.CREATE_DATABASE, handler); this.db = db; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java index 799ad973478c..bd3aa3d8ce47 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreCreateTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class PreCreateTableEvent extends PreEventContext { private final Table table; - public PreCreateTableEvent(Table table, HMSHandler handler) { + public PreCreateTableEvent(Table table, IHMSHandler handler) { super(PreEventType.CREATE_TABLE, handler); this.table = table; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java index adcca6f0624d..f5ebbcf24aa4 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class PreDropDatabaseEvent extends PreEventContext { private final Database db; - public PreDropDatabaseEvent(Database db, HMSHandler handler) { + public PreDropDatabaseEvent(Database db, IHMSHandler handler) { super(PreEventType.DROP_DATABASE, handler); this.db = db; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java index c7fd44b2347b..ad570db23dd4 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropIndexEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Index; @InterfaceAudience.Public @@ -29,7 +29,7 @@ public class PreDropIndexEvent extends PreEventContext { private final Index index; - public PreDropIndexEvent(Index index, HiveMetaStore.HMSHandler handler) { + public PreDropIndexEvent(Index index, IHMSHandler handler) { super(PreEventType.DROP_INDEX, handler); this.index = index; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java similarity index 84% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java index e890dedc9c12..a0daf3177c24 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; @@ -35,8 +35,8 @@ public class PreDropPartitionEvent extends PreEventContext { private final Table table; private final boolean deleteData; - public PreDropPartitionEvent (Table table, - Partition partition, boolean deleteData, HMSHandler handler) { + public PreDropPartitionEvent (Table table, Partition partition, boolean deleteData, + IHMSHandler handler) { super (PreEventType.DROP_PARTITION, handler); this.partitions = Collections.singletonList(partition); this.table = table; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java similarity index 83% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java index 6c7f169a9029..f06fdcd03418 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreDropTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; @InterfaceAudience.Public @@ -30,7 +30,7 @@ public class PreDropTableEvent extends PreEventContext { private final Table table; private final boolean deleteData; - public PreDropTableEvent(Table table, boolean deleteData, HMSHandler handler) { + public PreDropTableEvent(Table table, boolean deleteData, IHMSHandler handler) { super(PreEventType.DROP_TABLE, handler); this.table = table; // In HiveMetaStore, the deleteData flag indicates whether DFS data should be diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java similarity index 80% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java index 0bd2ea640030..a3f4836fa965 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreEventContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; - +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; /** * Base class for all the events which are defined for metastore. @@ -30,7 +29,7 @@ @InterfaceStability.Stable public abstract class PreEventContext { - public static enum PreEventType { + public enum PreEventType { CREATE_TABLE, DROP_TABLE, ALTER_TABLE, @@ -50,9 +49,9 @@ public static enum PreEventType { } private final PreEventType eventType; - private final HMSHandler handler; + private final IHMSHandler handler; - public PreEventContext(PreEventType eventType, HMSHandler handler) { + public PreEventContext(PreEventType eventType, IHMSHandler handler) { this.eventType = eventType; this.handler = handler; } @@ -67,7 +66,7 @@ public PreEventType getEventType() { /** * @return the handler */ - public HMSHandler getHandler() { + public IHMSHandler getHandler() { return handler; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java similarity index 85% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java index 82b72eda8ac6..999ec31f6587 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreLoadPartitionDoneEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,11 +18,11 @@ package org.apache.hadoop.hive.metastore.events; -import java.util.Map; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import java.util.Map; @InterfaceAudience.Public @InterfaceStability.Stable @@ -33,7 +33,7 @@ public class PreLoadPartitionDoneEvent extends PreEventContext { private final Map partSpec; public PreLoadPartitionDoneEvent(String dbName, String tableName, - Map partSpec, HMSHandler handler) { + Map partSpec, IHMSHandler handler) { super(PreEventType.LOAD_PARTITION_DONE, handler); this.dbName = dbName; this.tableName = tableName; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java index f223960449f7..88f3855bbc26 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; /** @@ -31,7 +31,7 @@ public class PreReadDatabaseEvent extends PreEventContext { private final Database db; - public PreReadDatabaseEvent(Database db, HMSHandler handler) { + public PreReadDatabaseEvent(Database db, IHMSHandler handler) { super(PreEventType.READ_DATABASE, handler); this.db = db; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java index e9d353af289e..beec72bc1221 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreReadTableEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Table; /** @@ -32,7 +32,7 @@ public class PreReadTableEvent extends PreEventContext { private final Table table; - public PreReadTableEvent(Table table, HMSHandler handler) { + public PreReadTableEvent(Table table, IHMSHandler handler) { super(PreEventType.READ_TABLE, handler); this.table = table; } From ad0d217dd261591cfa0d67ada87674cec4ddaecd Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Thu, 31 Aug 2017 11:00:44 -0700 Subject: [PATCH 05/10] Moved the listeners. Also moved one event that either I missed or that snuck in after I moved the others. --- .../hive/metastore/AcidEventListener.java | 23 ++++++++++--------- .../hive/metastore/HMSMetricsListener.java | 12 +++++----- .../metastore/MetaStoreEventListener.java | 6 ++--- .../MetaStoreEventListenerConstants.java | 2 +- .../hive/metastore/MetaStoreInitContext.java | 2 +- .../hive/metastore/MetaStoreInitListener.java | 2 +- .../metastore/MetaStoreListenerNotifier.java | 6 ++--- .../metastore/MetaStorePreEventListener.java | 6 ++--- .../metastore/SessionPropertiesListener.java | 7 +++--- .../TransactionalMetaStoreEventListener.java | 4 ++-- .../TransactionalValidationListener.java | 2 +- .../events/PreAlterDatabaseEvent.java | 6 ++--- 12 files changed, 40 insertions(+), 38 deletions(-) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/AcidEventListener.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java (87%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (97%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java (93%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java (81%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java (96%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java (99%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java (89%) diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java index 71ad916cb3dc..1512ffbfe123 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/AcidEventListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,9 +19,10 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; @@ -35,11 +36,11 @@ public class AcidEventListener extends MetaStoreEventListener { private TxnStore txnHandler; - private HiveConf hiveConf; + private Configuration conf; public AcidEventListener(Configuration configuration) { super(configuration); - hiveConf = (HiveConf) configuration; + conf = configuration; } @Override @@ -69,24 +70,24 @@ public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaExcep } private TxnStore getTxnHandler() { - boolean hackOn = HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST) || - HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEZ_TEST); + boolean hackOn = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) || + MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEZ_TEST); String origTxnMgr = null; boolean origConcurrency = false; // Since TxnUtils.getTxnStore calls TxnHandler.setConf -> checkQFileTestHack -> TxnDbUtil.setConfValues, // which may change the values of below two entries, we need to avoid pulluting the original values if (hackOn) { - origTxnMgr = hiveConf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER); - origConcurrency = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); + origTxnMgr = MetastoreConf.getVar(conf, ConfVars.HIVE_TXN_MANAGER); + origConcurrency = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY); } - txnHandler = TxnUtils.getTxnStore(hiveConf); + txnHandler = TxnUtils.getTxnStore(conf); // Set them back if (hackOn) { - hiveConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, origTxnMgr); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, origConcurrency); + MetastoreConf.setVar(conf, ConfVars.HIVE_TXN_MANAGER, origTxnMgr); + MetastoreConf.setBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY, origConcurrency); } return txnHandler; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java similarity index 87% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java index a2ad06f31ed6..1ee6d97b9a1d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HMSMetricsListener.java @@ -54,37 +54,37 @@ public HMSMetricsListener(Configuration config) { @Override public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException { - HiveMetaStore.HMSHandler.databaseCount.incrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).incrementAndGet(); createdDatabases.inc(); } @Override public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { - HiveMetaStore.HMSHandler.databaseCount.decrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).decrementAndGet(); deletedDatabases.inc(); } @Override public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { - HiveMetaStore.HMSHandler.tableCount.incrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).incrementAndGet(); createdTables.inc(); } @Override public void onDropTable(DropTableEvent tableEvent) throws MetaException { - HiveMetaStore.HMSHandler.tableCount.decrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).decrementAndGet(); deletedTables.inc(); } @Override public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException { - HiveMetaStore.HMSHandler.partCount.decrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).decrementAndGet(); deletedParts.inc(); } @Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { - HiveMetaStore.HMSHandler.partCount.incrementAndGet(); + Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).incrementAndGet(); createdParts.inc(); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java similarity index 97% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java index 5a72082e9522..fc4f4d725363 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,10 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; import org.apache.hadoop.hive.metastore.events.AddIndexEvent; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java index 79de79d11f61..0b3de494ea51 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListenerConstants.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java index 2ce621e1cd31..0968f42f314b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java index c0f62bc56584..12130eacca1f 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java index 331621ed2935..f6e25c6c0aff 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; @@ -53,7 +53,7 @@ /** * This class is used to notify a list of listeners about specific MetaStore events. */ -@Private +@InterfaceAudience.Private public class MetaStoreListenerNotifier { private interface EventNotifier { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java similarity index 93% rename from metastore/src/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java index bca6a31f5402..a757dff8060b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStorePreEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,10 +18,10 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java similarity index 81% rename from metastore/src/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java index ee9667809fbd..214e353401c1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/SessionPropertiesListener.java @@ -19,8 +19,8 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import java.util.concurrent.TimeUnit; @@ -36,9 +36,10 @@ public SessionPropertiesListener(Configuration configuration) { @Override public void onConfigChange(ConfigChangeEvent changeEvent) throws MetaException { - if (changeEvent.getKey().equals(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.varname)) { + if (changeEvent.getKey().equals(MetastoreConf.ConfVars.CLIENT_SOCKET_TIMEOUT.varname) || + changeEvent.getKey().equals(MetastoreConf.ConfVars.CLIENT_SOCKET_TIMEOUT.hiveName)) { // TODO: this only applies to current thread, so it's not useful at all. - Deadline.resetTimeout(HiveConf.toTime(changeEvent.getNewValue(), TimeUnit.SECONDS, + Deadline.resetTimeout(MetastoreConf.convertTimeStr(changeEvent.getNewValue(), TimeUnit.SECONDS, TimeUnit.MILLISECONDS)); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java similarity index 96% rename from metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java index d7f1c644878d..b6a67acdf336 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -30,7 +30,7 @@ public abstract class TransactionalMetaStoreEventListener extends MetaStoreEvent /** * Constructor * - * @param config + * @param config configuration object */ public TransactionalMetaStoreEventListener(Configuration config) { super(config); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java similarity index 99% rename from metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 29d8da8082a8..8f68e8100f0a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java similarity index 89% rename from metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java index 62e2674cd309..6ac1a4eff8a2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/PreAlterDatabaseEvent.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,14 +18,14 @@ package org.apache.hadoop.hive.metastore.events; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.api.Database; public class PreAlterDatabaseEvent extends PreEventContext { private final Database oldDB, newDB; - public PreAlterDatabaseEvent(Database oldDB, Database newDB, HMSHandler handler) { + public PreAlterDatabaseEvent(Database oldDB, Database newDB, IHMSHandler handler) { super (PreEventType.ALTER_DATABASE, handler); this.oldDB = oldDB; this.newDB = newDB; From 7d39f13dae76f25582c3a270b99d4b3552af793a Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Thu, 31 Aug 2017 13:02:13 -0700 Subject: [PATCH 06/10] Moved AlterHandler and HiveAlterHandler. Also did a partial cleanup of MetaStoreUtils. Need to do more. --- .../hadoop/hive/metastore/HiveMetaStore.java | 3 +- .../hadoop/hive/metastore/MetaStoreUtils.java | 189 +-------- .../hive/metastore/TestMetaStoreUtils.java | 43 -- .../hadoop/hive/metastore/AlterHandler.java | 15 +- .../hive/metastore/HiveAlterHandler.java | 59 ++- .../hadoop/hive/metastore/IHMSHandler.java | 12 +- .../hive/metastore/utils/FileUtils.java | 16 + .../hive/metastore/utils/MetaStoreUtils.java | 383 +++++++++++++++++- .../hive/metastore/TestHiveAlterHandler.java | 0 .../metastore/utils}/TestMetaStoreUtils.java | 34 +- 10 files changed, 485 insertions(+), 269 deletions(-) delete mode 100644 metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/AlterHandler.java (94%) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (95%) rename {metastore/src/test => standalone-metastore/src/test/java}/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java (100%) rename {itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore => standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils}/TestMetaStoreUtils.java (50%) diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index f067899fbc57..2ef9476c3e26 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -461,7 +461,8 @@ public HiveConf getHiveConf() { } } - List getTransactionalListeners() { + @Override + public List getTransactionalListeners() { return transactionalListeners; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index ef097acb0f7a..a4917898e4dc 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -22,13 +22,8 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.math.BigDecimal; -import java.math.BigInteger; import java.net.URL; import java.net.URLClassLoader; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,30 +35,16 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.base.Predicates; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.shims.ShimLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -81,15 +62,12 @@ import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; -import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMerger; import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMergerFactory; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; @@ -194,7 +172,7 @@ static public void recursiveDelete(File f) throws IOException { * @param partParams * @return True if the passed Parameters Map contains values for all "Fast Stats". */ - public static boolean containsAllFastStats(Map partParams) { + private static boolean containsAllFastStats(Map partParams) { for (String stat : StatsSetupConst.fastStats) { if (!partParams.containsKey(stat)) { return false; @@ -203,12 +181,12 @@ public static boolean containsAllFastStats(Map partParams) { return true; } - public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, boolean madeDir, EnvironmentContext environmentContext) throws MetaException { return updateTableStatsFast(db, tbl, wh, madeDir, false, environmentContext); } - public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + private static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { if (tbl.getPartitionKeysSize() == 0) { // Update stats only when unpartitioned @@ -288,70 +266,12 @@ public static void populateQuickStats(FileStatus[] fileStatus, Map params = part.getParameters(); boolean updated = false; @@ -407,7 +327,8 @@ public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionItera return updated; } - static void updateBasicState(EnvironmentContext environmentContext, Map params) { + private static void updateBasicState(EnvironmentContext environmentContext, Map + params) { if (params == null) { return; } @@ -612,7 +533,7 @@ static public boolean validateName(String name, Configuration conf) { /* * At the Metadata level there are no restrictions on Column Names. */ - public static final boolean validateColumnName(String name) { + public static boolean validateColumnName(String name) { return true; } @@ -629,53 +550,6 @@ static public String validateTblColumns(List cols) { return null; } - static void throwExceptionIfIncompatibleColTypeChange( - List oldCols, List newCols) - throws InvalidOperationException { - - List incompatibleCols = new ArrayList(); - int maxCols = Math.min(oldCols.size(), newCols.size()); - for (int i = 0; i < maxCols; i++) { - if (!areColTypesCompatible(oldCols.get(i).getType(), newCols.get(i).getType())) { - incompatibleCols.add(newCols.get(i).getName()); - } - } - if (!incompatibleCols.isEmpty()) { - throw new InvalidOperationException( - "The following columns have types incompatible with the existing " + - "columns in their respective positions :\n" + - StringUtils.join(incompatibleCols, ',') - ); - } - } - - static boolean areSameColumns(List oldCols, List newCols) { - return ListUtils.isEqualList(oldCols, newCols); - } - - /* - * This method is to check if the new column list includes all the old columns with same name and - * type. The column comment does not count. - */ - static boolean columnsIncludedByNameType(List oldCols, List newCols) { - if (oldCols.size() > newCols.size()) { - return false; - } - - Map columnNameTypePairMap = new HashMap(newCols.size()); - for (FieldSchema newCol : newCols) { - columnNameTypePairMap.put(newCol.getName().toLowerCase(), newCol.getType()); - } - for (final FieldSchema oldCol : oldCols) { - if (!columnNameTypePairMap.containsKey(oldCol.getName()) - || !columnNameTypePairMap.get(oldCol.getName()).equalsIgnoreCase(oldCol.getType())) { - return false; - } - } - - return true; - } - /** * @return true if oldType and newType are compatible. * Two types are compatible if we have internal functions to cast one to another. @@ -821,7 +695,7 @@ public static void setSerdeParam(SerDeInfo sdi, Properties schema, org.apache.hadoop.hive.serde.serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); } - static Set hiveThriftTypeMap; //for validation + private static Set hiveThriftTypeMap; //for validation static { hiveThriftTypeMap = new HashSet(); hiveThriftTypeMap.addAll(serdeConstants.PrimitiveTypes); @@ -1675,31 +1549,6 @@ public java.lang.String apply(@Nullable java.lang.String string) { } }; - /** - * We have aneed to sanity-check the map before conversion from persisted objects to - * metadata thrift objects because null values in maps will cause a NPE if we send - * across thrift. Pruning is appropriate for most cases except for databases such as - * Oracle where Empty strings are stored as nulls, in which case we need to handle that. - * See HIVE-8485 for motivations for this. - */ - public static Map trimMapNulls( - Map dnMap, boolean retrieveMapNullsAsEmptyStrings){ - if (dnMap == null){ - return null; - } - // Must be deterministic order map - see HIVE-8707 - // => we use Maps.newLinkedHashMap instead of Maps.newHashMap - if (retrieveMapNullsAsEmptyStrings) { - // convert any nulls present in map values to empty strings - this is done in the case - // of backing dbs like oracle which persist empty strings as nulls. - return Maps.newLinkedHashMap(Maps.transformValues(dnMap, transFormNullsToEmptyString)); - } else { - // prune any nulls present in map values - this is the typical case. - return Maps.newLinkedHashMap(Maps.filterValues(dnMap, Predicates.notNull())); - } - } - - /** * Create a URL from a string representing a path to a local file. * The path string can be just a path, or can start with file:/, file:/// @@ -1780,20 +1629,6 @@ public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) csNew.setStatsObj(list); } - /** - * convert Exception to MetaException, which sets the cause to such exception - * @param errorMessage the error message for this MetaException - * @param e cause of the exception - * @return the MetaException with the specified exception as the cause - */ - public static MetaException newMetaException(String errorMessage, Exception e) { - MetaException metaException = new MetaException(errorMessage); - if (e != null) { - metaException.initCause(e); - } - return metaException; - } - public static List getColumnNames(List schema) { List cols = new ArrayList<>(schema.size()); for (FieldSchema fs : schema) { diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java deleted file mode 100644 index e5c8a40e3977..000000000000 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; - -public class TestMetaStoreUtils { - - @Test - public void testcolumnsIncludedByNameType() { - FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); - FieldSchema col1a = new FieldSchema("col1", "string", "col1 but with a different comment"); - FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); - FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1a))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col2, col1))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3))); - Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1))); - Assert.assertFalse(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1))); - } -} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java similarity index 94% rename from metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index a3d322f3456d..fc0b4d7d08d5 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,7 +27,6 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; /** * Interface for Alter Table and Alter Partition code @@ -36,7 +35,7 @@ public interface AlterHandler extends Configurable { /** * @deprecated As of release 2.2.0. Replaced by {@link #alterTable(RawStore, Warehouse, String, - * String, Table, EnvironmentContext, HMSHandler)} + * String, Table, EnvironmentContext, IHMSHandler)} * * handles alter table, the changes could be cascaded to partitions if applicable * @@ -84,11 +83,11 @@ void alterTable(RawStore msdb, Warehouse wh, String dbname, */ void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newTable, EnvironmentContext envContext, - HMSHandler handler) throws InvalidOperationException, MetaException; + IHMSHandler handler) throws InvalidOperationException, MetaException; /** * @deprecated As of release 2.2.0. Replaced by {@link #alterPartition(RawStore, Warehouse, String, - * String, List, Partition, EnvironmentContext, HMSHandler)} + * String, List, Partition, EnvironmentContext, IHMSHandler)} * * handles alter partition * @@ -139,12 +138,12 @@ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, */ Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part, EnvironmentContext environmentContext, - HMSHandler handler) + IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; /** * @deprecated As of release 2.2.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, - * String, List, EnvironmentContext, HMSHandler)} + * String, List, EnvironmentContext, IHMSHandler)} * * handles alter partitions * @@ -191,6 +190,6 @@ List alterPartitions(final RawStore msdb, Warehouse wh, */ List alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List new_parts, - EnvironmentContext environmentContext,HMSHandler handler) + EnvironmentContext environmentContext,IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; } \ No newline at end of file diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java similarity index 95% rename from metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 65551ad169d8..c800d4f8e042 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,20 +21,21 @@ import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.messaging.EventMessage; +import org.apache.hadoop.hive.metastore.utils.FileUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; @@ -49,10 +50,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hive.common.util.HiveStringUtils; import java.io.IOException; import java.net.URI; @@ -93,7 +91,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, @Override public void alterTable(RawStore msdb, Warehouse wh, String dbname, String name, Table newt, EnvironmentContext environmentContext, - HMSHandler handler) throws InvalidOperationException, MetaException { + IHMSHandler handler) throws InvalidOperationException, MetaException { name = name.toLowerCase(); dbname = dbname.toLowerCase(); @@ -117,13 +115,13 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, } Path srcPath = null; - FileSystem srcFs = null; + FileSystem srcFs; Path destPath = null; FileSystem destFs = null; boolean success = false; boolean dataWasMoved = false; - Table oldt = null; + Table oldt; List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -132,7 +130,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, try { boolean rename = false; boolean isPartitionedTable = false; - List parts = null; + List parts; // check if table with the new name already exists if (!newTblName.equals(name) || !newDbName.equals(dbname)) { @@ -154,9 +152,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, isPartitionedTable = true; } - if (HiveConf.getBoolVar(hiveConf, - HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, - false)) { + if (MetastoreConf.getBoolVar(hiveConf, + MetastoreConf.ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES)) { // Throws InvalidOperationException if the new column types are not // compatible with the current column types. checkColTypeChangeCompatible(oldt.getSd().getCols(), newt.getSd().getCols()); @@ -239,7 +236,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, // also the location field in partition parts = msdb.getPartitions(dbname, name, -1); - Map columnStatsNeedUpdated = new HashMap(); + Map columnStatsNeedUpdated = new HashMap<>(); for (Partition part : parts) { String oldPartLoc = part.getSd().getLocation(); if (dataWasMoved && oldPartLoc.contains(oldTblLocPath)) { @@ -393,10 +390,10 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String @Override public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List part_vals, final Partition new_part, - EnvironmentContext environmentContext, HMSHandler handler) + EnvironmentContext environmentContext, IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { boolean success = false; - Partition oldPart = null; + Partition oldPart; List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -459,11 +456,11 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String } //rename partition - String oldPartLoc = null; - String newPartLoc = null; + String oldPartLoc; + String newPartLoc; Path srcPath = null; Path destPath = null; - FileSystem srcFs = null; + FileSystem srcFs; FileSystem destFs = null; boolean dataWasMoved = false; try { @@ -615,10 +612,10 @@ public List alterPartitions(final RawStore msdb, Warehouse wh, final @Override public List alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, final String name, final List new_parts, EnvironmentContext environmentContext, - HMSHandler handler) + IHMSHandler handler) throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { - List oldParts = new ArrayList(); - List> partValsList = new ArrayList>(); + List oldParts = new ArrayList<>(); + List> partValsList = new ArrayList<>(); List transactionalListeners = null; if (handler != null) { transactionalListeners = handler.getTransactionalListeners(); @@ -733,14 +730,14 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable) throws MetaException, InvalidObjectException { String dbName = oldTable.getDbName().toLowerCase(); - String tableName = HiveStringUtils.normalizeIdentifier(oldTable.getTableName()); + String tableName = org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(oldTable.getTableName()); String newDbName = newTable.getDbName().toLowerCase(); - String newTableName = HiveStringUtils.normalizeIdentifier(newTable.getTableName()); + String newTableName = org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(newTable.getTableName()); try { List oldCols = oldTable.getSd().getCols(); List newCols = newTable.getSd().getCols(); - List newStatsObjs = new ArrayList(); + List newStatsObjs = new ArrayList<>(); ColumnStatistics colStats = null; boolean updateColumnStats = true; @@ -752,7 +749,7 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa } if (updateColumnStats) { - List oldColNames = new ArrayList(oldCols.size()); + List oldColNames = new ArrayList<>(oldCols.size()); for (FieldSchema oldCol : oldCols) { oldColNames.add(oldCol.getName()); } @@ -764,7 +761,7 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa } else { List statsObjs = colStats.getStatsObj(); if (statsObjs != null) { - List deletedCols = new ArrayList(); + List deletedCols = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { boolean found = false; for (FieldSchema newCol : newCols) { @@ -815,7 +812,7 @@ private ColumnStatistics updateOrGetPartitionColumnStats( ColumnStatistics newPartsColStats = null; try { List newCols = part.getSd() == null ? - new ArrayList() : part.getSd().getCols(); + new ArrayList<>() : part.getSd().getCols(); String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals); String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname) @@ -825,7 +822,7 @@ private ColumnStatistics updateOrGetPartitionColumnStats( if (!rename && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) { return newPartsColStats; } - List oldColNames = new ArrayList(oldCols.size()); + List oldColNames = new ArrayList<>(oldCols.size()); for (FieldSchema oldCol : oldCols) { oldColNames.add(oldCol.getName()); } @@ -834,9 +831,9 @@ private ColumnStatistics updateOrGetPartitionColumnStats( oldPartNames, oldColNames); assert (partsColStats.size() <= 1); for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop - List newStatsObjs = new ArrayList(); + List newStatsObjs = new ArrayList<>(); List statsObjs = partColStats.getStatsObj(); - List deletedCols = new ArrayList(); + List deletedCols = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { boolean found =false; for (FieldSchema newCol : newCols) { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java index 53c631b1a9a3..85bdc4d96918 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java @@ -26,6 +26,8 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +import java.util.List; + /** * An interface wrapper for HMSHandler. This interface contains methods that need to be * called by internal classes but that are not part of the thrift interface. @@ -55,7 +57,7 @@ public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable { Warehouse getWh(); /** - * Equivalent to get_database, but does not write to audit logs, or fire pre-event listners. + * Equivalent to get_database, but does not write to audit logs, or fire pre-event listeners. * Meant to be used for internal hive classes that don't use the thrift interface. * @param name database name * @return database object @@ -74,6 +76,12 @@ public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable { * @throws NoSuchObjectException If the table does not exist. * @throws MetaException If another error occurs. */ - public Table get_table_core(final String dbname, final String name) throws MetaException, + Table get_table_core(final String dbname, final String name) throws MetaException, NoSuchObjectException; + + /** + * Get a list of all transactional listeners. + * @return list of listeners. + */ + List getTransactionalListeners(); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java index da0ee80a9e8d..2dac899d919c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java @@ -363,4 +363,20 @@ public static String makePartName(List partCols, List vals, } return name.toString(); } + + /** + * Determine if two objects reference the same file system. + * @param fs1 first file system + * @param fs2 second file system + * @return return true if both file system arguments point to same file system + */ + public static boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) { + //When file system cache is disabled, you get different FileSystem objects + // for same file system, so '==' can't be used in such cases + //FileSystem api doesn't have a .equals() function implemented, so using + //the uri for comparison. FileSystem already uses uri+Configuration for + //equality in its CACHE . + //Once equality has been added in HDFS-9159, we should make use of it + return fs1.getUri().equals(fs2.getUri()); + } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 77790adfff3c..d4b2bd536c3d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -21,13 +21,23 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.commons.collections.ListUtils; import org.apache.commons.lang.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Decimal; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -35,6 +45,7 @@ import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,9 +59,11 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; @@ -60,6 +73,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.regex.Matcher; import java.util.regex.Pattern; public class MetaStoreUtils { @@ -73,9 +87,20 @@ protected DateFormat initialValue() { return val; } }; + // Indicates a type was derived from the deserializer rather than Hive's metadata. + public static final String TYPE_FROM_DESERIALIZER = ""; + private static final Charset ENCODING = StandardCharsets.UTF_8; private static final Logger LOG = LoggerFactory.getLogger(MetaStoreUtils.class); + // Right now we only support one special character '/'. + // More special characters can be added accordingly in the future. + // NOTE: + // If the following array is updated, please also be sure to update the + // configuration parameter documentation + // HIVE_SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES in HiveConf as well. + private static final char[] specialCharactersInTableNames = new char[] { '/' }; + /** * Catches exceptions that can't be handled and bundles them to MetaException * @@ -139,8 +164,9 @@ public java.lang.String apply(@Nullable java.lang.String string) { return org.apache.commons.lang.StringUtils.defaultString(string); } }; + /** - * We have aneed to sanity-check the map before conversion from persisted objects to + * We have a need to sanity-check the map before conversion from persisted objects to * metadata thrift objects because null values in maps will cause a NPE if we send * across thrift. Pruning is appropriate for most cases except for databases such as * Oracle where Empty strings are stored as nulls, in which case we need to handle that. @@ -359,4 +385,359 @@ public static List getColumnNamesForTable(Table table) { } return colNames; } + + /** + * validateName + * + * Checks the name conforms to our standars which are: "[a-zA-z_0-9]+". checks + * this is just characters and numbers and _ + * + * @param name + * the name to validate + * @param conf + * hive configuration + * @return true or false depending on conformance + * if it doesn't match the pattern. + */ + public static boolean validateName(String name, Configuration conf) { + Pattern tpat = null; + String allowedCharacters = "\\w_"; + if (conf != null + && MetastoreConf.getBoolVar(conf, + MetastoreConf.ConfVars.SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES)) { + for (Character c : specialCharactersInTableNames) { + allowedCharacters += c; + } + } + tpat = Pattern.compile("[" + allowedCharacters + "]+"); + Matcher m = tpat.matcher(name); + return m.matches(); + } + + /* + * At the Metadata level there are no restrictions on Column Names. + */ + public static boolean validateColumnName(String name) { + return true; + } + + static public String validateTblColumns(List cols) { + for (FieldSchema fieldSchema : cols) { + // skip this, as validateColumnName always returns true + /* + if (!validateColumnName(fieldSchema.getName())) { + return "name: " + fieldSchema.getName(); + } + */ + String typeError = validateColumnType(fieldSchema.getType()); + if (typeError != null) { + return typeError; + } + } + return null; + } + + private static String validateColumnType(String type) { + if (type.equals(TYPE_FROM_DESERIALIZER)) return null; + int last = 0; + boolean lastAlphaDigit = isValidTypeChar(type.charAt(last)); + for (int i = 1; i <= type.length(); i++) { + if (i == type.length() + || isValidTypeChar(type.charAt(i)) != lastAlphaDigit) { + String token = type.substring(last, i); + last = i; + if (!ColumnType.AllTypes.contains(token)) { + return "type: " + type; + } + break; + } + } + return null; + } + + private static boolean isValidTypeChar(char c) { + return Character.isLetterOrDigit(c) || c == '_'; + } + + /** + * Determines whether a table is an external table. + * + * @param table table of interest + * + * @return true if external + */ + public static boolean isExternalTable(Table table) { + if (table == null) { + return false; + } + Map params = table.getParameters(); + if (params == null) { + return false; + } + + return "TRUE".equalsIgnoreCase(params.get("EXTERNAL")); + } + + // check if stats need to be (re)calculated + public static boolean requireCalStats(Configuration hiveConf, Partition oldPart, + Partition newPart, Table tbl, EnvironmentContext environmentContext) { + + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( + StatsSetupConst.DO_NOT_UPDATE_STATS))) { + return false; + } + + if (MetaStoreUtils.isView(tbl)) { + return false; + } + + if (oldPart == null && newPart == null) { + return true; + } + + // requires to calculate stats if new partition doesn't have it + if ((newPart == null) || (newPart.getParameters() == null) + || !containsAllFastStats(newPart.getParameters())) { + return true; + } + + if (environmentContext != null && environmentContext.isSetProperties()) { + String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED); + // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats: + // USER: alter table .. update statistics + // TASK: from some sql operation which could collect and compute stats + if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) { + return true; + } + } + + // requires to calculate stats if new and old have different fast stats + return !isFastStatsSame(oldPart, newPart); + } + + public static boolean isView(Table table) { + if (table == null) { + return false; + } + return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType()); + } + + /** + * @param partParams + * @return True if the passed Parameters Map contains values for all "Fast Stats". + */ + private static boolean containsAllFastStats(Map partParams) { + for (String stat : StatsSetupConst.fastStats) { + if (!partParams.containsKey(stat)) { + return false; + } + } + return true; + } + + public static boolean isFastStatsSame(Partition oldPart, Partition newPart) { + // requires to calculate stats if new and old have different fast stats + if ((oldPart != null) && (oldPart.getParameters() != null)) { + for (String stat : StatsSetupConst.fastStats) { + if (oldPart.getParameters().containsKey(stat)) { + Long oldStat = Long.parseLong(oldPart.getParameters().get(stat)); + Long newStat = Long.parseLong(newPart.getParameters().get(stat)); + if (!oldStat.equals(newStat)) { + return false; + } + } else { + return false; + } + } + return true; + } + return false; + } + + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + boolean madeDir, EnvironmentContext environmentContext) throws MetaException { + return updateTableStatsFast(db, tbl, wh, madeDir, false, environmentContext); + } + + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh, + boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + if (tbl.getPartitionKeysSize() == 0) { + // Update stats only when unpartitioned + FileStatus[] fileStatuses = wh.getFileStatusesForUnpartitionedTable(db, tbl); + return updateTableStatsFast(tbl, fileStatuses, madeDir, forceRecompute, environmentContext); + } else { + return false; + } + } + + /** + * Updates the numFiles and totalSize parameters for the passed Table by querying + * the warehouse if the passed Table does not already have values for these parameters. + * @param tbl + * @param fileStatus + * @param newDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Table already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updateTableStatsFast(Table tbl, FileStatus[] fileStatus, boolean newDir, + boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + + Map params = tbl.getParameters(); + + if ((params!=null) && params.containsKey(StatsSetupConst.DO_NOT_UPDATE_STATS)){ + boolean doNotUpdateStats = Boolean.valueOf(params.get(StatsSetupConst.DO_NOT_UPDATE_STATS)); + params.remove(StatsSetupConst.DO_NOT_UPDATE_STATS); + tbl.setParameters(params); // to make sure we remove this marker property + if (doNotUpdateStats){ + return false; + } + } + + boolean updated = false; + if (forceRecompute || + params == null || + !containsAllFastStats(params)) { + if (params == null) { + params = new HashMap(); + } + if (!newDir) { + // The table location already exists and may contain data. + // Let's try to populate those stats that don't require full scan. + LOG.info("Updating table stats fast for " + tbl.getTableName()); + populateQuickStats(fileStatus, params); + LOG.info("Updated size of table " + tbl.getTableName() +" to "+ params.get(StatsSetupConst.TOTAL_SIZE)); + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); + } + } + tbl.setParameters(params); + updated = true; + } + return updated; + } + + public static void populateQuickStats(FileStatus[] fileStatus, Map params) { + int numFiles = 0; + long tableSize = 0L; + for (FileStatus status : fileStatus) { + // don't take directories into account for quick stats + if (!status.isDir()) { + tableSize += status.getLen(); + numFiles += 1; + } + } + params.put(StatsSetupConst.NUM_FILES, Integer.toString(numFiles)); + params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(tableSize)); + } + + public static boolean areSameColumns(List oldCols, List newCols) { + return ListUtils.isEqualList(oldCols, newCols); + } + + public static void updateBasicState(EnvironmentContext environmentContext, Map + params) { + if (params == null) { + return; + } + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TASK.equals(environmentContext.getProperties().get( + StatsSetupConst.STATS_GENERATED))) { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE); + } + } + + public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, EnvironmentContext environmentContext) + throws MetaException { + return updatePartitionStatsFast(part, wh, false, false, environmentContext); + } + + public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, boolean madeDir, EnvironmentContext environmentContext) + throws MetaException { + return updatePartitionStatsFast(part, wh, madeDir, false, environmentContext); + } + + /** + * Updates the numFiles and totalSize parameters for the passed Partition by querying + * the warehouse if the passed Partition does not already have values for these parameters. + * @param part + * @param wh + * @param madeDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Partition already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, + boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + return updatePartitionStatsFast(new PartitionSpecProxy.SimplePartitionWrapperIterator(part), + wh, madeDir, forceRecompute, environmentContext); + } + /** + * Updates the numFiles and totalSize parameters for the passed Partition by querying + * the warehouse if the passed Partition does not already have values for these parameters. + * @param part + * @param wh + * @param madeDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Partition already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part, Warehouse wh, + boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException { + Map params = part.getParameters(); + boolean updated = false; + if (forceRecompute || + params == null || + !containsAllFastStats(params)) { + if (params == null) { + params = new HashMap(); + } + if (!madeDir) { + // The partition location already existed and may contain data. Lets try to + // populate those statistics that don't require a full scan of the data. + LOG.warn("Updating partition stats fast for: " + part.getTableName()); + FileStatus[] fileStatus = wh.getFileStatusesForLocation(part.getLocation()); + populateQuickStats(fileStatus, params); + LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE)); + updateBasicState(environmentContext, params); + } + part.setParameters(params); + updated = true; + } + return updated; + } + + /* + * This method is to check if the new column list includes all the old columns with same name and + * type. The column comment does not count. + */ + public static boolean columnsIncludedByNameType(List oldCols, + List newCols) { + if (oldCols.size() > newCols.size()) { + return false; + } + + Map columnNameTypePairMap = new HashMap(newCols.size()); + for (FieldSchema newCol : newCols) { + columnNameTypePairMap.put(newCol.getName().toLowerCase(), newCol.getType()); + } + for (final FieldSchema oldCol : oldCols) { + if (!columnNameTypePairMap.containsKey(oldCol.getName()) + || !columnNameTypePairMap.get(oldCol.getName()).equalsIgnoreCase(oldCol.getType())) { + return false; + } + } + + return true; + } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java similarity index 100% rename from metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java similarity index 50% rename from itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java rename to standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java index c4dd97ec6c12..32ad63a728d9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,17 +16,23 @@ * limitations under the License. */ -package org.apache.hadoop.hive.metastore; +package org.apache.hadoop.hive.metastore.utils; -import junit.framework.TestCase; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.junit.Assert; +import org.junit.Test; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; -public class TestMetaStoreUtils extends TestCase { +import static org.junit.Assert.assertEquals; +public class TestMetaStoreUtils { + + @Test public void testTrimMapNullsXform() throws Exception { - Map m = new HashMap(); + Map m = new HashMap<>(); m.put("akey","aval"); m.put("blank",""); m.put("null",null); @@ -41,8 +47,9 @@ public void testTrimMapNullsXform() throws Exception { assertEquals("",xformed.get("null")); } + @Test public void testTrimMapNullsPrune() throws Exception { - Map m = new HashMap(); + Map m = new HashMap<>(); m.put("akey","aval"); m.put("blank",""); m.put("null",null); @@ -57,6 +64,21 @@ public void testTrimMapNullsPrune() throws Exception { assert(!pruned.containsValue(null)); } + @Test + public void testcolumnsIncludedByNameType() { + FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); + FieldSchema col1a = new FieldSchema("col1", "string", "col1 but with a different comment"); + FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); + FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1a))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col2, col1))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3))); + Assert.assertTrue(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1))); + Assert.assertFalse(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1))); + } + } From c90002b083c5bb749b5f1566e77c53b7e14fd2f4 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Fri, 1 Sep 2017 12:11:50 -0700 Subject: [PATCH 07/10] Converted extenders of HouseKeeperService to instead implement RunnableConfigurable and use the general ThreadPool. --- .../hcatalog/streaming/TestStreaming.java | 16 +--- .../hadoop/hive/metastore/HiveMetaStore.java | 43 ++++----- .../hive/metastore/HouseKeeperService.java | 45 --------- .../ql/txn/AcidCompactionHistoryService.java | 82 ---------------- .../hive/ql/txn/AcidHouseKeeperService.java | 82 ---------------- .../hive/ql/txn/AcidWriteSetService.java | 78 --------------- .../txn/compactor/HouseKeeperServiceBase.java | 96 ------------------- .../hadoop/hive/ql/TestTxnCommands.java | 14 +-- .../hadoop/hive/ql/TestTxnCommands2.java | 30 +----- .../hive/ql/lockmgr/TestDbTxnManager.java | 25 ++--- .../hive/ql/lockmgr/TestDbTxnManager2.java | 12 ++- .../txn/AcidCompactionHistoryService.java | 61 ++++++++++++ .../metastore/txn/AcidHouseKeeperService.java | 62 ++++++++++++ .../metastore/txn/AcidWriteSetService.java | 60 ++++++++++++ .../hadoop/hive/metastore/txn/TxnDbUtil.java | 6 ++ 15 files changed, 240 insertions(+), 472 deletions(-) delete mode 100644 metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java delete mode 100644 ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java create mode 100644 standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java create mode 100644 standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java create mode 100644 standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java index 3b5066f0f374..49aad392d8d8 100644 --- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java +++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.TxnState; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; @@ -77,7 +78,6 @@ import org.apache.hadoop.hive.ql.io.orc.RecordReader; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.ql.txn.compactor.Worker; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -740,11 +740,8 @@ public void testTimeOutReaper() throws Exception { //ensure txn timesout conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.MILLISECONDS); AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService(); - houseKeeperService.start(conf); - while(houseKeeperService.getIsAliveCounter() <= Integer.MIN_VALUE) { - Thread.sleep(100);//make sure it has run at least once - } - houseKeeperService.stop(); + houseKeeperService.setConf(conf); + houseKeeperService.run(); try { //should fail because the TransactionBatch timed out txnBatch.commit(); @@ -757,12 +754,7 @@ public void testTimeOutReaper() throws Exception { txnBatch.beginNextTransaction(); txnBatch.commit(); txnBatch.beginNextTransaction(); - int lastCount = houseKeeperService.getIsAliveCounter(); - houseKeeperService.start(conf); - while(houseKeeperService.getIsAliveCounter() <= lastCount) { - Thread.sleep(100);//make sure it has run at least once - } - houseKeeperService.stop(); + houseKeeperService.run(); try { //should fail because the TransactionBatch timed out txnBatch.commit(); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 2ef9476c3e26..16cf9d703077 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -132,7 +132,10 @@ import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; +import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; +import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.serde2.Deserializer; @@ -7922,33 +7925,29 @@ private static void startHouseKeeperService(HiveConf conf) throws Exception { if(!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_INITIATOR_ON)) { return; } - startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService")); - startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService")); - startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidWriteSetService")); ThreadPool.initialize(conf); - RunnableConfigurable rc = new AcidOpenTxnsCounterService(); - rc.setConf(conf); - ThreadPool.getPool().scheduleAtFixedRate(rc, 100, MetastoreConf.getTimeVar(conf, - MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, TimeUnit.MILLISECONDS), - TimeUnit.MILLISECONDS); - + startOneHouseKeeperService(new AcidHouseKeeperService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TIMEDOUT_TXN_REAPER_INTERVAL, + TimeUnit.MILLISECONDS)); + startOneHouseKeeperService(new AcidOpenTxnsCounterService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, + TimeUnit.MILLISECONDS)); + startOneHouseKeeperService(new AcidCompactionHistoryService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, + TimeUnit.MILLISECONDS)); + startOneHouseKeeperService(new AcidWriteSetService(), conf, + MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.WRITE_SET_REAPER_INTERVAL, + TimeUnit.MILLISECONDS)); } - private static void startHouseKeeperService(HiveConf conf, Class c) throws Exception { - //todo: when metastore adds orderly-shutdown logic, houseKeeper.stop() - //should be called form it - HouseKeeperService houseKeeper = (HouseKeeperService)c.newInstance(); - try { - houseKeeper.start(conf); - } - catch (Exception ex) { - LOG.error("Failed to start {}" , houseKeeper.getClass() + - ". The system will not handle {} " , houseKeeper.getServiceDescription(), - ". Root Cause: ", ex); - } + + private static void startOneHouseKeeperService(RunnableConfigurable rc, Configuration conf, + long interval) { + rc.setConf(conf); + ThreadPool.getPool().scheduleAtFixedRate(rc, 0, interval, TimeUnit.MILLISECONDS); } - public static Map createHandlerMap() { + static Map createHandlerMap() { Map fmHandlers = new HashMap<>(); for (FileMetadataExprType v : FileMetadataExprType.values()) { switch (v) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java deleted file mode 100644 index 539ace00675b..000000000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.conf.HiveConf; - -/** - * Runs arbitrary background logic inside the metastore service. - */ -@InterfaceAudience.LimitedPrivate({"Hive"}) -@InterfaceStability.Evolving -public interface HouseKeeperService { - public void start(HiveConf hiveConf) throws Exception; - /** - * Should perform orderly shutdown - */ - public void stop(); - /** - * Returns short description of services this module provides. - */ - public String getServiceDescription(); - - /** - * This is incremented each time the service is performed. Can be useful to - * check if serivce is still alive. - */ - public int getIsAliveCounter(); -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java deleted file mode 100644 index 5d9e7bea9052..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidCompactionHistoryService.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.txn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Purges obsolete items from compaction history data - */ -public class AcidCompactionHistoryService extends HouseKeeperServiceBase { - private static final Logger LOG = LoggerFactory.getLogger(AcidCompactionHistoryService.class); - - @Override - protected long getStartDelayMs() { - return 0; - } - @Override - protected long getIntervalMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, TimeUnit.MILLISECONDS); - } - @Override - protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) { - return new ObsoleteEntryReaper(hiveConf, isAliveCounter); - } - - @Override - public String getServiceDescription() { - return "Removes obsolete entries from Compaction History"; - } - - private static final class ObsoleteEntryReaper implements Runnable { - private final TxnStore txnHandler; - private final AtomicInteger isAliveCounter; - private ObsoleteEntryReaper(HiveConf hiveConf, AtomicInteger isAliveCounter) { - txnHandler = TxnUtils.getTxnStore(hiveConf); - this.isAliveCounter = isAliveCounter; - } - - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); - long startTime = System.currentTimeMillis(); - txnHandler.purgeCompactionHistory(); - int count = isAliveCounter.incrementAndGet(); - LOG.info("History reaper reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds. isAliveCounter=" + count); - } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } - finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java deleted file mode 100644 index 13b10de5885d..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.txn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Performs background tasks for Transaction management in Hive. - * Runs inside Hive Metastore Service. - */ -public class AcidHouseKeeperService extends HouseKeeperServiceBase { - private static final Logger LOG = LoggerFactory.getLogger(AcidHouseKeeperService.class); - - @Override - protected long getStartDelayMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, TimeUnit.MILLISECONDS); - } - @Override - protected long getIntervalMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_INTERVAL, TimeUnit.MILLISECONDS); - } - @Override - protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) { - return new TimedoutTxnReaper(hiveConf, isAliveCounter); - } - - @Override - public String getServiceDescription() { - return "Abort expired transactions"; - } - - private static final class TimedoutTxnReaper implements Runnable { - private final TxnStore txnHandler; - private final AtomicInteger isAliveCounter; - private TimedoutTxnReaper(HiveConf hiveConf, AtomicInteger isAliveCounter) { - txnHandler = TxnUtils.getTxnStore(hiveConf); - this.isAliveCounter = isAliveCounter; - } - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); - long startTime = System.currentTimeMillis(); - txnHandler.performTimeOuts(); - int count = isAliveCounter.incrementAndGet(); - LOG.info("timeout reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds. isAliveCounter=" + count); - } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } - finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java deleted file mode 100644 index 2b8f8154abdd..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidWriteSetService.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.txn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Periodically cleans WriteSet tracking information used in Transaction management - */ -public class AcidWriteSetService extends HouseKeeperServiceBase { - private static final Logger LOG = LoggerFactory.getLogger(AcidWriteSetService.class); - @Override - protected long getStartDelayMs() { - return 0; - } - @Override - protected long getIntervalMs() { - return hiveConf.getTimeVar(HiveConf.ConfVars.WRITE_SET_REAPER_INTERVAL, TimeUnit.MILLISECONDS); - } - @Override - protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) { - return new WriteSetReaper(hiveConf, isAliveCounter); - } - @Override - public String getServiceDescription() { - return "Periodically cleans obsolete WriteSet tracking information used in Transaction management"; - } - private static final class WriteSetReaper implements Runnable { - private final TxnStore txnHandler; - private final AtomicInteger isAliveCounter; - private WriteSetReaper(HiveConf hiveConf, AtomicInteger isAliveCounter) { - txnHandler = TxnUtils.getTxnStore(hiveConf); - this.isAliveCounter = isAliveCounter; - } - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); - long startTime = System.currentTimeMillis(); - txnHandler.performWriteSetGC(); - int count = isAliveCounter.incrementAndGet(); - LOG.info("cleaner ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds. isAliveCounter=" + count); - } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } - finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java deleted file mode 100644 index 0aa160cb3998..000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/HouseKeeperServiceBase.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */package org.apache.hadoop.hive.ql.txn.compactor; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HouseKeeperService; -import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; -import org.apache.hadoop.hive.ql.lockmgr.TxnManagerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -public abstract class HouseKeeperServiceBase implements HouseKeeperService { - private static final Logger LOG = LoggerFactory.getLogger(HouseKeeperServiceBase.class); - private ScheduledExecutorService pool = null; - protected final AtomicInteger isAliveCounter = new AtomicInteger(Integer.MIN_VALUE); - protected HiveConf hiveConf; - - @Override - public void start(HiveConf hiveConf) throws Exception { - this.hiveConf = hiveConf; - HiveTxnManager mgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(hiveConf); - if(!mgr.supportsAcid()) { - LOG.info(this.getClass().getName() + " not started since " + - mgr.getClass().getName() + " does not support Acid."); - return;//there are no transactions in this case - } - pool = Executors.newScheduledThreadPool(1, new ThreadFactory() { - private final AtomicInteger threadCounter = new AtomicInteger(); - @Override - public Thread newThread(Runnable r) { - Thread t = - new Thread(r, HouseKeeperServiceBase.this.getClass().getName() + "-" - + threadCounter.getAndIncrement()); - t.setDaemon(true); - return t; - } - }); - - TimeUnit tu = TimeUnit.MILLISECONDS; - pool.scheduleAtFixedRate(getScheduedAction(hiveConf, isAliveCounter), getStartDelayMs(), - getIntervalMs(), tu); - LOG.info("Started " + this.getClass().getName() + " with delay/interval = " + getStartDelayMs() + "/" + - getIntervalMs() + " " + tu); - } - - @Override - public void stop() { - if(pool != null && !pool.isShutdown()) { - pool.shutdown(); - } - pool = null; - } - - /** - * This is used for testing only. Each time the housekeeper runs, counter is incremented by 1. - * Starts with {@link java.lang.Integer#MIN_VALUE} - */ - @Override - public int getIsAliveCounter() { - return isAliveCounter.get(); - } - - /** - * Delay in millis before first run of the task of this service. - */ - protected abstract long getStartDelayMs(); - /** - * Determines how frequently the service is running its task. - */ - protected abstract long getIntervalMs(); - - /** - * The actual task implementation. Must increment the counter on each iteration. - */ - protected abstract Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter); -} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 39d6b2b41464..034d5660d2ee 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -17,9 +17,9 @@ */ package org.apache.hadoop.hive.ql; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.metastore.api.LockType; @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.TxnState; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -36,7 +37,6 @@ import org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager2; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.junit.After; import org.junit.Assert; import org.junit.Ignore; @@ -337,11 +337,11 @@ public void testTimeOutReaper() throws Exception { runStatementOnDriver("start transaction"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 5"); //make sure currently running txn is considered aborted by housekeeper - hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS); hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 2, TimeUnit.MILLISECONDS); - AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService(); + RunnableConfigurable houseKeeperService = new AcidHouseKeeperService(); + houseKeeperService.setConf(hiveConf); //this will abort the txn - TestTxnCommands2.runHouseKeeperService(houseKeeperService, hiveConf); + houseKeeperService.run(); //this should fail because txn aborted due to timeout CommandProcessorResponse cpr = runStatementOnDriverNegative("delete from " + Table.ACIDTBL + " where a = 5"); Assert.assertTrue("Actual: " + cpr.getErrorMessage(), cpr.getErrorMessage().contains("Transaction manager has aborted the transaction txnid:1")); @@ -380,14 +380,14 @@ public void testTimeOutReaper() throws Exception { ShowLocksResponse slr = txnHandler.showLocks(new ShowLocksRequest()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); pause(750); - TestTxnCommands2.runHouseKeeperService(houseKeeperService, hiveConf); + houseKeeperService.run(); pause(750); slr = txnHandler.showLocks(new ShowLocksRequest()); Assert.assertEquals("Unexpected lock count: " + slr, 1, slr.getLocks().size()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); pause(750); - TestTxnCommands2.runHouseKeeperService(houseKeeperService, hiveConf); + houseKeeperService.run(); slr = txnHandler.showLocks(new ShowLocksRequest()); Assert.assertEquals("Unexpected lock count: " + slr, 1, slr.getLocks().size()); TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 3737b6a4d5c8..2faf098f6a4e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HouseKeeperService; import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; import org.apache.hadoop.hive.metastore.api.CompactionRequest; import org.apache.hadoop.hive.metastore.api.CompactionType; @@ -46,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; +import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService; import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; import org.apache.hadoop.hive.ql.txn.compactor.Cleaner; import org.apache.hadoop.hive.ql.txn.compactor.Initiator; @@ -1041,7 +1040,8 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS); AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService(); - runHouseKeeperService(compactionHistoryService, hiveConf);//should not remove anything from history + compactionHistoryService.setConf(hiveConf); + compactionHistoryService.run(); checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MAJOR)); @@ -1054,7 +1054,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb numAttemptedCompactions++; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions + 2,0,0,0,0,numFailedCompactions + 2 + numAttemptedCompactions), countCompacts(txnHandler)); - runHouseKeeperService(compactionHistoryService, hiveConf);//should remove history so that we have + compactionHistoryService.run(); //COMPACTOR_HISTORY_RETENTION_FAILED failed compacts left (and no other since we only have failed ones here) checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), @@ -1078,7 +1078,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)+ 1), countCompacts(txnHandler)); runCleaner(hiveConf); // transition to Success state - runHouseKeeperService(compactionHistoryService, hiveConf);//should not purge anything as all items within retention sizes + compactionHistoryService.run(); checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,1,0, @@ -1198,26 +1198,6 @@ public static void runCleaner(HiveConf hiveConf) throws MetaException { t.run(); } - public static void runHouseKeeperService(HouseKeeperService houseKeeperService, HiveConf conf) throws Exception { - int lastCount = houseKeeperService.getIsAliveCounter(); - houseKeeperService.start(conf); - int maxIter = 10; - int iterCount = 0; - while(houseKeeperService.getIsAliveCounter() <= lastCount) { - if(iterCount++ >= maxIter) { - //prevent test hangs - throw new IllegalStateException("HouseKeeper didn't run after " + (iterCount - 1) + " waits"); - } - try { - Thread.sleep(100);//make sure it has run at least once - } - catch(InterruptedException ex) { - //... - } - } - houseKeeperService.stop(); - } - /** * HIVE-12352 has details * @throws Exception diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java index e46e65be42c3..406bdea96aae 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java @@ -19,12 +19,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.ThreadPool; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement; import org.apache.hadoop.hive.metastore.api.TxnState; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -38,7 +40,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -202,24 +203,9 @@ public void testWriteDynamicPartition() throws Exception { * aborts timed out transactions */ private void runReaper() throws Exception { - int lastCount = houseKeeperService.getIsAliveCounter(); - houseKeeperService.start(conf); - int maxIter = 10; - int iterCount = 0; - while(houseKeeperService.getIsAliveCounter() <= lastCount) { - if(iterCount++ >= maxIter) { - //prevent test hangs - throw new IllegalStateException("Reaper didn't run after " + iterCount + " waits"); - } - try { - Thread.sleep(100);//make sure it has run at least once - } - catch(InterruptedException ex) { - //... - } - } - houseKeeperService.stop(); + houseKeeperService.run(); } + @Test public void testExceptions() throws Exception { addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT); @@ -491,13 +477,14 @@ public void setUp() throws Exception { conf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS); conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 10, TimeUnit.SECONDS); houseKeeperService = new AcidHouseKeeperService(); + houseKeeperService.setConf(conf); } @After public void tearDown() throws Exception { - if(houseKeeperService != null) houseKeeperService.stop(); if (txnMgr != null) txnMgr.closeTxnManager(); TxnDbUtil.cleanDb(conf); + ThreadPool.shutdown(); } private static class MockQueryPlan extends QueryPlan { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index 15045d694b77..3c172b936f21 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -19,12 +19,13 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions; import org.apache.hadoop.hive.metastore.api.DataOperationType; +import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.TestTxnCommands2; -import org.apache.hadoop.hive.ql.txn.AcidWriteSetService; import org.junit.After; import org.junit.Assert; import org.apache.hadoop.hive.common.FileUtils; @@ -1036,7 +1037,8 @@ public void testWriteSetTracking4() throws Exception { 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); AcidWriteSetService houseKeeper = new AcidWriteSetService(); - TestTxnCommands2.runHouseKeeperService(houseKeeper, conf); + houseKeeper.setConf(conf); + houseKeeper.run(); //since T3 overlaps with Long Running (still open) GC does nothing Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));//no rows match @@ -1050,7 +1052,7 @@ public void testWriteSetTracking4() throws Exception { locks = getLocks(txnMgr); Assert.assertEquals("Unexpected lock count", 0, locks.size()); - TestTxnCommands2.runHouseKeeperService(houseKeeper, conf); + houseKeeper.run(); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } /** @@ -1120,7 +1122,9 @@ public void testWriteSetTracking6() throws Exception { Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); txnMgr.commitTxn(); - TestTxnCommands2.runHouseKeeperService(new AcidWriteSetService(), conf); + RunnableConfigurable writeSetService = new AcidWriteSetService(); + writeSetService.setConf(conf); + writeSetService.run(); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET")); } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java new file mode 100644 index 000000000000..5cd043da4400 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Purges obsolete items from compaction history data + */ +public class AcidCompactionHistoryService implements RunnableConfigurable { + private static final Logger LOG = LoggerFactory.getLogger(AcidCompactionHistoryService.class); + + private Configuration conf; + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + TxnStore.MutexAPI.LockHandle handle = null; + try { + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); + long startTime = System.currentTimeMillis(); + txnHandler.purgeCompactionHistory(); + LOG.debug("History reaper reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + + "seconds."); + } catch(Throwable t) { + LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } finally { + if(handle != null) { + handle.releaseLocks(); + } + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java new file mode 100644 index 000000000000..0e7358ab380d --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Performs background tasks for Transaction management in Hive. + * Runs inside Hive Metastore Service. + */ +public class AcidHouseKeeperService implements RunnableConfigurable { + private static final Logger LOG = LoggerFactory.getLogger(AcidHouseKeeperService.class); + + private Configuration conf; + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + TxnStore.MutexAPI.LockHandle handle = null; + try { + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); + long startTime = System.currentTimeMillis(); + txnHandler.performTimeOuts(); + LOG.debug("timeout reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + + "seconds."); + } catch(Throwable t) { + LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } finally { + if(handle != null) { + handle.releaseLocks(); + } + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java new file mode 100644 index 000000000000..ae7e225e23cf --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.RunnableConfigurable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Periodically cleans WriteSet tracking information used in Transaction management + */ +public class AcidWriteSetService implements RunnableConfigurable { + private static final Logger LOG = LoggerFactory.getLogger(AcidWriteSetService.class); + + private Configuration conf; + + @Override + public void setConf(Configuration configuration) { + this.conf = configuration; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + TxnStore.MutexAPI.LockHandle handle = null; + try { + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); + long startTime = System.currentTimeMillis(); + txnHandler.performWriteSetGC(); + LOG.debug("cleaner ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds."); + } catch(Throwable t) { + LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } finally { + if(handle != null) { + handle.releaseLocks(); + } + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index d09c9580ffb3..756cb4cec24a 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -172,6 +172,12 @@ public static void prepDb(Configuration conf) throws Exception { LOG.error("Error rolling back: " + re.getMessage()); } + // Another thread might have already created these tables. + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + LOG.info("Txn tables already exist, returning"); + return; + } + // This might be a deadlock, if so, let's retry if (e instanceof SQLTransactionRollbackException && deadlockCnt++ < 5) { LOG.warn("Caught deadlock, retrying db creation"); From aacc0862d353c9d8f639a1e292e1cda322592214 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Fri, 1 Sep 2017 15:03:42 -0700 Subject: [PATCH 08/10] Moved FileMetadataManager. --- .../org/apache/hadoop/hive/io/HdfsUtils.java | 11 -------- .../hive/metastore/FileMetadataManager.java | 25 ++++++------------- .../hive/metastore/conf/MetastoreConf.java | 4 +++ .../hive/metastore/utils/HdfsUtils.java | 21 ++++++++++++++++ 4 files changed, 33 insertions(+), 28 deletions(-) rename {metastore/src => standalone-metastore/src/main}/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java (85%) diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java index 277738fac705..85ee8c75b2ea 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java @@ -51,17 +51,6 @@ public class HdfsUtils { private static final Logger LOG = LoggerFactory.getLogger("shims.HdfsUtils"); - // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this - // is still going to work. Otherwise, file IDs can be turned off. Later, we should use - // as public utility method in HDFS to obtain the inode-based path. - private static final String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/"; - - public static Path getFileIdPath( - FileSystem fileSystem, Path path, long fileId) { - return (fileSystem instanceof DistributedFileSystem) - ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path; - } - /** * Copy the permissions, group, and ACLs from a source {@link HadoopFileStatus} to a target {@link Path}. This method * will only log a warning if permissions cannot be set, no exception will be thrown. diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java similarity index 85% rename from metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java rename to standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java index 9115480fb299..0b8b3102e18e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,18 +18,14 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; -import java.util.LinkedList; - -import java.util.Queue; - import java.util.ArrayList; - import java.util.List; import com.google.common.collect.Lists; @@ -43,23 +39,19 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.io.HdfsUtils; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.HdfsUtils; public class FileMetadataManager { private static final Log LOG = LogFactory.getLog(FileMetadataManager.class); - private static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); private final RawStore tlms; private final ExecutorService threadPool; - private final HiveConf conf; + private final Configuration conf; private final class CacheUpdateRequest implements Callable { FileMetadataExprType type; @@ -84,10 +76,10 @@ public Void call() throws Exception { } } - public FileMetadataManager(RawStore tlms, HiveConf conf) { + public FileMetadataManager(RawStore tlms, Configuration conf) { this.tlms = tlms; this.conf = conf; - int numThreads = HiveConf.getIntVar(conf, ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS); + int numThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.FILE_METADATA_THREADS); this.threadPool = Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setNameFormat("File-Metadata-%d").setDaemon(true).build()); } @@ -118,8 +110,7 @@ private void cacheMetadata(FileMetadataExprType type, String location) // TODO: use the other HdfsUtils here if (!(fs instanceof DistributedFileSystem)) return; try { - fileId = SHIMS.getFileId((DistributedFileSystem)fs, - Path.getPathWithoutSchemeAndAuthority(file).toString()); + fileId = HdfsUtils.getFileId(fs, Path.getPathWithoutSchemeAndAuthority(file).toString()); } catch (UnsupportedOperationException ex) { LOG.error("Cannot cache file metadata for " + location + "; " + fs.getClass().getCanonicalName() + " does not support fileId"); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 31dae375d0c4..049445ed4b1b 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -151,6 +151,7 @@ public String toString() { ConfVars.AGGREGATE_STATS_CACHE_MAX_FULL, ConfVars.AGGREGATE_STATS_CACHE_CLEAN_UNTIL, ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES, + ConfVars.FILE_METADATA_THREADS }; /** @@ -430,6 +431,9 @@ public enum ConfVars { EXPRESSION_PROXY_CLASS("metastore.expression.proxy", "hive.metastore.expression.proxy", "org.apache.hadoop.hive.ql.optimizer.ppr.PartitionExpressionForMetastore", "Class to use to process expressions in partition pruning."), + FILE_METADATA_THREADS("metastore.file.metadata.threads", + "hive.metastore.hbase.file.metadata.threads", 1, + "Number of threads to use to read file metadata in background to cache it."), FILTER_HOOK("metastore.filter.hook", "hive.metastore.filter.hook", "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl", "Metastore hook class for filtering the metadata read results. If hive.security.authorization.manager" diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java index c10e36f94a5e..ecbddc3e7f6c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -44,6 +45,10 @@ public class HdfsUtils { private static final Logger LOG = LoggerFactory.getLogger(HdfsUtils.class); private static final String DISTCP_OPTIONS_PREFIX = "distcp.options."; + // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this + // is still going to work. Otherwise, file IDs can be turned off. Later, we should use + // as public utility method in HDFS to obtain the inode-based path. + private static final String HDFS_ID_PATH_PREFIX = "/.reserved/.inodes/"; /** * Check the permissions on a file. @@ -197,4 +202,20 @@ private static List constructDistCpParams(List srcPaths, Path dst, return params; } + public static Path getFileIdPath( + FileSystem fileSystem, Path path, long fileId) { + return (fileSystem instanceof DistributedFileSystem) + ? new Path(HDFS_ID_PATH_PREFIX + fileId) : path; + } + + public static long getFileId(FileSystem fs, String path) throws IOException { + return ensureDfs(fs).getClient().getFileInfo(path).getFileId(); + } + + private static DistributedFileSystem ensureDfs(FileSystem fs) { + if (!(fs instanceof DistributedFileSystem)) { + throw new UnsupportedOperationException("Only supported for DFS; got " + fs.getClass()); + } + return (DistributedFileSystem)fs; + } } From e5aedc318647f55d982c1eb3cbf3b4eb82e64467 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Sat, 14 Oct 2017 12:02:27 -0700 Subject: [PATCH 09/10] Fixes after the rebase. Also fixed acid house keeping threads to only open txn handler once instead of on every run. --- .../hive/metastore/TransactionalValidationListener.java | 1 + .../hive/metastore/txn/AcidCompactionHistoryService.java | 3 ++- .../hadoop/hive/metastore/txn/AcidHouseKeeperService.java | 3 ++- .../hadoop/hive/metastore/txn/AcidWriteSetService.java | 3 ++- .../apache/hadoop/hive/metastore/utils/MetaStoreUtils.java | 7 +++++++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 8f68e8100f0a..7d2537d6247f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java index 5cd043da4400..97bff0e9b343 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java @@ -29,10 +29,12 @@ public class AcidCompactionHistoryService implements RunnableConfigurable { private static final Logger LOG = LoggerFactory.getLogger(AcidCompactionHistoryService.class); private Configuration conf; + private TxnStore txnHandler; @Override public void setConf(Configuration configuration) { this.conf = configuration; + txnHandler = TxnUtils.getTxnStore(conf); } @Override @@ -44,7 +46,6 @@ public Configuration getConf() { public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { - TxnStore txnHandler = TxnUtils.getTxnStore(conf); handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); long startTime = System.currentTimeMillis(); txnHandler.purgeCompactionHistory(); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java index 0e7358ab380d..7450a2f3ac5d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java @@ -30,10 +30,12 @@ public class AcidHouseKeeperService implements RunnableConfigurable { private static final Logger LOG = LoggerFactory.getLogger(AcidHouseKeeperService.class); private Configuration conf; + private TxnStore txnHandler; @Override public void setConf(Configuration configuration) { this.conf = configuration; + txnHandler = TxnUtils.getTxnStore(conf); } @Override @@ -45,7 +47,6 @@ public Configuration getConf() { public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { - TxnStore txnHandler = TxnUtils.getTxnStore(conf); handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); long startTime = System.currentTimeMillis(); txnHandler.performTimeOuts(); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java index ae7e225e23cf..413fe96a18cc 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java @@ -29,10 +29,12 @@ public class AcidWriteSetService implements RunnableConfigurable { private static final Logger LOG = LoggerFactory.getLogger(AcidWriteSetService.class); private Configuration conf; + private TxnStore txnHandler; @Override public void setConf(Configuration configuration) { this.conf = configuration; + txnHandler = TxnUtils.getTxnStore(conf); } @Override @@ -44,7 +46,6 @@ public Configuration getConf() { public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { - TxnStore txnHandler = TxnUtils.getTxnStore(conf); handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); long startTime = System.currentTimeMillis(); txnHandler.performWriteSetGC(); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index d4b2bd536c3d..6e71019bb606 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -740,4 +741,10 @@ public static boolean columnsIncludedByNameType(List oldCols, return true; } + + /** Duplicates AcidUtils; used in a couple places in metastore. */ + public static boolean isInsertOnlyTableParam(Map params) { + String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); + return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp)); + } } From e12fda0fc853cd3da7f18e05b61bbf7cb2ab4030 Mon Sep 17 00:00:00 2001 From: Alan Gates Date: Mon, 16 Oct 2017 14:08:23 -0700 Subject: [PATCH 10/10] Fixed failure in TestTxnCommands --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 034d5660d2ee..662462c005f8 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -349,6 +349,8 @@ public void testTimeOutReaper() throws Exception { //now test that we don't timeout locks we should not //heartbeater should be running in the background every 1/2 second hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.SECONDS); + // Have to reset the conf when we change it so that the change takes affect + houseKeeperService.setConf(hiveConf); //hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILHEARTBEATER, true); runStatementOnDriver("start transaction"); runStatementOnDriver("select count(*) from " + Table.ACIDTBL + " where a = 17");