From a4b1bfbd3542cda42cab89e981326b18d875660e Mon Sep 17 00:00:00 2001 From: zhangbutao Date: Fri, 20 May 2022 16:30:05 +0800 Subject: [PATCH] HIVE-26246: Filter out results 'show connectors' on HMS client-side --- .../AuthorizationMetaStoreFilterHook.java | 10 ++++-- .../plugin/HivePrivilegeObjectUtils.java | 14 +++++++++ .../metastore/HiveMetaStoreAuthorizer.java | 5 +++ .../DefaultMetaStoreFilterHookImpl.java | 5 +++ .../hive/metastore/HiveMetaStoreClient.java | 3 +- .../hive/metastore/MetaStoreFilterHook.java | 7 +++++ .../hive/metastore/utils/FilterUtils.java | 2 +- .../hive/metastore/TestFilterHooks.java | 31 +++++++++++++++++++ 8 files changed, 72 insertions(+), 5 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java index 3bacfb458b1d..e40297620c20 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java @@ -49,7 +49,7 @@ public AuthorizationMetaStoreFilterHook(Configuration conf) { public List filterTableNames(String catName, String dbName, List tableList) throws MetaException { List listObjs = getHivePrivObjects(dbName, tableList); - return getTableNames(getFilteredObjects(listObjs)); + return getFilteredObjectNames(getFilteredObjects(listObjs)); } @Override public List filterTables(List
tableList) throws MetaException { @@ -98,7 +98,7 @@ private List getDbNames(List filteredObjects) { return tnames; } - private List getTableNames(List filteredObjects) { + private List getFilteredObjectNames(List filteredObjects) { List tnames = new ArrayList(); for(HivePrivilegeObject obj : filteredObjects) { tnames.add(obj.getObjectName()); @@ -153,6 +153,10 @@ public List filterTableMetas(String catName,String dbName,List filteredTableNames.contains(e.getTableName())).collect(Collectors.toList()); } - + @Override + public List filterDataConnectors(List dcList) throws MetaException { + List listObjs = HivePrivilegeObjectUtils.getHivePrivDcObjects(dcList); + return getFilteredObjectNames(getFilteredObjects(listObjs)); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java index 74a53f24ebac..4ab7870e7b36 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java @@ -48,4 +48,18 @@ public static List getHivePrivDbObjects(List dbList } + /** + * Convert list of dcnames into list of HivePrivilegeObject + * @param dcList + * @return + */ + public static List getHivePrivDcObjects(List dcList) { + List objs = new ArrayList(); + for (String dcname : dcList) { + objs.add(new HivePrivilegeObject(HivePrivilegeObjectType.DATACONNECTOR, null, dcname)); + } + return objs; + + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java index 969254ab1ae1..c276e6cccb89 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java @@ -233,6 +233,11 @@ public final List filterPartitionNames(String s, String s1, String s2, L return list; } + @Override + public List filterDataConnectors(List dcList) throws MetaException { + return dcList; + } + private List filterDatabaseObjects(HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo) throws MetaException { List ret = null; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java index 36a82c5149a5..3af78502040e 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java @@ -90,4 +90,9 @@ public List filterPartitionNames(String catName, String dbName, String t return partitionNames; } + @Override + public List filterDataConnectors(List dcList) throws MetaException { + return dcList; + } + } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index ac6979c20ff7..93229b950077 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1342,7 +1342,8 @@ public DataConnector getDataConnector(String name) */ @Override public List getAllDataConnectorNames() throws MetaException, TException { - return client.get_dataconnectors(); + List connectorNames = client.get_dataconnectors(); + return FilterUtils.filterDataConnectorsIfEnabled(isClientFilterEnabled, filterHook, connectorNames); } /** diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java index 2f48aed37f12..e2da15e54f21 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java @@ -143,5 +143,12 @@ List filterPartitionSpecs(List partitionSpecList) */ List filterPartitionNames(String catName, String dbName, String tblName, List partitionNames) throws MetaException; + + /** + * Filter given list of data connectors + * @param dcList + * @return List of filtered Dc names + */ + List filterDataConnectors(List dcList) throws MetaException; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java index c125f6993e45..d1ebe0aeaaa3 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java @@ -110,7 +110,7 @@ public static List filterDataConnectorsIfEnabled( List connectorNames) throws MetaException { if (isFilterEnabled) { - return filterHook.filterDatabases(connectorNames); // TODO add a new ATZ call + return filterHook.filterDataConnectors(connectorNames); } return connectorNames; } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 6e37c3b13a74..3f91c92e1cb2 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.CompactionType; +import org.apache.hadoop.hive.metastore.api.DataConnector; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -141,6 +142,14 @@ public List filterPartitionNames(String catName, String dbName, String t } return partitionNames; } + + @Override + public List filterDataConnectors(List dcList) throws MetaException { + if (blockResults) { + return new ArrayList<>(); + } + return dcList; + } } protected static HiveMetaStoreClient client; @@ -153,6 +162,12 @@ public List filterPartitionNames(String catName, String dbName, String t private static String DBNAME2 = "testdb2"; private static final String TAB1 = "tab1"; private static final String TAB2 = "tab2"; + private static String DCNAME1 = "test_connector1"; + private static String DCNAME2 = "test_connector2"; + private static String mysql_type = "mysql"; + private static String mysql_url = "jdbc:mysql://localhost:3306/hive"; + private static String postgres_type = "postgres"; + private static String postgres_url = "jdbc:postgresql://localhost:5432"; protected HiveMetaStoreClient createClient(Configuration metaStoreConf) throws Exception { @@ -210,6 +225,8 @@ protected void creatEnv(Configuration conf) throws Exception { client.dropDatabase(DBNAME1, true, true, true); client.dropDatabase(DBNAME2, true, true, true); + client.dropDataConnector(DCNAME1, true, true); + client.dropDataConnector(DCNAME2, true, true); Database db1 = new DatabaseBuilder() .setName(DBNAME1) .setCatalogName(Warehouse.DEFAULT_CATALOG_NAME) @@ -238,6 +255,10 @@ protected void creatEnv(Configuration conf) throws Exception { .inTable(tab2) .addValue("value2") .addToTable(client, conf); + DataConnector dc1 = new DataConnector(DCNAME1, mysql_type, mysql_url); + DataConnector dc2 = new DataConnector(DCNAME2, postgres_type, postgres_url); + client.createDataConnector(dc1); + client.createDataConnector(dc2); TestTxnDbUtil.cleanDb(conf); TestTxnDbUtil.prepDb(conf); @@ -271,6 +292,8 @@ public void testHMSServerWithoutFilter() throws Exception { assertEquals(1, client.getPartitionsByNames(DBNAME1, TAB2, Lists.newArrayList("name=value1")).size()); assertEquals(2, client.showCompactions().getCompacts().size()); + + assertEquals(2, client.getAllDataConnectorNames().size()); } /** @@ -318,6 +341,8 @@ public void testHMSClientWithoutFilter() throws Exception { assertEquals(1, client.getPartitionsByNames(DBNAME1, TAB2, Lists.newArrayList("name=value1")).size()); assertEquals(2, client.showCompactions().getCompacts().size()); + + assertEquals(2, client.getAllDataConnectorNames().size()); } /** @@ -336,6 +361,7 @@ public void testHMSClientWithFilter() throws Exception { testFilterForTables(false); testFilterForPartition(false); testFilterForCompaction(); + testFilterForDataConnector(); } protected void testFilterForDb(boolean filterAtServer) throws Exception { @@ -404,4 +430,9 @@ protected void testFilterForPartition(boolean filterAtServer) throws Exception { protected void testFilterForCompaction() throws Exception { assertEquals(0, client.showCompactions().getCompacts().size()); } + + protected void testFilterForDataConnector() throws Exception { + assertNotNull(client.getDataConnector(DCNAME1)); + assertEquals(0, client.getAllDataConnectorNames().size()); + } }