From ae81c2a093555a7f60d16c9d89b8c0ea5b79c4ff Mon Sep 17 00:00:00 2001 From: Naveen Gangam Date: Wed, 29 May 2024 19:16:27 -0400 Subject: [PATCH 1/2] HIVE-28286: Add filtering support for get_table_metas (Naveen Gangam) --- .../metastore/HiveMetaStoreAuthorizer.java | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java index b5199ad45d15..6b4d101d04ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java @@ -67,6 +67,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * HiveMetaStoreAuthorizer : Do authorization checks on MetaStore Events in MetaStorePreEventListener @@ -213,15 +214,46 @@ public final List filterCatalogs(List catalogs) throws MetaExcep @Override @Deprecated - public List filterTableMetas(String catName, String dbName,List tableMetas) + public List filterTableMetas(String catName, String dbName, List tableMetas) throws MetaException { - return filterTableMetas(tableMetas); + if (LOG.isDebugEnabled()) { + LOG.debug("==> HiveMetaStoreAuthorizer.filterTableMetas()"); + } + List tableNames = new ArrayList<>(); + List filteredTableMetas = new ArrayList<>(); + if (tableMetas != null) { + for (TableMeta tableMeta : tableMetas) { + tableNames.add(tableMeta.getTableName()); + } + TableFilterContext tableFilterContext = new TableFilterContext(dbName, tableNames); + HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo = tableFilterContext.getAuthzContext(); + final List filteredTableNames = filterTableNames(hiveMetaStoreAuthzInfo, dbName, tableNames); + if (CollectionUtils.isEmpty(filteredTableNames)) { + filteredTableMetas = Collections.emptyList(); + if (LOG.isInfoEnabled()) { + LOG.info("<== HiveMetaStoreAuthorizer.filterTableMetas() : returning empty set"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("<== HiveMetaStoreAuthorizer.filterTableMetas() : " + filteredTableNames); + } + filteredTableMetas = tableMetas.stream().filter(tblMeta -> filteredTableNames.stream() + .anyMatch(tblName -> tblName.equals(tblMeta.getTableName()))).collect(Collectors.toList()); + } + } + return filteredTableMetas; } @Override public final List filterTableMetas(List tableMetas) throws MetaException { - return tableMetas; + String catName = null; + String dbName = null; + if (tableMetas != null) { + catName = tableMetas.get(0).getCatName(); + dbName = tableMetas.get(0).getDbName(); + } + return filterTableMetas(catName, dbName, tableMetas); } @Override From 85a2d5d6f2a18e256628c420e01dd121378f2faf Mon Sep 17 00:00:00 2001 From: zdeng Date: Wed, 5 Jun 2024 14:25:46 +0800 Subject: [PATCH 2/2] review1 --- .../metastore/HiveMetaStoreAuthorizer.java | 63 +++++++++++-------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java index 6b4d101d04ae..25eb646e8b3d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java @@ -65,8 +65,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -216,44 +219,52 @@ public final List filterCatalogs(List catalogs) throws MetaExcep @Deprecated public List filterTableMetas(String catName, String dbName, List tableMetas) throws MetaException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> HiveMetaStoreAuthorizer.filterTableMetas()"); - } - List tableNames = new ArrayList<>(); - List filteredTableMetas = new ArrayList<>(); - if (tableMetas != null) { - for (TableMeta tableMeta : tableMetas) { + LOG.debug("==> HiveMetaStoreAuthorizer.filterTableMetas()"); + if (!CollectionUtils.isEmpty(tableMetas)) { + List tableNames = new ArrayList<>(); + tableMetas.forEach(tableMeta -> { + if (!tableMeta.getCatName().equalsIgnoreCase(catName) || + !tableMeta.getDbName().equalsIgnoreCase(dbName)) { + throw new IllegalArgumentException(String.format("Table: %s doesn't belong to the catalog: %s, database: %s", + tableMeta.getCatName() + "." + tableMeta.getDbName() + "." + tableMeta.getTableName(), catName, dbName)); + } tableNames.add(tableMeta.getTableName()); - } + }); TableFilterContext tableFilterContext = new TableFilterContext(dbName, tableNames); HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo = tableFilterContext.getAuthzContext(); final List filteredTableNames = filterTableNames(hiveMetaStoreAuthzInfo, dbName, tableNames); - if (CollectionUtils.isEmpty(filteredTableNames)) { - filteredTableMetas = Collections.emptyList(); - if (LOG.isInfoEnabled()) { - LOG.info("<== HiveMetaStoreAuthorizer.filterTableMetas() : returning empty set"); - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("<== HiveMetaStoreAuthorizer.filterTableMetas() : " + filteredTableNames); - } - filteredTableMetas = tableMetas.stream().filter(tblMeta -> filteredTableNames.stream() - .anyMatch(tblName -> tblName.equals(tblMeta.getTableName()))).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(filteredTableNames)) { + Set filteredTabs = new HashSet<>(filteredTableNames); + LOG.debug("<== HiveMetaStoreAuthorizer.filterTableMetas() : {}", filteredTabs); + return tableMetas.stream().filter(tblMeta -> filteredTabs.contains(tblMeta.getTableName())) + .collect(Collectors.toList()); } } - return filteredTableMetas; + LOG.info("<== HiveMetaStoreAuthorizer.filterTableMetas() : returning empty set"); + return Collections.emptyList(); } @Override public final List filterTableMetas(List tableMetas) throws MetaException { - String catName = null; - String dbName = null; - if (tableMetas != null) { - catName = tableMetas.get(0).getCatName(); - dbName = tableMetas.get(0).getDbName(); + LOG.debug("==> HiveMetaStoreAuthorizer.filterTableMetas()"); + if (!CollectionUtils.isEmpty(tableMetas)) { + Map> metaGroupByCatDb = new HashMap<>(); + tableMetas.forEach(tableMeta -> { + String key = MetaStoreUtils.prependCatalogToDbName(tableMeta.getCatName(), + tableMeta.getDbName(), getConf()).toLowerCase(); + metaGroupByCatDb.computeIfAbsent(key, s -> new ArrayList<>()).add(tableMeta); + }); + List filteredTabs = new ArrayList<>(); + for (Map.Entry> entry : metaGroupByCatDb.entrySet()) { + TableMeta firstTabMeta = entry.getValue().get(0); + filteredTabs.addAll(filterTableMetas(firstTabMeta.getCatName(), + firstTabMeta.getDbName(), entry.getValue())); + } + return filteredTabs; } - return filterTableMetas(catName, dbName, tableMetas); + LOG.info("<== HiveMetaStoreAuthorizer.filterTableMetas() : returning empty set"); + return Collections.emptyList(); } @Override