Skip to content

Commit

Permalink
Refactor GRANT/REVOKE in Hive
Browse files Browse the repository at this point in the history 
Leverage newly introduced method for recursive role grants traversal
  • Loading branch information
@arhimondr
Andrii Rosa authored and arhimondr committed Feb 27, 2019
1 parent 56c41c8 commit 0e03eff
Show file tree
Hide file tree
Showing 19 changed files with 250 additions and 609 deletions.
13 changes: 8 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.facebook.presto.hive.metastore.SortingColumn; import com.facebook.presto.hive.metastore.SortingColumn;
import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.metastore.StorageFormat;
import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.hive.metastore.Table;
import com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil;
import com.facebook.presto.hive.statistics.HiveStatisticsProvider; import com.facebook.presto.hive.statistics.HiveStatisticsProvider;
import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ColumnMetadata;
Expand Down Expand Up @@ -179,6 +180,7 @@
import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline;
import static com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; import static com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT;
import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat;
import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.listApplicableTablePrivileges;
import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf; import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf;
import static com.facebook.presto.hive.util.Statistics.ReduceOperator.ADD; import static com.facebook.presto.hive.util.Statistics.ReduceOperator.ADD;
import static com.facebook.presto.hive.util.Statistics.createComputedStatisticsToPartitionMap; import static com.facebook.presto.hive.util.Statistics.createComputedStatisticsToPartitionMap;
Expand Down Expand Up @@ -1942,7 +1944,7 @@ public void revokeRoles(ConnectorSession session, Set<String> roles, Set<PrestoP
@Override @Override
public Set<RoleGrant> listApplicableRoles(ConnectorSession session, PrestoPrincipal principal) public Set<RoleGrant> listApplicableRoles(ConnectorSession session, PrestoPrincipal principal)
{ {
return metastore.listApplicableRoles(principal); return ThriftMetastoreUtil.listApplicableRoles(principal, metastore::listRoleGrants);
} }


@Override @Override
Expand Down Expand Up @@ -1976,10 +1978,11 @@ public List<GrantInfo> listTablePrivileges(ConnectorSession session, SchemaTable
{ {
ImmutableList.Builder<GrantInfo> grantInfos = ImmutableList.builder(); ImmutableList.Builder<GrantInfo> grantInfos = ImmutableList.builder();
for (SchemaTableName tableName : listTables(session, schemaTablePrefix)) { for (SchemaTableName tableName : listTables(session, schemaTablePrefix)) {
Set<PrivilegeInfo> privileges = metastore.getTablePrivileges(session.getUser(), tableName.getSchemaName(), tableName.getTableName()).stream() Set<PrivilegeInfo> privileges =
.map(HivePrivilegeInfo::toPrivilegeInfo) listApplicableTablePrivileges(metastore, tableName.getSchemaName(), tableName.getTableName(), new PrestoPrincipal(USER, session.getUser())).stream()
.flatMap(Set::stream) .map(HivePrivilegeInfo::toPrivilegeInfo)
.collect(toImmutableSet()); .flatMap(Set::stream)
.collect(toImmutableSet());


grantInfos.add( grantInfos.add(
new GrantInfo( new GrantInfo(
Expand Down
63 changes: 20 additions & 43 deletions presto-hive/src/main/java/com/facebook/presto/hive/metastore/CachingHiveMetastore.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import static com.facebook.presto.hive.metastore.HivePartitionName.hivePartitionName; import static com.facebook.presto.hive.metastore.HivePartitionName.hivePartitionName;
import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName; import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName;
import static com.facebook.presto.hive.metastore.PartitionFilter.partitionFilter; import static com.facebook.presto.hive.metastore.PartitionFilter.partitionFilter;
import static com.facebook.presto.spi.security.PrincipalType.USER;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.base.Throwables.throwIfInstanceOf;
import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.base.Throwables.throwIfUnchecked;
Expand Down Expand Up @@ -83,8 +84,7 @@ public class CachingHiveMetastore
private final LoadingCache<HivePartitionName, Optional<Partition>> partitionCache; private final LoadingCache<HivePartitionName, Optional<Partition>> partitionCache;
private final LoadingCache<PartitionFilter, Optional<List<String>>> partitionFilterCache; private final LoadingCache<PartitionFilter, Optional<List<String>>> partitionFilterCache;
private final LoadingCache<HiveTableName, Optional<List<String>>> partitionNamesCache; private final LoadingCache<HiveTableName, Optional<List<String>>> partitionNamesCache;
private final LoadingCache<String, Set<String>> userRolesCache; private final LoadingCache<UserTableKey, Set<HivePrivilegeInfo>> tablePrivilegesCache;
private final LoadingCache<UserTableKey, Set<HivePrivilegeInfo>> userTablePrivileges;
private final LoadingCache<String, Set<String>> rolesCache; private final LoadingCache<String, Set<String>> rolesCache;
private final LoadingCache<PrestoPrincipal, Set<RoleGrant>> roleGrantsCache; private final LoadingCache<PrestoPrincipal, Set<RoleGrant>> roleGrantsCache;


Expand Down Expand Up @@ -187,11 +187,8 @@ public Map<HivePartitionName, Optional<Partition>> loadAll(Iterable<? extends Hi
} }
}, executor)); }, executor));


userRolesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) tablePrivilegesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
.build(asyncReloading(CacheLoader.from(user -> loadRoles(user)), executor)); .build(asyncReloading(CacheLoader.from(key -> loadTablePrivileges(key.getDatabase(), key.getTable(), key.getPrincipal())), executor));

userTablePrivileges = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
.build(asyncReloading(CacheLoader.from(key -> loadTablePrivileges(key.getUser(), key.getDatabase(), key.getTable())), executor));


rolesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize) rolesCache = newCacheBuilder(expiresAfterWriteMillis, refreshMills, maximumSize)
.build(asyncReloading(CacheLoader.from(() -> loadRoles()), executor)); .build(asyncReloading(CacheLoader.from(() -> loadRoles()), executor));
Expand All @@ -211,10 +208,9 @@ public void flushCache()
tableCache.invalidateAll(); tableCache.invalidateAll();
partitionCache.invalidateAll(); partitionCache.invalidateAll();
partitionFilterCache.invalidateAll(); partitionFilterCache.invalidateAll();
userTablePrivileges.invalidateAll(); tablePrivilegesCache.invalidateAll();
tableStatisticsCache.invalidateAll(); tableStatisticsCache.invalidateAll();
partitionStatisticsCache.invalidateAll(); partitionStatisticsCache.invalidateAll();
userRolesCache.invalidateAll();
rolesCache.invalidateAll(); rolesCache.invalidateAll();
} }


Expand Down Expand Up @@ -504,9 +500,9 @@ protected void invalidateTable(String databaseName, String tableName)
tableCache.invalidate(hiveTableName(databaseName, tableName)); tableCache.invalidate(hiveTableName(databaseName, tableName));
tableNamesCache.invalidate(databaseName); tableNamesCache.invalidate(databaseName);
viewNamesCache.invalidate(databaseName); viewNamesCache.invalidate(databaseName);
userTablePrivileges.asMap().keySet().stream() tablePrivilegesCache.asMap().keySet().stream()
.filter(userTableKey -> userTableKey.matches(databaseName, tableName)) .filter(userTableKey -> userTableKey.matches(databaseName, tableName))
.forEach(userTablePrivileges::invalidate); .forEach(tablePrivilegesCache::invalidate);
tableStatisticsCache.invalidate(hiveTableName(databaseName, tableName)); tableStatisticsCache.invalidate(hiveTableName(databaseName, tableName));
invalidatePartitionCache(databaseName, tableName); invalidatePartitionCache(databaseName, tableName);
} }
Expand Down Expand Up @@ -631,7 +627,6 @@ public void createRole(String role, String grantor)
} }
finally { finally {
rolesCache.invalidateAll(); rolesCache.invalidateAll();
userRolesCache.invalidate(grantor);
} }
} }


Expand All @@ -643,7 +638,6 @@ public void dropRole(String role)
} }
finally { finally {
rolesCache.invalidateAll(); rolesCache.invalidateAll();
userRolesCache.invalidateAll();
roleGrantsCache.invalidateAll(); roleGrantsCache.invalidateAll();
} }
} }
Expand Down Expand Up @@ -707,42 +701,14 @@ private void invalidatePartitionCache(String databaseName, String tableName)
.forEach(partitionStatisticsCache::invalidate); .forEach(partitionStatisticsCache::invalidate);
} }


@Override
public Set<String> getRoles(String user)
{
return get(userRolesCache, user);
}

private Set<String> loadRoles(String user)
{
return delegate.getRoles(user);
}

@Override
public Set<HivePrivilegeInfo> getDatabasePrivileges(String user, String databaseName)
{
return delegate.getDatabasePrivileges(user, databaseName);
}

@Override
public Set<HivePrivilegeInfo> getTablePrivileges(String user, String databaseName, String tableName)
{
return get(userTablePrivileges, new UserTableKey(user, tableName, databaseName));
}

private Set<HivePrivilegeInfo> loadTablePrivileges(String user, String databaseName, String tableName)
{
return delegate.getTablePrivileges(user, databaseName, tableName);
}

@Override @Override
public void grantTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges) public void grantTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges)
{ {
try { try {
delegate.grantTablePrivileges(databaseName, tableName, grantee, privileges); delegate.grantTablePrivileges(databaseName, tableName, grantee, privileges);
} }
finally { finally {
userTablePrivileges.invalidate(new UserTableKey(grantee, tableName, databaseName)); tablePrivilegesCache.invalidate(new UserTableKey(new PrestoPrincipal(USER, grantee), databaseName, tableName));
} }
} }


Expand All @@ -753,10 +719,21 @@ public void revokeTablePrivileges(String databaseName, String tableName, String
delegate.revokeTablePrivileges(databaseName, tableName, grantee, privileges); delegate.revokeTablePrivileges(databaseName, tableName, grantee, privileges);
} }
finally { finally {
userTablePrivileges.invalidate(new UserTableKey(grantee, tableName, databaseName)); tablePrivilegesCache.invalidate(new UserTableKey(new PrestoPrincipal(USER, grantee), databaseName, tableName));
} }
} }


@Override
public Set<HivePrivilegeInfo> listTablePrivileges(String databaseName, String tableName, PrestoPrincipal principal)
{
return get(tablePrivilegesCache, new UserTableKey(principal, databaseName, tableName));
}

public Set<HivePrivilegeInfo> loadTablePrivileges(String databaseName, String tableName, PrestoPrincipal principal)
{
return delegate.listTablePrivileges(databaseName, tableName, principal);
}

private static CacheBuilder<Object, Object> newCacheBuilder(OptionalLong expiresAfterWriteMillis, OptionalLong refreshMillis, long maximumSize) private static CacheBuilder<Object, Object> newCacheBuilder(OptionalLong expiresAfterWriteMillis, OptionalLong refreshMillis, long maximumSize)
{ {
CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder(); CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder();
Expand Down
8 changes: 2 additions & 6 deletions presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -99,13 +99,9 @@ public interface ExtendedHiveMetastore


Set<RoleGrant> listRoleGrants(PrestoPrincipal principal); Set<RoleGrant> listRoleGrants(PrestoPrincipal principal);


Set<String> getRoles(String user);

Set<HivePrivilegeInfo> getDatabasePrivileges(String user, String databaseName);

Set<HivePrivilegeInfo> getTablePrivileges(String user, String databaseName, String tableName);

void grantTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges); void grantTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges);


void revokeTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges); void revokeTablePrivileges(String databaseName, String tableName, String grantee, Set<HivePrivilegeInfo> privileges);

Set<HivePrivilegeInfo> listTablePrivileges(String databaseName, String tableName, PrestoPrincipal principal);
} }
67 changes: 13 additions & 54 deletions presto-hive/src/main/java/com/facebook/presto/hive/metastore/RecordingHiveMetastore.java
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ public class RecordingHiveMetastore
private final Cache<HiveTableName, Optional<List<String>>> partitionNamesCache; private final Cache<HiveTableName, Optional<List<String>>> partitionNamesCache;
private final Cache<PartitionFilter, Optional<List<String>>> partitionNamesByPartsCache; private final Cache<PartitionFilter, Optional<List<String>>> partitionNamesByPartsCache;
private final Cache<Set<HivePartitionName>, Map<String, Optional<Partition>>> partitionsByNamesCache; private final Cache<Set<HivePartitionName>, Map<String, Optional<Partition>>> partitionsByNamesCache;
private final Cache<String, Set<String>> rolesCache; private final Cache<UserTableKey, Set<HivePrivilegeInfo>> listTablePrivilegesCache;
private final Cache<UserDatabaseKey, Set<HivePrivilegeInfo>> databasePrivilegesCache;
private final Cache<UserTableKey, Set<HivePrivilegeInfo>> tablePrivilegesCache;
private final Cache<String, Set<String>> listRolesCache; private final Cache<String, Set<String>> listRolesCache;
private final Cache<PrestoPrincipal, Set<RoleGrant>> listRoleGrantsCache; private final Cache<PrestoPrincipal, Set<RoleGrant>> listRoleGrantsCache;


Expand All @@ -100,9 +98,7 @@ public RecordingHiveMetastore(@ForRecordingHiveMetastore ExtendedHiveMetastore d
partitionNamesCache = createCache(hiveClientConfig); partitionNamesCache = createCache(hiveClientConfig);
partitionNamesByPartsCache = createCache(hiveClientConfig); partitionNamesByPartsCache = createCache(hiveClientConfig);
partitionsByNamesCache = createCache(hiveClientConfig); partitionsByNamesCache = createCache(hiveClientConfig);
rolesCache = createCache(hiveClientConfig); listTablePrivilegesCache = createCache(hiveClientConfig);
databasePrivilegesCache = createCache(hiveClientConfig);
tablePrivilegesCache = createCache(hiveClientConfig);
listRolesCache = createCache(hiveClientConfig); listRolesCache = createCache(hiveClientConfig);
listRoleGrantsCache = createCache(hiveClientConfig); listRoleGrantsCache = createCache(hiveClientConfig);


Expand All @@ -129,9 +125,7 @@ void loadRecording()
partitionNamesCache.putAll(toMap(recording.getPartitionNames())); partitionNamesCache.putAll(toMap(recording.getPartitionNames()));
partitionNamesByPartsCache.putAll(toMap(recording.getPartitionNamesByParts())); partitionNamesByPartsCache.putAll(toMap(recording.getPartitionNamesByParts()));
partitionsByNamesCache.putAll(toMap(recording.getPartitionsByNames())); partitionsByNamesCache.putAll(toMap(recording.getPartitionsByNames()));
rolesCache.putAll(toMap(recording.getRoles())); listTablePrivilegesCache.putAll(toMap(recording.getListTablePrivileges()));
databasePrivilegesCache.putAll(toMap(recording.getDatabasePrivileges()));
tablePrivilegesCache.putAll(toMap(recording.getTablePrivileges()));
listRolesCache.putAll(toMap(recording.getListRoles())); listRolesCache.putAll(toMap(recording.getListRoles()));
listRoleGrantsCache.putAll(toMap(recording.getListRoleGrants())); listRoleGrantsCache.putAll(toMap(recording.getListRoleGrants()));
} }
Expand Down Expand Up @@ -169,9 +163,7 @@ public void writeRecording()
toPairs(partitionNamesCache), toPairs(partitionNamesCache),
toPairs(partitionNamesByPartsCache), toPairs(partitionNamesByPartsCache),
toPairs(partitionsByNamesCache), toPairs(partitionsByNamesCache),
toPairs(rolesCache), toPairs(listTablePrivilegesCache),
toPairs(databasePrivilegesCache),
toPairs(tablePrivilegesCache),
toPairs(listRolesCache), toPairs(listRolesCache),
toPairs(listRoleGrantsCache)); toPairs(listRoleGrantsCache));
new ObjectMapperProvider().get() new ObjectMapperProvider().get()
Expand Down Expand Up @@ -394,27 +386,12 @@ public void alterPartition(String databaseName, String tableName, PartitionWithS
} }


@Override @Override
public Set<String> getRoles(String user) public Set<HivePrivilegeInfo> listTablePrivileges(String databaseName, String tableName, PrestoPrincipal principal)
{
return loadValue(rolesCache, user, () -> delegate.getRoles(user));
}

@Override
public Set<HivePrivilegeInfo> getDatabasePrivileges(String user, String databaseName)
{
return loadValue(
databasePrivilegesCache,
new UserDatabaseKey(user, databaseName),
() -> delegate.getDatabasePrivileges(user, databaseName));
}

@Override
public Set<HivePrivilegeInfo> getTablePrivileges(String user, String databaseName, String tableName)
{ {
return loadValue( return loadValue(
tablePrivilegesCache, listTablePrivilegesCache,
new UserTableKey(user, databaseName, tableName), new UserTableKey(principal, databaseName, tableName),
() -> delegate.getTablePrivileges(user, databaseName, tableName)); () -> delegate.listTablePrivileges(databaseName, tableName, principal));
} }


@Override @Override
Expand Down Expand Up @@ -518,9 +495,7 @@ public static class Recording
private final List<Pair<HiveTableName, Optional<List<String>>>> partitionNames; private final List<Pair<HiveTableName, Optional<List<String>>>> partitionNames;
private final List<Pair<PartitionFilter, Optional<List<String>>>> partitionNamesByParts; private final List<Pair<PartitionFilter, Optional<List<String>>>> partitionNamesByParts;
private final List<Pair<Set<HivePartitionName>, Map<String, Optional<Partition>>>> partitionsByNames; private final List<Pair<Set<HivePartitionName>, Map<String, Optional<Partition>>>> partitionsByNames;
private final List<Pair<String, Set<String>>> roles; private final List<Pair<UserTableKey, Set<HivePrivilegeInfo>>> listTablePrivileges;
private final List<Pair<UserDatabaseKey, Set<HivePrivilegeInfo>>> databasePrivileges;
private final List<Pair<UserTableKey, Set<HivePrivilegeInfo>>> tablePrivileges;
private final List<Pair<String, Set<String>>> listRoles; private final List<Pair<String, Set<String>>> listRoles;
private final List<Pair<PrestoPrincipal, Set<RoleGrant>>> listRoleGrants; private final List<Pair<PrestoPrincipal, Set<RoleGrant>>> listRoleGrants;


Expand All @@ -538,9 +513,7 @@ public Recording(
@JsonProperty("partitionNames") List<Pair<HiveTableName, Optional<List<String>>>> partitionNames, @JsonProperty("partitionNames") List<Pair<HiveTableName, Optional<List<String>>>> partitionNames,
@JsonProperty("partitionNamesByParts") List<Pair<PartitionFilter, Optional<List<String>>>> partitionNamesByParts, @JsonProperty("partitionNamesByParts") List<Pair<PartitionFilter, Optional<List<String>>>> partitionNamesByParts,
@JsonProperty("partitionsByNames") List<Pair<Set<HivePartitionName>, Map<String, Optional<Partition>>>> partitionsByNames, @JsonProperty("partitionsByNames") List<Pair<Set<HivePartitionName>, Map<String, Optional<Partition>>>> partitionsByNames,
@JsonProperty("roles") List<Pair<String, Set<String>>> roles, @JsonProperty("listTablePrivileges") List<Pair<UserTableKey, Set<HivePrivilegeInfo>>> listTablePrivileges,
@JsonProperty("databasePrivileges") List<Pair<UserDatabaseKey, Set<HivePrivilegeInfo>>> databasePrivileges,
@JsonProperty("tablePrivileges") List<Pair<UserTableKey, Set<HivePrivilegeInfo>>> tablePrivileges,
@JsonProperty("listRoles") List<Pair<String, Set<String>>> listRoles, @JsonProperty("listRoles") List<Pair<String, Set<String>>> listRoles,
@JsonProperty("listRoleGrants") List<Pair<PrestoPrincipal, Set<RoleGrant>>> listRoleGrants) @JsonProperty("listRoleGrants") List<Pair<PrestoPrincipal, Set<RoleGrant>>> listRoleGrants)
{ {
Expand All @@ -556,9 +529,7 @@ public Recording(
this.partitionNames = partitionNames; this.partitionNames = partitionNames;
this.partitionNamesByParts = partitionNamesByParts; this.partitionNamesByParts = partitionNamesByParts;
this.partitionsByNames = partitionsByNames; this.partitionsByNames = partitionsByNames;
this.roles = roles; this.listTablePrivileges = listTablePrivileges;
this.databasePrivileges = databasePrivileges;
this.tablePrivileges = tablePrivileges;
this.listRoles = listRoles; this.listRoles = listRoles;
this.listRoleGrants = listRoleGrants; this.listRoleGrants = listRoleGrants;
} }
Expand Down Expand Up @@ -636,21 +607,9 @@ public List<Pair<Set<HivePartitionName>, Map<String, Optional<Partition>>>> getP
} }


@JsonProperty @JsonProperty
public List<Pair<String, Set<String>>> getRoles() public List<Pair<UserTableKey, Set<HivePrivilegeInfo>>> getListTablePrivileges()
{
return roles;
}

@JsonProperty
public List<Pair<UserDatabaseKey, Set<HivePrivilegeInfo>>> getDatabasePrivileges()
{
return databasePrivileges;
}

@JsonProperty
public List<Pair<UserTableKey, Set<HivePrivilegeInfo>>> getTablePrivileges()
{ {
return tablePrivileges; return listTablePrivileges;
} }


@JsonProperty @JsonProperty
Expand Down
Loading

0 comments on commit 0e03eff

Please sign in to comment.