From fb5a0892099772663fe0c977307001c0eb88e8e5 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Sat, 8 Jun 2024 01:16:54 +0800 Subject: [PATCH] [#3343] feat(core): Supports multiple securable objects (#3497) ### What changes were proposed in this pull request? Refactor the underlying layout to support multiple securable objects. ### Why are the changes needed? Fix: #3343 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing UT. --------- Co-authored-by: Heng Qin --- .../java/com/datastrato/gravitino/Entity.java | 12 ++ .../authorization/AccessControlManager.java | 9 +- .../gravitino/authorization/RoleManager.java | 7 +- .../datastrato/gravitino/meta/RoleEntity.java | 23 +- .../gravitino/proto/RoleEntitySerDe.java | 59 +++--- .../relational/mapper/CatalogMetaMapper.java | 11 + .../relational/mapper/FilesetMetaMapper.java | 38 ++++ .../relational/mapper/MetalakeMetaMapper.java | 11 + .../relational/mapper/RoleMetaMapper.java | 32 --- .../relational/mapper/SchemaMetaMapper.java | 11 + .../mapper/SecurableObjectMapper.java | 83 ++++++++ .../relational/mapper/TableMetaMapper.java | 11 + .../relational/mapper/TopicMetaMapper.java | 11 + .../storage/relational/po/RolePO.java | 59 +----- .../relational/po/SecurableObjectPO.java | 155 ++++++++++++++ .../service/CatalogMetaService.java | 11 + .../service/FilesetMetaService.java | 8 + .../service/MetalakeMetaService.java | 17 ++ .../relational/service/RoleMetaService.java | 114 ++++++++-- .../relational/service/SchemaMetaService.java | 6 + .../relational/service/TableMetaService.java | 8 + .../relational/service/TopicMetaService.java | 10 +- .../session/SqlSessionFactoryHelper.java | 2 + .../relational/utils/MetadataObjectUtils.java | 164 +++++++++++++++ .../relational/utils/POConverters.java | 74 ++++--- .../TestAccessControlManager.java | 18 +- ...estAccessControlManagerForPermissions.java | 14 +- .../datastrato/gravitino/meta/TestEntity.java | 18 +- .../gravitino/proto/TestEntityProtoSerDe.java | 8 +- .../gravitino/storage/TestEntityStorage.java | 6 +- .../storage/memory/TestMemoryEntityStore.java | 7 +- .../storage/relational/TestJDBCBackend.java | 25 ++- .../service/TestGroupMetaService.java | 89 ++++++-- .../service/TestRoleMetaService.java | 82 +++++++- .../service/TestSecurableObjects.java | 113 ++++++++++ .../service/TestUserMetaService.java | 89 ++++++-- core/src/test/resources/h2/schema-h2.sql | 18 +- docs/how-to-use-relational-backend-storage.md | 4 +- .../integration/test/util/AbstractIT.java | 2 +- meta/src/main/proto/gravitino_meta.proto | 16 +- scripts/mysql/schema-0.6.0-mysql.sql | 196 ++++++++++++++++++ .../mysql/upgrade-0.5.0-to-0.6.0-mysql.sql | 22 ++ .../server/web/rest/RoleOperations.java | 48 ++--- .../server/web/rest/TestRoleOperations.java | 26 ++- 44 files changed, 1457 insertions(+), 290 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SecurableObjectMapper.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/po/SecurableObjectPO.java create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java create mode 100644 core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestSecurableObjects.java create mode 100644 scripts/mysql/schema-0.6.0-mysql.sql create mode 100644 scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql diff --git a/core/src/main/java/com/datastrato/gravitino/Entity.java b/core/src/main/java/com/datastrato/gravitino/Entity.java index d8d6ab03cd..bf42fda72f 100644 --- a/core/src/main/java/com/datastrato/gravitino/Entity.java +++ b/core/src/main/java/com/datastrato/gravitino/Entity.java @@ -40,6 +40,18 @@ public interface Entity extends Serializable { /** The admin schema name in the authorization catalog of the system metalake. */ String ADMIN_SCHEMA_NAME = "admin"; + /** + * All metalakes are a virtual entity. It represents all the metalakes. We don't store it. We use + * a specific type to represent its entity type. + */ + String ALL_METALAKES_ENTITY_TYPE = "ROOT"; + + /** + * All metalakes are a virtual entity. It represents all the metalakes. We don't store it. We use + * a specific id to represent its entity id. + */ + long ALL_METALAKES_ENTITY_ID = 0; + /** Enumeration defining the types of entities in the Gravitino framework. */ @Getter enum EntityType { diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java index d8024b924f..23eeb1f7e6 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/AccessControlManager.java @@ -249,17 +249,20 @@ public boolean isMetalakeAdmin(String user) { * @param metalake The Metalake of the Role. * @param role The name of the Role. * @param properties The properties of the Role. - * @param securableObject The securable object of the Role. + * @param securableObjects The securable objects of the Role. * @return The created Role instance. * @throws RoleAlreadyExistsException If a Role with the same name already exists. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( - String metalake, String role, Map properties, SecurableObject securableObject) + String metalake, + String role, + Map properties, + List securableObjects) throws RoleAlreadyExistsException, NoSuchMetalakeException { return doWithNonAdminLock( - () -> roleManager.createRole(metalake, role, properties, securableObject)); + () -> roleManager.createRole(metalake, role, properties, securableObjects)); } /** diff --git a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java index 654aab9a02..a282e89d06 100644 --- a/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/com/datastrato/gravitino/authorization/RoleManager.java @@ -72,7 +72,10 @@ class RoleManager { } RoleEntity createRole( - String metalake, String role, Map properties, SecurableObject securableObject) + String metalake, + String role, + Map properties, + List securableObjects) throws RoleAlreadyExistsException { AuthorizationUtils.checkMetalakeExists(metalake); RoleEntity roleEntity = @@ -80,7 +83,7 @@ RoleEntity createRole( .withId(idGenerator.nextId()) .withName(role) .withProperties(properties) - .withSecurableObject(securableObject) + .withSecurableObjects(securableObjects) .withNamespace(AuthorizationUtils.ofRoleNamespace(metalake)) .withAuditInfo( AuditInfo.builder() diff --git a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java index aaa8c15b36..3cad2c83f9 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/RoleEntity.java @@ -11,7 +11,7 @@ import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Role; import com.datastrato.gravitino.authorization.SecurableObject; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import java.util.Collections; import java.util.List; @@ -33,15 +33,14 @@ public class RoleEntity implements Role, Entity, Auditable, HasIdentifier { Field.required("audit_info", AuditInfo.class, "The audit details of the role entity."); public static final Field SECURABLE_OBJECT = - Field.required( - "securable_object", SecurableObject.class, "The securable object of the role entity."); + Field.required("securable_objects", List.class, "The securable objects of the role entity."); private Long id; private String name; private Map properties; private AuditInfo auditInfo; private Namespace namespace; - private SecurableObject securableObject; + private List securableObjects; /** * The name of the role. @@ -87,7 +86,7 @@ public List securableObjects() { // So one type of them can't be the securable object at least if there are the two same // identifier // entities . - return Lists.newArrayList(securableObject); + return securableObjects; } /** @@ -102,7 +101,7 @@ public Map fields() { fields.put(NAME, name); fields.put(AUDIT_INFO, auditInfo); fields.put(PROPERTIES, properties); - fields.put(SECURABLE_OBJECT, securableObject); + fields.put(SECURABLE_OBJECT, securableObjects); return Collections.unmodifiableMap(fields); } @@ -138,12 +137,12 @@ public boolean equals(Object o) { && Objects.equals(namespace, that.namespace) && Objects.equals(auditInfo, that.auditInfo) && Objects.equals(properties, that.properties) - && Objects.equals(securableObject, that.securableObject); + && Objects.equals(securableObjects, that.securableObjects); } @Override public int hashCode() { - return Objects.hash(id, name, properties, auditInfo, securableObject); + return Objects.hash(id, name, properties, auditInfo, securableObjects); } /** @@ -212,13 +211,13 @@ public Builder withAuditInfo(AuditInfo auditInfo) { } /** - * Sets the securable object of the role entity. + * Sets the securable objects of the role entity. * - * @param securableObject The securable object of the role entity. + * @param securableObjects The securable objects of the role entity. * @return The builder instance. */ - public Builder withSecurableObject(SecurableObject securableObject) { - roleEntity.securableObject = securableObject; + public Builder withSecurableObjects(List securableObjects) { + roleEntity.securableObjects = ImmutableList.copyOf(securableObjects); return this; } diff --git a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java index e0add5b3f0..3867b24d3e 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/RoleEntitySerDe.java @@ -28,17 +28,25 @@ public Role serialize(RoleEntity roleEntity) { Role.newBuilder() .setId(roleEntity.id()) .setName(roleEntity.name()) - .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())) - .addAllPrivileges( - roleEntity.securableObjects().get(0).privileges().stream() - .map(privilege -> privilege.name().toString()) - .collect(Collectors.toList())) - .addAllPrivilegeConditions( - roleEntity.securableObjects().get(0).privileges().stream() - .map(privilege -> privilege.condition().toString()) - .collect(Collectors.toList())) - .setSecurableObjectFullName(roleEntity.securableObjects().get(0).fullName()) - .setSecurableObjectType(roleEntity.securableObjects().get(0).type().name()); + .setAuditInfo(new AuditInfoSerDe().serialize(roleEntity.auditInfo())); + + for (SecurableObject securableObject : roleEntity.securableObjects()) { + builder.addSecurableObjects( + com.datastrato.gravitino.proto.SecurableObject.newBuilder() + .setFullName(securableObject.fullName()) + .setType(securableObject.type().name()) + .addAllPrivilegeConditions( + securableObject.privileges().stream() + .map(Privilege::condition) + .map(Privilege.Condition::name) + .collect(Collectors.toList())) + .addAllPrivilegeNames( + securableObject.privileges().stream() + .map(Privilege::name) + .map(Privilege.Name::name) + .collect(Collectors.toList())) + .build()); + } if (roleEntity.properties() != null && !roleEntity.properties().isEmpty()) { builder.putAllProperties(roleEntity.properties()); @@ -55,31 +63,32 @@ public Role serialize(RoleEntity roleEntity) { */ @Override public RoleEntity deserialize(Role role, Namespace namespace) { - List privileges = Lists.newArrayList(); - - if (!role.getPrivilegesList().isEmpty()) { + List securableObjects = Lists.newArrayList(); - for (int index = 0; index < role.getPrivilegeConditionsCount(); index++) { - if (Privilege.Condition.ALLOW.name().equals(role.getPrivilegeConditions(index))) { - privileges.add(Privileges.allow(role.getPrivileges(index))); + for (int index = 0; index < role.getSecurableObjectsCount(); index++) { + List privileges = Lists.newArrayList(); + com.datastrato.gravitino.proto.SecurableObject object = role.getSecurableObjects(index); + for (int privIndex = 0; privIndex < object.getPrivilegeConditionsCount(); privIndex++) { + if (Privilege.Condition.ALLOW.name().equals(object.getPrivilegeConditions(privIndex))) { + privileges.add(Privileges.allow(object.getPrivilegeNames(privIndex))); } else { - privileges.add(Privileges.deny(role.getPrivileges(index))); + privileges.add(Privileges.deny(object.getPrivilegeNames(privIndex))); } } - } - SecurableObject securableObject = - SecurableObjects.parse( - role.getSecurableObjectFullName(), - SecurableObject.Type.valueOf(role.getSecurableObjectType()), - privileges); + SecurableObject securableObject = + SecurableObjects.parse( + object.getFullName(), SecurableObject.Type.valueOf(object.getType()), privileges); + + securableObjects.add(securableObject); + } RoleEntity.Builder builder = RoleEntity.builder() .withId(role.getId()) .withName(role.getName()) .withNamespace(namespace) - .withSecurableObject(securableObject) + .withSecurableObjects(securableObjects) .withAuditInfo(new AuditInfoSerDe().deserialize(role.getAuditInfo(), namespace)); if (!role.getPropertiesMap().isEmpty()) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java index 5e40deb39b..62ad43f887 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -54,6 +54,17 @@ Long selectCatalogIdByMetalakeIdAndName( CatalogPO selectCatalogMetaByMetalakeIdAndName( @Param("metalakeId") Long metalakeId, @Param("catalogName") String name); + @Select( + "SELECT catalog_id as catalogId, catalog_name as catalogName," + + " metalake_id as metalakeId, type, provider," + + " catalog_comment as catalogComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE catalog_id = #{catalogId} AND deleted_at = 0") + CatalogPO selectCatalogMetaById(@Param("catalogId") Long catalogId); + @Insert( "INSERT INTO " + TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java index ee87daa43f..d1acd515f0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/FilesetMetaMapper.java @@ -112,6 +112,44 @@ Long selectFilesetIdBySchemaIdAndName( FilesetPO selectFilesetMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("filesetName") String name); + @Select( + "SELECT fm.fileset_id, fm.fileset_name, fm.metalake_id, fm.catalog_id, fm.schema_id," + + " fm.type, fm.audit_info, fm.current_version, fm.last_version, fm.deleted_at," + + " vi.id, vi.metalake_id as version_metalake_id, vi.catalog_id as version_catalog_id," + + " vi.schema_id as version_schema_id, vi.fileset_id as version_fileset_id," + + " vi.version, vi.fileset_comment, vi.properties, vi.storage_location," + + " vi.deleted_at as version_deleted_at" + + " FROM " + + META_TABLE_NAME + + " fm INNER JOIN " + + VERSION_TABLE_NAME + + " vi ON fm.fileset_id = vi.fileset_id AND fm.current_version = vi.version" + + " WHERE fm.fileset_id = #{filesetId}" + + " AND fm.deleted_at = 0 AND vi.deleted_at = 0") + @Results({ + @Result(property = "filesetId", column = "fileset_id"), + @Result(property = "filesetName", column = "fileset_name"), + @Result(property = "metalakeId", column = "metalake_id"), + @Result(property = "catalogId", column = "catalog_id"), + @Result(property = "schemaId", column = "schema_id"), + @Result(property = "type", column = "type"), + @Result(property = "auditInfo", column = "audit_info"), + @Result(property = "currentVersion", column = "current_version"), + @Result(property = "lastVersion", column = "last_version"), + @Result(property = "deletedAt", column = "deleted_at"), + @Result(property = "filesetVersionPO.id", column = "id"), + @Result(property = "filesetVersionPO.metalakeId", column = "version_metalake_id"), + @Result(property = "filesetVersionPO.catalogId", column = "version_catalog_id"), + @Result(property = "filesetVersionPO.schemaId", column = "version_schema_id"), + @Result(property = "filesetVersionPO.filesetId", column = "version_fileset_id"), + @Result(property = "filesetVersionPO.version", column = "version"), + @Result(property = "filesetVersionPO.filesetComment", column = "fileset_comment"), + @Result(property = "filesetVersionPO.properties", column = "properties"), + @Result(property = "filesetVersionPO.storageLocation", column = "storage_location"), + @Result(property = "filesetVersionPO.deletedAt", column = "version_deleted_at") + }) + FilesetPO selectFilesetMetaById(@Param("filesetId") Long filesetId); + @Insert( "INSERT INTO " + META_TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java index 29b87d64b5..324b81f29f 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/MetalakeMetaMapper.java @@ -46,6 +46,17 @@ public interface MetalakeMetaMapper { + " WHERE metalake_name = #{metalakeName} and deleted_at = 0") MetalakePO selectMetalakeMetaByName(@Param("metalakeName") String name); + @Select( + "SELECT metalake_id as metalakeId, metalake_name as metalakeName," + + " metalake_comment as metalakeComment, properties," + + " audit_info as auditInfo, schema_version as schemaVersion," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE metalake_id = #{metalaId} and deleted_at = 0") + MetalakePO selectMetalakeMetaById(@Param("metalakeId") Long metalakeId); + @Select( "SELECT metalake_id FROM " + TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java index 805d5bd5a0..81945c424a 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java @@ -29,10 +29,6 @@ public interface RoleMetaMapper { @Select( "SELECT role_id as roleId, role_name as roleName," + " metalake_id as metalakeId, properties as properties," - + " securable_object_full_name as securableObjectFullName," - + " securable_object_type as securableObjectType," - + " privileges as privileges," - + " privilege_conditions as privilegeConditions," + " audit_info as auditInfo, current_version as currentVersion," + " last_version as lastVersion, deleted_at as deletedAt" + " FROM " @@ -53,10 +49,6 @@ Long selectRoleIdByMetalakeIdAndName( @Select( "SELECT ro.role_id as roleId, ro.role_name as roleName," + " ro.metalake_id as metalakeId, ro.properties as properties," - + " securable_object_full_name as securableObjectFullName," - + " securable_object_type as securableObjectType," - + " ro.privileges as privileges," - + " ro.privilege_conditions as privilegeConditions," + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + " FROM " @@ -71,10 +63,6 @@ Long selectRoleIdByMetalakeIdAndName( @Select( "SELECT ro.role_id as roleId, ro.role_name as roleName," + " ro.metalake_id as metalakeId, ro.properties as properties," - + " ro.securable_object_full_name as securableObjectFullName," - + " ro.securable_object_type as securableObjectType," - + " ro.privileges as privileges," - + " ro.privilege_conditions as privilegeConditions," + " ro.audit_info as auditInfo, ro.current_version as currentVersion," + " ro.last_version as lastVersion, ro.deleted_at as deletedAt" + " FROM " @@ -91,20 +79,12 @@ Long selectRoleIdByMetalakeIdAndName( + ROLE_TABLE_NAME + "(role_id, role_name," + " metalake_id, properties," - + " securable_object_full_name," - + " securable_object_type," - + " privileges," - + " privilege_conditions," + " audit_info, current_version, last_version, deleted_at)" + " VALUES(" + " #{roleMeta.roleId}," + " #{roleMeta.roleName}," + " #{roleMeta.metalakeId}," + " #{roleMeta.properties}," - + " #{roleMeta.securableObjectFullName}," - + " #{roleMeta.securableObjectType}," - + " #{roleMeta.privileges}," - + " #{roleMeta.privilegeConditions}," + " #{roleMeta.auditInfo}," + " #{roleMeta.currentVersion}," + " #{roleMeta.lastVersion}," @@ -117,20 +97,12 @@ Long selectRoleIdByMetalakeIdAndName( + ROLE_TABLE_NAME + "(role_id, role_name," + " metalake_id, properties," - + " securable_object_full_name," - + " securable_object_type," - + " privileges," - + " privilege_conditions," + " audit_info, current_version, last_version, deleted_at)" + " VALUES(" + " #{roleMeta.roleId}," + " #{roleMeta.roleName}," + " #{roleMeta.metalakeId}," + " #{roleMeta.properties}," - + " #{roleMeta.securableObjectFullName}," - + " #{roleMeta.securableObjectType}," - + " #{roleMeta.privileges}," - + " #{roleMeta.privilegeConditions}," + " #{roleMeta.auditInfo}," + " #{roleMeta.currentVersion}," + " #{roleMeta.lastVersion}," @@ -139,10 +111,6 @@ Long selectRoleIdByMetalakeIdAndName( + " role_name = #{roleMeta.roleName}," + " metalake_id = #{roleMeta.metalakeId}," + " properties = #{roleMeta.properties}," - + " securable_object_full_name = #{roleMeta.securableObjectFullName}," - + " securable_object_type = #{roleMeta.securableObjectType}," - + " privileges = #{roleMeta.privileges}," - + " privilege_conditions = #{roleMeta.privilegeConditions}," + " audit_info = #{roleMeta.auditInfo}," + " current_version = #{roleMeta.currentVersion}," + " last_version = #{roleMeta.lastVersion}," diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java index 8a27a440ab..ee809e4b34 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -55,6 +55,17 @@ Long selectSchemaIdByCatalogIdAndName( SchemaPO selectSchemaMetaByCatalogIdAndName( @Param("catalogId") Long catalogId, @Param("schemaName") String name); + @Select( + "SELECT schema_id as schemaId, schema_name as schemaName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_comment as schemaComment, properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE schema_id = #{schemaId} AND deleted_at = 0") + SchemaPO selectSchemaMetaById(@Param("schemaId") Long schemaId); + @Insert( "INSERT INTO " + TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SecurableObjectMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SecurableObjectMapper.java new file mode 100644 index 0000000000..55cf19a584 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/SecurableObjectMapper.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.mapper; + +import com.datastrato.gravitino.storage.relational.po.SecurableObjectPO; +import java.util.List; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +/** + * A MyBatis Mapper for table meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ +public interface SecurableObjectMapper { + + String SECURABLE_OBJECT_TABLE_NAME = "role_meta_securable_object"; + String ROLE_TABLE_NAME = "role_meta"; + + @Insert({ + "" + }) + void batchInsertSecurableObjects( + @Param("securableObjects") List securableObjectPOs); + + @Update( + "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " WHERE role_id = #{roleId} AND deleted_at = 0") + void softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId); + + @Update( + "UPDATE " + + SECURABLE_OBJECT_TABLE_NAME + + " ob SET ob.deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0" + + " where exists ( select * from " + + ROLE_TABLE_NAME + + " ro WHERE ro.metalake_id = #{metalakeId} AND ro.role_id = ob.role_id" + + " AND ro.deleted_at = 0) AND ob.deleted_at = 0") + void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId); + + @Select( + "SELECT role_id as roleId, entity_id as entityId," + + " type as type, privilege_names as privilegeNames," + + " privilege_conditions as privilegeConditions, current_version as currentVersion," + + " last_version as lastVersion, deleted_at as deletedAt" + + " FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE role_id = #{roleId} AND deleted_at = 0") + List listSecurableObjectsByRoleId(@Param("roleId") Long roleId); + + @Delete( + "DELETE FROM " + + SECURABLE_OBJECT_TABLE_NAME + + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}") + Integer deleteSecurableObjectsByLegacyTimeLine( + @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit); +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java index aeeaa1eac1..b3b17c3eb7 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TableMetaMapper.java @@ -55,6 +55,17 @@ Long selectTableIdBySchemaIdAndName( TablePO selectTableMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("tableName") String name); + @Select( + "SELECT table_id as tableId, table_name as tableName," + + " metalake_id as metalakeId, catalog_id as catalogId," + + " schema_id as schemaId, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE table_id = #{tableId} AND deleted_at = 0") + TablePO selectTableMetaById(@Param("tableId") Long tableId); + @Insert( "INSERT INTO " + TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java index 06f0d611db..c25975f9c3 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/TopicMetaMapper.java @@ -91,6 +91,17 @@ public interface TopicMetaMapper { TopicPO selectTopicMetaBySchemaIdAndName( @Param("schemaId") Long schemaId, @Param("topicName") String topicName); + @Select( + "SELECT topic_id as topicId, topic_name as topicName," + + " metalake_id as metalakeId, catalog_id as catalogId, schema_id as schemaId," + + " comment as comment, properties as properties, audit_info as auditInfo," + + " current_version as currentVersion, last_version as lastVersion," + + " deleted_at as deletedAt" + + " FROM " + + TABLE_NAME + + " WHERE topic_id = #{topicId} AND deleted_at = 0") + TopicPO selectTopicMetaById(@Param("topicId") Long topicId); + @Update( "UPDATE " + TABLE_NAME diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java index 82d8ba6373..eec1087ec0 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/RolePO.java @@ -12,10 +12,6 @@ public class RolePO { private String roleName; private Long metalakeId; private String properties; - private String securableObjectFullName; - private String securableObjectType; - private String privileges; - private String privilegeConditions; private String auditInfo; private Long currentVersion; private Long lastVersion; @@ -37,22 +33,6 @@ public String getProperties() { return properties; } - public String getSecurableObjectFullName() { - return securableObjectFullName; - } - - public String getSecurableObjectType() { - return securableObjectType; - } - - public String getPrivileges() { - return privileges; - } - - public String getPrivilegeConditions() { - return privilegeConditions; - } - public String getAuditInfo() { return auditInfo; } @@ -82,14 +62,10 @@ public boolean equals(Object o) { && Objects.equal(getRoleName(), tablePO.getRoleName()) && Objects.equal(getMetalakeId(), tablePO.getMetalakeId()) && Objects.equal(getProperties(), tablePO.getProperties()) - && Objects.equal(getSecurableObjectFullName(), tablePO.getSecurableObjectFullName()) - && Objects.equal(getSecurableObjectType(), tablePO.getSecurableObjectType()) - && Objects.equal(getPrivileges(), tablePO.getPrivileges()) && Objects.equal(getAuditInfo(), tablePO.getAuditInfo()) && Objects.equal(getCurrentVersion(), tablePO.getCurrentVersion()) && Objects.equal(getLastVersion(), tablePO.getLastVersion()) - && Objects.equal(getDeletedAt(), tablePO.getDeletedAt()) - && Objects.equal(getPrivilegeConditions(), tablePO.getPrivilegeConditions()); + && Objects.equal(getDeletedAt(), tablePO.getDeletedAt()); } @Override @@ -99,14 +75,10 @@ public int hashCode() { getRoleName(), getMetalakeId(), getProperties(), - getSecurableObjectFullName(), - getSecurableObjectType(), - getPrivileges(), getAuditInfo(), getCurrentVersion(), getLastVersion(), - getDeletedAt(), - getPrivilegeConditions()); + getDeletedAt()); } public static class Builder { @@ -136,26 +108,6 @@ public Builder withProperties(String properties) { return this; } - public Builder withSecurableObjectFullName(String securableObjectFullName) { - rolePO.securableObjectFullName = securableObjectFullName; - return this; - } - - public Builder withSecurableObjectType(String securableObjectType) { - rolePO.securableObjectType = securableObjectType; - return this; - } - - public Builder withPrivileges(String privileges) { - rolePO.privileges = privileges; - return this; - } - - public Builder withPrivilegeConditions(String privilegeConditions) { - rolePO.privilegeConditions = privilegeConditions; - return this; - } - public Builder withAuditInfo(String auditInfo) { rolePO.auditInfo = auditInfo; return this; @@ -180,13 +132,6 @@ private void validate() { Preconditions.checkArgument(rolePO.roleId != null, "Role id is required"); Preconditions.checkArgument(rolePO.roleName != null, "Role name is required"); Preconditions.checkArgument(rolePO.metalakeId != null, "Metalake id is required"); - Preconditions.checkArgument( - rolePO.securableObjectFullName != null, "Securable object full name is required"); - Preconditions.checkArgument( - rolePO.securableObjectType != null, "Securable object type is required"); - Preconditions.checkArgument(rolePO.privileges != null, "Privileges are required"); - Preconditions.checkArgument( - rolePO.privilegeConditions != null, "Privilege conditions are required"); Preconditions.checkArgument(rolePO.auditInfo != null, "Audit info is required"); Preconditions.checkArgument(rolePO.currentVersion != null, "Current version is required"); Preconditions.checkArgument(rolePO.lastVersion != null, "Last version is required"); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/po/SecurableObjectPO.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/SecurableObjectPO.java new file mode 100644 index 0000000000..12c90f2873 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/po/SecurableObjectPO.java @@ -0,0 +1,155 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.po; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +public class SecurableObjectPO { + + private Long roleId; + private Long entityId; + private String type; + private String privilegeNames; + private String privilegeConditions; + private Long currentVersion; + private Long lastVersion; + private Long deletedAt; + + public Long getRoleId() { + return roleId; + } + + public Long getEntityId() { + return entityId; + } + + public String getType() { + return type; + } + + public String getPrivilegeNames() { + return privilegeNames; + } + + public String getPrivilegeConditions() { + return privilegeConditions; + } + + public Long getCurrentVersion() { + return currentVersion; + } + + public Long getLastVersion() { + return lastVersion; + } + + public Long getDeletedAt() { + return deletedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SecurableObjectPO)) { + return false; + } + SecurableObjectPO securableObjectPO = (SecurableObjectPO) o; + return Objects.equal(getRoleId(), securableObjectPO.getRoleId()) + && Objects.equal(getEntityId(), securableObjectPO.getEntityId()) + && Objects.equal(getType(), securableObjectPO.getType()) + && Objects.equal(getPrivilegeConditions(), securableObjectPO.getPrivilegeConditions()) + && Objects.equal(getPrivilegeNames(), securableObjectPO.getPrivilegeNames()) + && Objects.equal(getCurrentVersion(), securableObjectPO.getCurrentVersion()) + && Objects.equal(getLastVersion(), securableObjectPO.getLastVersion()) + && Objects.equal(getDeletedAt(), securableObjectPO.getDeletedAt()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getRoleId(), + getEntityId(), + getType(), + getPrivilegeNames(), + getPrivilegeConditions(), + getCurrentVersion(), + getLastVersion(), + getDeletedAt()); + } + + public static class Builder { + private final SecurableObjectPO securableObjectPO; + + private Builder() { + securableObjectPO = new SecurableObjectPO(); + } + + public Builder withRoleId(Long roleId) { + securableObjectPO.roleId = roleId; + return this; + } + + public Builder withEntityId(long entityId) { + securableObjectPO.entityId = entityId; + return this; + } + + public Builder withType(String type) { + securableObjectPO.type = type; + return this; + } + + public Builder withPrivilegeNames(String privilegeNames) { + securableObjectPO.privilegeNames = privilegeNames; + return this; + } + + public Builder withPrivilegeConditions(String privilegeConditions) { + securableObjectPO.privilegeConditions = privilegeConditions; + return this; + } + + public Builder withCurrentVersion(Long currentVersion) { + securableObjectPO.currentVersion = currentVersion; + return this; + } + + public Builder withLastVersion(Long lastVersion) { + securableObjectPO.lastVersion = lastVersion; + return this; + } + + public Builder withDeletedAt(Long deletedAt) { + securableObjectPO.deletedAt = deletedAt; + return this; + } + + private void validate() { + Preconditions.checkArgument(securableObjectPO.roleId != null, "Role id is required"); + Preconditions.checkArgument(securableObjectPO.type != null, "Type is required"); + Preconditions.checkArgument( + securableObjectPO.privilegeNames != null, "Privilege names are required"); + Preconditions.checkArgument( + securableObjectPO.privilegeConditions != null, "Priviege conditions are required"); + Preconditions.checkArgument( + securableObjectPO.currentVersion != null, "Current version is required"); + Preconditions.checkArgument( + securableObjectPO.lastVersion != null, "Last version is required"); + Preconditions.checkArgument(securableObjectPO.deletedAt != null, "Deleted at is required"); + } + + public SecurableObjectPO build() { + validate(); + return securableObjectPO; + } + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index 418ff06f1c..1d94e43ce5 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Objects; import java.util.function.Function; +import javax.annotation.Nullable; /** * The service class for catalog metadata. It provides the basic database operations for catalog. @@ -57,6 +58,16 @@ public CatalogPO getCatalogPOByMetalakeIdAndName(Long metalakeId, String catalog return catalogPO; } + // Catalog may be deleted, so the CatalogPO may be null. + @Nullable + public CatalogPO getCatalogPOById(Long catalogId) { + CatalogPO catalogPO = + SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, mapper -> mapper.selectCatalogMetaById(catalogId)); + + return catalogPO; + } + public Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) { Long catalogId = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java index 8da9c9fdf9..90b7aa943d 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java @@ -57,6 +57,14 @@ public FilesetPO getFilesetPOBySchemaIdAndName(Long schemaId, String filesetName return filesetPO; } + // Filset may be deleted, so the FilesetPO may be null. + public FilesetPO getFilesetPOById(Long filesetId) { + FilesetPO filesetPO = + SessionUtils.getWithoutCommit( + FilesetMetaMapper.class, mapper -> mapper.selectFilesetMetaById(filesetId)); + return filesetPO; + } + public Long getFilesetIdBySchemaIdAndName(Long schemaId, String filesetName) { Long filesetId = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index d6055033ff..cdf5ad4838 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -20,6 +20,7 @@ import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.SchemaMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.SecurableObjectMapper; import com.datastrato.gravitino.storage.relational.mapper.TableMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TopicMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; @@ -82,6 +83,14 @@ public BaseMetalake getMetalakeByIdentifier(NameIdentifier ident) { return POConverters.fromMetalakePO(metalakePO); } + // Metalake may be deleted, so the MetalakePO may be null. + public MetalakePO getMetalakePOById(Long id) { + MetalakePO metalakePO = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaById(id)); + return metalakePO; + } + public void insertMetalake(BaseMetalake baseMetalake, boolean overwrite) { try { NameIdentifierUtil.checkMetalake(baseMetalake.nameIdentifier()); @@ -196,6 +205,10 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { () -> SessionUtils.doWithoutCommit( RoleMetaMapper.class, + mapper -> mapper.softDeleteRoleMetasByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + SecurableObjectMapper.class, mapper -> mapper.softDeleteRoleMetasByMetalakeId(metalakeId))); } else { List catalogEntities = @@ -229,6 +242,10 @@ public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { () -> SessionUtils.doWithoutCommit( RoleMetaMapper.class, + mapper -> mapper.softDeleteRoleMetasByMetalakeId(metalakeId)), + () -> + SessionUtils.doWithoutCommit( + SecurableObjectMapper.class, mapper -> mapper.softDeleteRoleMetasByMetalakeId(metalakeId))); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java index 394b3b8082..0f848959f7 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java @@ -5,21 +5,31 @@ package com.datastrato.gravitino.storage.relational.service; import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.authorization.AuthorizationUtils; +import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.storage.relational.mapper.GroupRoleRelMapper; import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.SecurableObjectMapper; import com.datastrato.gravitino.storage.relational.mapper.UserRoleRelMapper; import com.datastrato.gravitino.storage.relational.po.RolePO; +import com.datastrato.gravitino.storage.relational.po.SecurableObjectPO; import com.datastrato.gravitino.storage.relational.utils.ExceptionUtils; +import com.datastrato.gravitino.storage.relational.utils.MetadataObjectUtils; import com.datastrato.gravitino.storage.relational.utils.POConverters; import com.datastrato.gravitino.storage.relational.utils.SessionUtils; +import com.google.common.collect.Lists; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** The service class for role metadata. It provides the basic database operations for role. */ public class RoleMetaService { + + private static final Logger LOG = LoggerFactory.getLogger(RoleMetaService.class); private static final RoleMetaService INSTANCE = new RoleMetaService(); public static RoleMetaService getInstance() { @@ -76,16 +86,39 @@ public void insertRole(RoleEntity roleEntity, boolean overwritten) { MetalakeMetaService.getInstance().getMetalakeIdByName(roleEntity.namespace().level(0)); RolePO.Builder builder = RolePO.builder().withMetalakeId(metalakeId); RolePO rolePO = POConverters.initializeRolePOWithVersion(roleEntity, builder); - - SessionUtils.doWithCommit( - RoleMetaMapper.class, - mapper -> { - if (overwritten) { - mapper.insertRoleMetaOnDuplicateKeyUpdate(rolePO); - } else { - mapper.insertRoleMeta(rolePO); - } - }); + List securableObjectPOs = Lists.newArrayList(); + for (SecurableObject object : roleEntity.securableObjects()) { + SecurableObjectPO.Builder objectBuilder = + POConverters.initializeSecurablePOBuilderWithVersion( + roleEntity.id(), object, getEntityType(object)); + objectBuilder.withEntityId( + MetadataObjectUtils.getSecurableObjectEntityId( + metalakeId, object.fullName(), object.type())); + securableObjectPOs.add(objectBuilder.build()); + } + + SessionUtils.doMultipleWithCommit( + () -> + SessionUtils.doWithoutCommit( + SecurableObjectMapper.class, + mapper -> { + if (overwritten) { + mapper.softDeleteSecurableObjectsByRoleId(rolePO.getRoleId()); + } + if (!securableObjectPOs.isEmpty()) { + mapper.batchInsertSecurableObjects(securableObjectPOs); + } + }), + () -> + SessionUtils.doWithoutCommit( + RoleMetaMapper.class, + mapper -> { + if (overwritten) { + mapper.insertRoleMetaOnDuplicateKeyUpdate(rolePO); + } else { + mapper.insertRoleMeta(rolePO); + } + })); } catch (RuntimeException re) { ExceptionUtils.checkSQLException( @@ -101,7 +134,26 @@ public RoleEntity getRoleByIdentifier(NameIdentifier identifier) { MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); RolePO rolePO = getRolePOByMetalakeIdAndName(metalakeId, identifier.name()); - return POConverters.fromRolePO(rolePO, identifier.namespace()); + List securableObjectPOs = listSecurableObjectsByRoleId(rolePO.getRoleId()); + List securableObjects = Lists.newArrayList(); + + for (SecurableObjectPO securableObjectPO : securableObjectPOs) { + String fullName = + MetadataObjectUtils.getSecurableObjectFullName( + securableObjectPO.getType(), securableObjectPO.getEntityId()); + if (fullName != null) { + securableObjects.add( + POConverters.fromSecurableObjectPO( + fullName, securableObjectPO, getType(securableObjectPO.getType()))); + } else { + LOG.info( + "The securable object {} {} may be deleted", + securableObjectPO.getEntityId(), + securableObjectPO.getType()); + } + } + + return POConverters.fromRolePO(rolePO, securableObjects, identifier.namespace()); } public boolean deleteRole(NameIdentifier identifier) { @@ -120,14 +172,24 @@ public boolean deleteRole(NameIdentifier identifier) { UserRoleRelMapper.class, mapper -> mapper.softDeleteUserRoleRelByRoleId(roleId)), () -> SessionUtils.doWithoutCommit( - GroupRoleRelMapper.class, mapper -> mapper.softDeleteGroupRoleRelByRoleId(roleId))); + GroupRoleRelMapper.class, mapper -> mapper.softDeleteGroupRoleRelByRoleId(roleId)), + () -> + SessionUtils.doWithoutCommit( + SecurableObjectMapper.class, + mapper -> mapper.softDeleteSecurableObjectsByRoleId(roleId))); return true; } + private List listSecurableObjectsByRoleId(Long roleId) { + return SessionUtils.getWithoutCommit( + SecurableObjectMapper.class, mapper -> mapper.listSecurableObjectsByRoleId(roleId)); + } + public int deleteRoleMetasByLegacyTimeLine(long legacyTimeLine, int limit) { int[] roleDeletedCount = new int[] {0}; int[] userRoleRelDeletedCount = new int[] {0}; int[] groupRoleRelDeletedCount = new int[] {0}; + int[] securableObjectsCount = new int[] {0}; SessionUtils.doMultipleWithCommit( () -> @@ -145,8 +207,32 @@ public int deleteRoleMetasByLegacyTimeLine(long legacyTimeLine, int limit) { SessionUtils.doWithoutCommitAndFetchResult( GroupRoleRelMapper.class, mapper -> - mapper.deleteGroupRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit))); + mapper.deleteGroupRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)), + () -> + securableObjectsCount[0] = + SessionUtils.doWithoutCommitAndFetchResult( + SecurableObjectMapper.class, + mapper -> + mapper.deleteSecurableObjectsByLegacyTimeLine(legacyTimeLine, limit))); + + return roleDeletedCount[0] + + userRoleRelDeletedCount[0] + + groupRoleRelDeletedCount[0] + + securableObjectsCount[0]; + } - return roleDeletedCount[0] + userRoleRelDeletedCount[0] + groupRoleRelDeletedCount[0]; + private MetadataObject.Type getType(String type) { + if (Entity.ALL_METALAKES_ENTITY_TYPE.equals(type)) { + return MetadataObject.Type.METALAKE; + } + return MetadataObject.Type.valueOf(type); + } + + private String getEntityType(SecurableObject securableObject) { + if (securableObject.type() == MetadataObject.Type.METALAKE + && securableObject.name().equals(Entity.SECURABLE_ENTITY_RESERVED_NAME)) { + return Entity.ALL_METALAKES_ENTITY_TYPE; + } + return securableObject.type().name(); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java index 14b53d121f..dc691cad0d 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java @@ -55,6 +55,12 @@ public SchemaPO getSchemaPOByCatalogIdAndName(Long catalogId, String schemaName) return schemaPO; } + // Schema may be deleted, so the SchemaPO may be null. + public SchemaPO getSchemaPOById(Long schemaId) { + return SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, mapper -> mapper.selectSchemaMetaById(schemaId)); + } + public Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { Long schemaId = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java index 47d9d0ea83..4b0f2e9107 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java @@ -48,6 +48,14 @@ public TablePO getTablePOBySchemaIdAndName(Long schemaId, String tableName) { return tablePO; } + // Table may be deleted, so the TablePO may be null. + public TablePO getTablePOById(Long tableId) { + TablePO tablePO = + SessionUtils.getWithoutCommit( + TableMetaMapper.class, mapper -> mapper.selectTableMetaById(tableId)); + return tablePO; + } + public Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { Long tableId = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java index b79db5ea80..d372c805db 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java @@ -126,6 +126,14 @@ private TopicPO getTopicPOBySchemaIdAndName(Long schemaId, String topicName) { return topicPO; } + // Topic may be deleted, so the TopicPO may be null. + public TopicPO getTopicPOById(Long topicId) { + TopicPO topicPO = + SessionUtils.getWithoutCommit( + TopicMetaMapper.class, mapper -> mapper.selectTopicMetaById(topicId)); + return topicPO; + } + private void fillTopicPOBuilderParentEntityId(TopicPO.Builder builder, Namespace namespace) { NamespaceUtil.checkTopic(namespace); Long parentEntityId = null; @@ -186,7 +194,7 @@ public int deleteTopicMetasByLegacyTimeLine(Long legacyTimeLine, int limit) { }); } - private Long getTopicIdBySchemaIdAndName(Long schemaId, String topicName) { + public Long getTopicIdBySchemaIdAndName(Long schemaId, String topicName) { Long topicId = SessionUtils.getWithoutCommit( TopicMetaMapper.class, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java index 64526ebf35..da7028ac31 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/session/SqlSessionFactoryHelper.java @@ -15,6 +15,7 @@ import com.datastrato.gravitino.storage.relational.mapper.MetalakeMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.RoleMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.SchemaMetaMapper; +import com.datastrato.gravitino.storage.relational.mapper.SecurableObjectMapper; import com.datastrato.gravitino.storage.relational.mapper.TableMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.TopicMetaMapper; import com.datastrato.gravitino.storage.relational.mapper.UserMetaMapper; @@ -96,6 +97,7 @@ public void init(Config config) { configuration.addMapper(UserRoleRelMapper.class); configuration.addMapper(GroupMetaMapper.class); configuration.addMapper(GroupRoleRelMapper.class); + configuration.addMapper(SecurableObjectMapper.class); // Create the SqlSessionFactory object, it is a singleton object if (sqlSessionFactory == null) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java new file mode 100644 index 0000000000..fd5c0c5fc9 --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java @@ -0,0 +1,164 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.utils; + +import com.datastrato.gravitino.Entity; +import com.datastrato.gravitino.MetadataObject; +import com.datastrato.gravitino.storage.relational.po.CatalogPO; +import com.datastrato.gravitino.storage.relational.po.FilesetPO; +import com.datastrato.gravitino.storage.relational.po.MetalakePO; +import com.datastrato.gravitino.storage.relational.po.SchemaPO; +import com.datastrato.gravitino.storage.relational.po.TablePO; +import com.datastrato.gravitino.storage.relational.po.TopicPO; +import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; +import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; +import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; +import com.datastrato.gravitino.storage.relational.service.SchemaMetaService; +import com.datastrato.gravitino.storage.relational.service.TableMetaService; +import com.datastrato.gravitino.storage.relational.service.TopicMetaService; +import com.google.common.base.Splitter; +import java.util.List; +import javax.annotation.Nullable; + +/** + * MetadataObjectUtils is used for converting full name to entity id and converting entity id to + * full name. + */ +public class MetadataObjectUtils { + + private static final String DOT = "."; + private static final Splitter DOT_SPLITTER = Splitter.on(DOT); + + private MetadataObjectUtils() {} + + public static long getSecurableObjectEntityId( + long metalakeId, String fullName, MetadataObject.Type type) { + if (fullName.equals(Entity.SECURABLE_ENTITY_RESERVED_NAME) + && type == MetadataObject.Type.METALAKE) { + return Entity.ALL_METALAKES_ENTITY_ID; + } + + if (type == MetadataObject.Type.METALAKE) { + return MetalakeMetaService.getInstance().getMetalakeIdByName(fullName); + } + + List names = DOT_SPLITTER.splitToList(fullName); + long catalogId = + CatalogMetaService.getInstance().getCatalogIdByMetalakeIdAndName(metalakeId, names.get(0)); + if (type == MetadataObject.Type.CATALOG) { + return catalogId; + } + + long schemaId = + SchemaMetaService.getInstance().getSchemaIdByCatalogIdAndName(catalogId, names.get(1)); + if (type == MetadataObject.Type.SCHEMA) { + return schemaId; + } + + if (type == MetadataObject.Type.FILESET) { + return FilesetMetaService.getInstance().getFilesetIdBySchemaIdAndName(schemaId, names.get(2)); + } else if (type == MetadataObject.Type.TOPIC) { + return TopicMetaService.getInstance().getTopicIdBySchemaIdAndName(schemaId, names.get(2)); + } else if (type == MetadataObject.Type.TABLE) { + return TableMetaService.getInstance().getTableIdBySchemaIdAndName(schemaId, names.get(2)); + } + + throw new IllegalArgumentException(String.format("Doesn't support the type %s", type)); + } + + // Securable object may be null because the securable object may be deleted. + @Nullable + public static String getSecurableObjectFullName(String type, long entityId) { + if (type.equals(Entity.ALL_METALAKES_ENTITY_TYPE)) { + return Entity.SECURABLE_ENTITY_RESERVED_NAME; + } + + MetadataObject.Type metadatatype = MetadataObject.Type.valueOf(type); + if (metadatatype == MetadataObject.Type.METALAKE) { + MetalakePO metalakePO = MetalakeMetaService.getInstance().getMetalakePOById(entityId); + if (metalakePO == null) { + return null; + } + + return metalakePO.getMetalakeName(); + } + + if (metadatatype == MetadataObject.Type.CATALOG) { + return getCatalogFullName(entityId); + } + + if (metadatatype == MetadataObject.Type.SCHEMA) { + return getSchemaFullName(entityId); + } + + if (metadatatype == MetadataObject.Type.TABLE) { + TablePO tablePO = TableMetaService.getInstance().getTablePOById(entityId); + if (tablePO == null) { + return null; + } + + String schemaName = getSchemaFullName(tablePO.getSchemaId()); + if (schemaName == null) { + return null; + } + + return String.join(DOT, schemaName, tablePO.getTableName()); + } + + if (metadatatype == MetadataObject.Type.TOPIC) { + TopicPO topicPO = TopicMetaService.getInstance().getTopicPOById(entityId); + if (topicPO == null) { + return null; + } + + String schemaName = getSchemaFullName(topicPO.getSchemaId()); + if (schemaName == null) { + return null; + } + + return String.join(DOT, schemaName, topicPO.getTopicName()); + } + + if (metadatatype == MetadataObject.Type.FILESET) { + FilesetPO filesetPO = FilesetMetaService.getInstance().getFilesetPOById(entityId); + if (filesetPO == null) { + return null; + } + + String schemaName = getSchemaFullName(filesetPO.getSchemaId()); + if (schemaName == null) { + return null; + } + + return String.join(DOT, schemaName, filesetPO.getFilesetName()); + } + + throw new IllegalArgumentException(String.format("Doesn't support the type %s", metadatatype)); + } + + @Nullable + private static String getCatalogFullName(Long entityId) { + CatalogPO catalogPO = CatalogMetaService.getInstance().getCatalogPOById(entityId); + if (catalogPO == null) { + return null; + } + return catalogPO.getCatalogName(); + } + + @Nullable + private static String getSchemaFullName(Long entityId) { + SchemaPO schemaPO = SchemaMetaService.getInstance().getSchemaPOById(entityId); + + if (schemaPO == null) { + return null; + } + + String catalogName = getCatalogFullName(schemaPO.getCatalogId()); + if (catalogName == null) { + return null; + } + return String.join(DOT, catalogName, schemaPO.getSchemaName()); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index eb89e8a64d..ec9c6b3271 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -6,6 +6,7 @@ package com.datastrato.gravitino.storage.relational.utils; import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.Privilege; import com.datastrato.gravitino.authorization.Privileges; @@ -32,12 +33,12 @@ import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.datastrato.gravitino.storage.relational.po.RolePO; import com.datastrato.gravitino.storage.relational.po.SchemaPO; +import com.datastrato.gravitino.storage.relational.po.SecurableObjectPO; import com.datastrato.gravitino.storage.relational.po.TablePO; import com.datastrato.gravitino.storage.relational.po.TopicPO; import com.datastrato.gravitino.storage.relational.po.UserPO; import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.Lists; import java.util.List; import java.util.Map; @@ -783,20 +784,6 @@ public static RolePO initializeRolePOWithVersion(RoleEntity roleEntity, RolePO.B .withRoleId(roleEntity.id()) .withRoleName(roleEntity.name()) .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(roleEntity.properties())) - .withSecurableObjectFullName(roleEntity.securableObjects().get(0).fullName()) - .withSecurableObjectType(roleEntity.securableObjects().get(0).type().name()) - .withPrivileges( - JsonUtils.anyFieldMapper() - .writeValueAsString( - roleEntity.securableObjects().get(0).privileges().stream() - .map(privilege -> privilege.name().toString()) - .collect(Collectors.toList()))) - .withPrivilegeConditions( - JsonUtils.anyFieldMapper() - .writeValueAsString( - roleEntity.securableObjects().get(0).privileges().stream() - .map(privilege -> privilege.condition().toString()) - .collect(Collectors.toList()))) .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(roleEntity.auditInfo())) .withCurrentVersion(INIT_VERSION) .withLastVersion(INIT_VERSION) @@ -886,37 +873,39 @@ public static List initializeGroupRoleRelsPOWithVersion( } } - public static RoleEntity fromRolePO(RolePO rolePO, Namespace namespace) { + public static SecurableObject fromSecurableObjectPO( + String fullName, SecurableObjectPO securableObjectPO, MetadataObject.Type type) { try { - List privilegeNames = - JsonUtils.anyFieldMapper() - .readValue(rolePO.getPrivileges(), new TypeReference>() {}); + JsonUtils.anyFieldMapper().readValue(securableObjectPO.getPrivilegeNames(), List.class); List privilegeConditions = JsonUtils.anyFieldMapper() - .readValue(rolePO.getPrivilegeConditions(), new TypeReference>() {}); + .readValue(securableObjectPO.getPrivilegeConditions(), List.class); List privileges = Lists.newArrayList(); for (int index = 0; index < privilegeNames.size(); index++) { if (Privilege.Condition.ALLOW.name().equals(privilegeConditions.get(index))) { privileges.add(Privileges.allow(privilegeNames.get(index))); } else { - privileges.add(Privileges.allow(privilegeNames.get(index))); + privileges.add(Privileges.deny(privilegeNames.get(index))); } } - SecurableObject securableObject = - SecurableObjects.parse( - rolePO.getSecurableObjectFullName(), - SecurableObject.Type.valueOf(rolePO.getSecurableObjectType()), - privileges); + return SecurableObjects.parse(fullName, type, privileges); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to deserialize json object:", e); + } + } + public static RoleEntity fromRolePO( + RolePO rolePO, List securableObjects, Namespace namespace) { + try { return RoleEntity.builder() .withId(rolePO.getRoleId()) .withName(rolePO.getRoleName()) .withNamespace(namespace) .withProperties(JsonUtils.anyFieldMapper().readValue(rolePO.getProperties(), Map.class)) - .withSecurableObject(securableObject) + .withSecurableObjects(securableObjects) .withAuditInfo( JsonUtils.anyFieldMapper().readValue(rolePO.getAuditInfo(), AuditInfo.class)) .build(); @@ -924,4 +913,35 @@ public static RoleEntity fromRolePO(RolePO rolePO, Namespace namespace) { throw new RuntimeException("Failed to deserialize json object:", e); } } + + public static SecurableObjectPO.Builder initializeSecurablePOBuilderWithVersion( + long roleId, SecurableObject securableObject, String type) { + try { + SecurableObjectPO.Builder builder = SecurableObjectPO.builder(); + builder + .withRoleId(roleId) + .withType(type) + .withPrivilegeConditions( + JsonUtils.anyFieldMapper() + .writeValueAsString( + securableObject.privileges().stream() + .map(Privilege::condition) + .map(Privilege.Condition::name) + .collect(Collectors.toList()))) + .withPrivilegeNames( + JsonUtils.anyFieldMapper() + .writeValueAsString( + securableObject.privileges().stream() + .map(Privilege::name) + .map(Privilege.Name::name) + .collect(Collectors.toList()))) + .withCurrentVersion(INIT_VERSION) + .withLastVersion(INIT_VERSION) + .withDeletedAt(DEFAULT_DELETED_AT); + + return builder; + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } + } } diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java index daa78f7ee5..a78ea934d1 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManager.java @@ -225,8 +225,9 @@ public void testCreateRole() { "metalake", "create", props, - SecurableObjects.ofCatalog( - "catalog", Lists.newArrayList(Privileges.UseCatalog.allow()))); + Lists.newArrayList( + SecurableObjects.ofCatalog( + "catalog", Lists.newArrayList(Privileges.UseCatalog.allow())))); Assertions.assertEquals("create", role.name()); testProperties(props, role.properties()); @@ -238,8 +239,9 @@ public void testCreateRole() { "metalake", "create", props, - SecurableObjects.ofCatalog( - "catalog", Lists.newArrayList(Privileges.UseCatalog.allow())))); + Lists.newArrayList( + SecurableObjects.ofCatalog( + "catalog", Lists.newArrayList(Privileges.UseCatalog.allow()))))); } @Test @@ -250,7 +252,9 @@ public void testLoadRole() { "metalake", "loadRole", props, - SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.UseCatalog.allow()))); + Lists.newArrayList( + SecurableObjects.ofCatalog( + "catalog", Lists.newArrayList(Privileges.UseCatalog.allow())))); Role cachedRole = accessControlManager.getRole("metalake", "loadRole"); accessControlManager.getRoleManager().getCache().invalidateAll(); @@ -277,7 +281,9 @@ public void testDropRole() { "metalake", "testDrop", props, - SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.UseCatalog.allow()))); + Lists.newArrayList( + SecurableObjects.ofCatalog( + "catalog", Lists.newArrayList(Privileges.UseCatalog.allow())))); // Test drop role boolean dropped = accessControlManager.deleteRole("metalake", "testDrop"); diff --git a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java index 5ae68af773..f67437dfe6 100644 --- a/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/com/datastrato/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -85,9 +85,10 @@ public class TestAccessControlManagerForPermissions { .withId(1L) .withName("role") .withProperties(Maps.newHashMap()) - .withSecurableObject( - SecurableObjects.ofCatalog( - CATALOG, Lists.newArrayList(Privileges.UseCatalog.allow()))) + .withSecurableObjects( + Lists.newArrayList( + SecurableObjects.ofCatalog( + CATALOG, Lists.newArrayList(Privileges.UseCatalog.allow())))) .withAuditInfo(auditInfo) .build(); @@ -271,9 +272,10 @@ public void testDropRole() throws IOException { .withId(1L) .withName(anotherRole) .withProperties(Maps.newHashMap()) - .withSecurableObject( - SecurableObjects.ofCatalog( - CATALOG, Lists.newArrayList(Privileges.UseCatalog.allow()))) + .withSecurableObjects( + Lists.newArrayList( + SecurableObjects.ofCatalog( + CATALOG, Lists.newArrayList(Privileges.UseCatalog.allow())))) .withAuditInfo(auditInfo) .build(); diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java index fdd645526c..1b340e4113 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestEntity.java @@ -266,9 +266,10 @@ public void testRole() { .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .withSecurableObject( - SecurableObjects.ofCatalog( - catalogName, Lists.newArrayList(Privileges.UseCatalog.allow()))) + .withSecurableObjects( + Lists.newArrayList( + SecurableObjects.ofCatalog( + catalogName, Lists.newArrayList(Privileges.UseCatalog.allow())))) .withProperties(map) .build(); @@ -278,7 +279,9 @@ public void testRole() { Assertions.assertEquals(auditInfo, fields.get(RoleEntity.AUDIT_INFO)); Assertions.assertEquals(map, fields.get(RoleEntity.PROPERTIES)); Assertions.assertEquals( - SecurableObjects.ofCatalog(catalogName, Lists.newArrayList(Privileges.UseCatalog.allow())), + Lists.newArrayList( + SecurableObjects.ofCatalog( + catalogName, Lists.newArrayList(Privileges.UseCatalog.allow()))), fields.get(RoleEntity.SECURABLE_OBJECT)); RoleEntity roleWithoutFields = @@ -286,9 +289,10 @@ public void testRole() { .withId(1L) .withName(roleName) .withAuditInfo(auditInfo) - .withSecurableObject( - SecurableObjects.ofCatalog( - catalogName, Lists.newArrayList(Privileges.UseCatalog.allow()))) + .withSecurableObjects( + Lists.newArrayList( + SecurableObjects.ofCatalog( + catalogName, Lists.newArrayList(Privileges.UseCatalog.allow())))) .build(); Assertions.assertNull(roleWithoutFields.properties()); } diff --git a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java index 6d11e2ceef..9c21176aa9 100644 --- a/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java +++ b/core/src/test/java/com/datastrato/gravitino/proto/TestEntityProtoSerDe.java @@ -381,10 +381,14 @@ public void testEntitiesSerDe() throws IOException { Namespace.of("metalake", Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.ROLE_SCHEMA_NAME); Long roleId = 1L; String roleName = "testRole"; + String anotherCatalogName = "anotheCatalog"; SecurableObject securableObject = SecurableObjects.ofCatalog( catalogName, Lists.newArrayList(Privileges.UseCatalog.allow(), Privileges.DropCatalog.deny())); + SecurableObject anotherSecurableObject = + SecurableObjects.ofCatalog( + anotherCatalogName, Lists.newArrayList(Privileges.UseCatalog.allow())); RoleEntity roleEntity = RoleEntity.builder() @@ -392,7 +396,7 @@ public void testEntitiesSerDe() throws IOException { .withName(roleName) .withNamespace(roleNamespace) .withAuditInfo(auditInfo) - .withSecurableObject(securableObject) + .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) .withProperties(props) .build(); byte[] roleBytes = protoEntitySerDe.serialize(roleEntity); @@ -406,7 +410,7 @@ public void testEntitiesSerDe() throws IOException { .withName(roleName) .withNamespace(roleNamespace) .withAuditInfo(auditInfo) - .withSecurableObject(securableObject) + .withSecurableObjects(Lists.newArrayList(securableObject, anotherSecurableObject)) .build(); roleBytes = protoEntitySerDe.serialize(roleWithoutFields); roleFromBytes = protoEntitySerDe.deserialize(roleBytes, RoleEntity.class, roleNamespace); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index 2dbb832c21..7fcd72ee63 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -492,6 +492,8 @@ public void testAuthorizationEntityDelete(String type) throws IOException { BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); store.put(metalake); + CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo); + store.put(catalog); UserEntity oneUser = createUser(1L, "metalake", "oneUser", auditInfo); store.put(oneUser); UserEntity anotherUser = createUser(2L, "metalake", "anotherUser", auditInfo); @@ -510,6 +512,8 @@ public void testAuthorizationEntityDelete(String type) throws IOException { Assertions.assertTrue(store.exists(anotherGroup.nameIdentifier(), Entity.EntityType.GROUP)); Assertions.assertTrue(store.exists(oneRole.nameIdentifier(), Entity.EntityType.ROLE)); Assertions.assertTrue(store.exists(anotherRole.nameIdentifier(), Entity.EntityType.ROLE)); + + store.delete(catalog.nameIdentifier(), Entity.EntityType.CATALOG); store.delete(metalake.nameIdentifier(), Entity.EntityType.METALAKE); Assertions.assertFalse(store.exists(oneUser.nameIdentifier(), Entity.EntityType.USER)); Assertions.assertFalse(store.exists(anotherUser.nameIdentifier(), Entity.EntityType.USER)); @@ -1270,7 +1274,7 @@ private static RoleEntity createRole(Long id, String metalake, String name, Audi .withNamespace(AuthorizationUtils.ofRoleNamespace(metalake)) .withName(name) .withAuditInfo(auditInfo) - .withSecurableObject(securableObject) + .withSecurableObjects(Lists.newArrayList(securableObject)) .withProperties(null) .build(); } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java b/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java index 520a1e05ee..c3f916c603 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/memory/TestMemoryEntityStore.java @@ -228,9 +228,10 @@ public void testEntityStoreAndRetrieve() throws Exception { .withName("role") .withNamespace(AuthorizationUtils.ofRoleNamespace("metalake")) .withAuditInfo(auditInfo) - .withSecurableObject( - SecurableObjects.ofCatalog( - "catalog", Lists.newArrayList(Privileges.UseCatalog.allow()))) + .withSecurableObjects( + Lists.newArrayList( + SecurableObjects.ofCatalog( + "catalog", Lists.newArrayList(Privileges.UseCatalog.allow())))) .build(); InMemoryEntityStore store = new InMemoryEntityStore(); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index d9cb21d079..60e822ad64 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -477,7 +477,8 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace("metalake"), "role", - auditInfo); + auditInfo, + "catalog"); backend.insert(role, false); UserEntity user = @@ -554,7 +555,8 @@ public void testMetaLifeCycleFromCreationToDeletion() throws IOException { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace("another-metalake"), "another-role", - auditInfo); + auditInfo, + "another-catalog"); backend.insert(anotherRole, false); UserEntity anotherUser = @@ -914,9 +916,9 @@ public static UserEntity createUserEntity( } public static RoleEntity createRoleEntity( - Long id, Namespace namespace, String name, AuditInfo auditInfo) { + Long id, Namespace namespace, String name, AuditInfo auditInfo, String catalogName) { SecurableObject securableObject = - SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.UseCatalog.allow())); + SecurableObjects.ofCatalog(catalogName, Lists.newArrayList(Privileges.UseCatalog.allow())); return RoleEntity.builder() .withId(id) @@ -924,7 +926,7 @@ public static RoleEntity createRoleEntity( .withNamespace(namespace) .withProperties(null) .withAuditInfo(auditInfo) - .withSecurableObject(securableObject) + .withSecurableObjects(Lists.newArrayList(securableObject)) .build(); } @@ -965,13 +967,24 @@ public static RoleEntity createRoleEntity( SecurableObject securableObject, Map properties) { + return createRoleEntity( + id, namespace, name, auditInfo, Lists.newArrayList(securableObject), properties); + } + + public static RoleEntity createRoleEntity( + Long id, + Namespace namespace, + String name, + AuditInfo auditInfo, + List securableObjects, + Map properties) { return RoleEntity.builder() .withId(id) .withName(name) .withProperties(properties) .withNamespace(namespace) .withAuditInfo(auditInfo) - .withSecurableObject(securableObject) + .withSecurableObjects(securableObjects) .build(); } } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java index a5ce83434a..c18a5dca8c 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestGroupMetaService.java @@ -5,11 +5,13 @@ package com.datastrato.gravitino.storage.relational.service; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.AuthorizationUtils; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.storage.RandomIdGenerator; @@ -43,6 +45,10 @@ void getGroupByIdentifier() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -70,13 +76,15 @@ void getGroupByIdentifier() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); GroupEntity group2 = @@ -101,6 +109,10 @@ void insertGroup() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -148,13 +160,15 @@ void insertGroup() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); GroupEntity group2 = @@ -206,7 +220,8 @@ void insertGroup() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role3, false); GroupEntity group3Overwrite = createGroupEntity( @@ -246,6 +261,10 @@ void deleteGroup() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -273,13 +292,15 @@ void deleteGroup() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); GroupEntity group2 = @@ -317,6 +338,10 @@ void updateGroup() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -326,13 +351,15 @@ void updateGroup() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); GroupEntity group1 = @@ -354,7 +381,8 @@ void updateGroup() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role3, false); // update group (grant) @@ -438,7 +466,8 @@ void updateGroup() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role4", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role4, false); // update group (grant & revoke) @@ -524,6 +553,10 @@ void deleteMetalake() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -533,19 +566,22 @@ void deleteMetalake() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); RoleEntity role3 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); roleMetaService.insertRole(role3, false); @@ -575,6 +611,8 @@ void deleteMetalake() { group2.name(), groupMetaService.getGroupByIdentifier(group2.nameIdentifier()).name()); Assertions.assertEquals(1, roleMetaService.listRolesByGroupId(group2.id()).size()); + Assertions.assertTrue( + CatalogMetaService.getInstance().deleteCatalog(catalog.nameIdentifier(), false)); // delete metalake without cascade Assertions.assertTrue( MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), false)); @@ -596,6 +634,10 @@ void deleteMetalakeCascade() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -605,19 +647,22 @@ void deleteMetalakeCascade() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); RoleEntity role3 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); roleMetaService.insertRole(role3, false); @@ -668,6 +713,10 @@ void deleteGroupMetasByLegacyTimeLine() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -677,13 +726,15 @@ void deleteGroupMetasByLegacyTimeLine() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestRoleMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestRoleMetaService.java index 7e9a826793..666bf5cfcd 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestRoleMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestRoleMetaService.java @@ -5,13 +5,16 @@ package com.datastrato.gravitino.storage.relational.service; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.AuthorizationUtils; import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObject; import com.datastrato.gravitino.authorization.SecurableObjects; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.GroupEntity; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.UserEntity; @@ -47,6 +50,18 @@ void getRoleByIdentifier() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); + + CatalogEntity anotherCatalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake"), + "anotherCatalog", + auditInfo); + backend.insert(anotherCatalog, false); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -78,9 +93,32 @@ void insertRole() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); + CatalogEntity anotherCatalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake"), + "anotherCatalog", + auditInfo); + backend.insert(anotherCatalog, false); + CatalogEntity overwriteCatalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake"), + "catalogOverwrite", + auditInfo); + backend.insert(overwriteCatalog, false); RoleMetaService roleMetaService = RoleMetaService.getInstance(); + SecurableObject catalogObject = + SecurableObjects.ofCatalog( + "catalog", + Lists.newArrayList(Privileges.UseCatalog.allow(), Privileges.DropCatalog.deny())); + // insert role RoleEntity role1 = createRoleEntity( @@ -88,8 +126,10 @@ void insertRole() { AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", auditInfo, - SecurableObjects.ofCatalog( - "catalog", Lists.newArrayList(Privileges.UseCatalog.allow())), + Lists.newArrayList( + catalogObject, + SecurableObjects.ofCatalog( + "anotherCatalog", Lists.newArrayList(Privileges.UseCatalog.allow()))), ImmutableMap.of("k1", "v1")); Assertions.assertThrows( NoSuchEntityException.class, @@ -135,6 +175,10 @@ void deleteRole() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); GroupMetaService groupMetaService = GroupMetaService.getInstance(); @@ -246,6 +290,11 @@ void deleteMetalake() { createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); + UserMetaService userMetaService = UserMetaService.getInstance(); GroupMetaService groupMetaService = GroupMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -255,13 +304,15 @@ void deleteMetalake() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); GroupEntity group1 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), @@ -307,6 +358,9 @@ void deleteMetalake() { Assertions.assertEquals(1, groupRole1Rels.size()); Assertions.assertEquals(1, groupRole2Rels.size()); + Assertions.assertTrue( + CatalogMetaService.getInstance().deleteCatalog(catalog.nameIdentifier(), false)); + Assertions.assertTrue( MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), false)); @@ -348,6 +402,10 @@ void deleteMetalakeCascade() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); GroupMetaService groupMetaService = GroupMetaService.getInstance(); @@ -358,13 +416,15 @@ void deleteMetalakeCascade() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); GroupEntity group1 = createGroupEntity( RandomIdGenerator.INSTANCE.nextId(), @@ -451,6 +511,10 @@ void deleteRoleMetasByLegacyTimeLine() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); GroupMetaService groupMetaService = GroupMetaService.getInstance(); @@ -461,13 +525,15 @@ void deleteRoleMetasByLegacyTimeLine() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestSecurableObjects.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestSecurableObjects.java new file mode 100644 index 0000000000..f7dbc03d17 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestSecurableObjects.java @@ -0,0 +1,113 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.authorization.AuthorizationUtils; +import com.datastrato.gravitino.authorization.Privileges; +import com.datastrato.gravitino.authorization.SecurableObject; +import com.datastrato.gravitino.authorization.SecurableObjects; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; +import com.datastrato.gravitino.meta.FilesetEntity; +import com.datastrato.gravitino.meta.RoleEntity; +import com.datastrato.gravitino.meta.SchemaEntity; +import com.datastrato.gravitino.meta.TableEntity; +import com.datastrato.gravitino.meta.TopicEntity; +import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.relational.TestJDBCBackend; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import java.time.Instant; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestSecurableObjects extends TestJDBCBackend { + RoleMetaService roleMetaService = RoleMetaService.getInstance(); + + @Test + public void testAllTypeSecurableObjects() { + String metalakeName = "metalake"; + AuditInfo auditInfo = + AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + BaseMetalake metalake = + createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); + backend.insert(metalake, false); + + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of("metalake"), "catalog", auditInfo); + backend.insert(catalog, false); + + SchemaEntity schema = + createSchemaEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake", "catalog"), + "schema", + auditInfo); + backend.insert(schema, false); + + FilesetEntity fileset = + createFilesetEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake", "catalog", "schema"), + "fileset", + auditInfo); + backend.insert(fileset, false); + TableEntity table = + createTableEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake", "catalog", "schema"), + "table", + auditInfo); + backend.insert(table, false); + TopicEntity topic = + createTopicEntity( + RandomIdGenerator.INSTANCE.nextId(), + Namespace.of("metalake", "catalog", "schema"), + "topic", + auditInfo); + backend.insert(topic, false); + + SecurableObject catalogObject = + SecurableObjects.ofCatalog( + "catalog", + Lists.newArrayList(Privileges.UseCatalog.allow(), Privileges.DropCatalog.deny())); + + SecurableObject schemaObject = + SecurableObjects.ofSchema( + catalogObject, "schema", Lists.newArrayList(Privileges.UseSchema.allow())); + SecurableObject tableObject = + SecurableObjects.ofTable( + schemaObject, "table", Lists.newArrayList(Privileges.ReadTable.allow())); + SecurableObject filesetObject = + SecurableObjects.ofFileset( + schemaObject, "fileset", Lists.newArrayList(Privileges.ReadFileset.allow())); + SecurableObject topicObject = + SecurableObjects.ofTopic( + schemaObject, "topic", Lists.newArrayList(Privileges.ReadTopic.deny())); + SecurableObject allMetalakesObject = + SecurableObjects.ofAllMetalakes(Lists.newArrayList(Privileges.UseMetalake.allow())); + + RoleEntity role1 = + createRoleEntity( + RandomIdGenerator.INSTANCE.nextId(), + AuthorizationUtils.ofRoleNamespace(metalakeName), + "role1", + auditInfo, + Lists.newArrayList( + catalogObject, + schemaObject, + tableObject, + filesetObject, + topicObject, + allMetalakesObject), + ImmutableMap.of("k1", "v1")); + + Assertions.assertDoesNotThrow(() -> roleMetaService.insertRole(role1, false)); + Assertions.assertEquals(role1, roleMetaService.getRoleByIdentifier(role1.nameIdentifier())); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java index 8c0ed352fa..cda2e6dc62 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/service/TestUserMetaService.java @@ -5,11 +5,13 @@ package com.datastrato.gravitino.storage.relational.service; +import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.authorization.AuthorizationUtils; import com.datastrato.gravitino.exceptions.AlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; +import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.meta.RoleEntity; import com.datastrato.gravitino.meta.UserEntity; import com.datastrato.gravitino.storage.RandomIdGenerator; @@ -43,6 +45,10 @@ void getUserByIdentifier() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -69,13 +75,15 @@ void getUserByIdentifier() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); UserEntity user2 = @@ -100,6 +108,10 @@ void insertUser() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -147,13 +159,15 @@ void insertUser() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); UserEntity user2 = @@ -205,7 +219,8 @@ void insertUser() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role3, false); UserEntity user3Overwrite = createUserEntity( @@ -245,6 +260,10 @@ void deleteUser() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -272,13 +291,15 @@ void deleteUser() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); UserEntity user2 = @@ -316,6 +337,10 @@ void updateUser() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -325,13 +350,15 @@ void updateUser() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); UserEntity user1 = @@ -353,7 +380,8 @@ void updateUser() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role3, false); // update user (grant) @@ -437,7 +465,8 @@ void updateUser() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role4", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role4, false); // update user (grant & revoke) @@ -522,6 +551,10 @@ void deleteMetalake() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -531,19 +564,22 @@ void deleteMetalake() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); RoleEntity role3 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); roleMetaService.insertRole(role3, false); @@ -573,6 +609,8 @@ void deleteMetalake() { user2.name(), userMetaService.getUserByIdentifier(user2.nameIdentifier()).name()); Assertions.assertEquals(1, roleMetaService.listRolesByUserId(user2.id()).size()); + Assertions.assertTrue( + CatalogMetaService.getInstance().deleteCatalog(catalog.nameIdentifier(), false)); // delete metalake without cascade Assertions.assertTrue( MetalakeMetaService.getInstance().deleteMetalake(metalake.nameIdentifier(), false)); @@ -594,6 +632,10 @@ void deleteMetalakeCascade() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -603,19 +645,22 @@ void deleteMetalakeCascade() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); RoleEntity role3 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role3", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); roleMetaService.insertRole(role3, false); @@ -666,6 +711,10 @@ void deleteUserMetasByLegacyTimeLine() { BaseMetalake metalake = createBaseMakeLake(RandomIdGenerator.INSTANCE.nextId(), metalakeName, auditInfo); backend.insert(metalake, false); + CatalogEntity catalog = + createCatalog( + RandomIdGenerator.INSTANCE.nextId(), Namespace.of(metalakeName), "catalog", auditInfo); + backend.insert(catalog, false); UserMetaService userMetaService = UserMetaService.getInstance(); RoleMetaService roleMetaService = RoleMetaService.getInstance(); @@ -675,13 +724,15 @@ void deleteUserMetasByLegacyTimeLine() { RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role1", - auditInfo); + auditInfo, + "catalog"); RoleEntity role2 = createRoleEntity( RandomIdGenerator.INSTANCE.nextId(), AuthorizationUtils.ofRoleNamespace(metalakeName), "role2", - auditInfo); + auditInfo, + "catalog"); roleMetaService.insertRole(role1, false); roleMetaService.insertRole(role2, false); diff --git a/core/src/test/resources/h2/schema-h2.sql b/core/src/test/resources/h2/schema-h2.sql index 40cc0f358b..9a24733add 100644 --- a/core/src/test/resources/h2/schema-h2.sql +++ b/core/src/test/resources/h2/schema-h2.sql @@ -143,10 +143,6 @@ CREATE TABLE IF NOT EXISTS `role_meta` ( `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', - `securable_object_full_name` VARCHAR(256) NOT NULL COMMENT 'securable object full name', - `securable_object_type` VARCHAR(32) NOT NULL COMMENT 'securable object type', - `privileges` VARCHAR(64) NOT NULL COMMENT 'securable privileges', - `privilege_conditions` VARCHAR(64) NOT NULL COMMENT 'securable privilege conditions', `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', @@ -155,6 +151,20 @@ CREATE TABLE IF NOT EXISTS `role_meta` ( CONSTRAINT `uk_mid_rn_del` UNIQUE (`metalake_id`, `role_name`, `deleted_at`) ) ENGINE=InnoDB; +CREATE TABLE IF NOT EXISTS `role_meta_securable_object` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `entity_id` BIGINT(20) NOT NULL COMMENT 'securable object entity id', + `type` VARCHAR(128) NOT NULL COMMENT 'securable object type', + `privilege_names` VARCHAR(256) NOT NULL COMMENT 'securable object privilege names', + `privilege_conditions` VARCHAR(256) NOT NULL COMMENT 'securable object privilege conditions', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable objectcurrent version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable object deleted at', + PRIMARY KEY (`id`), + KEY `idx_obj_rid` (`role_id`) + ) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS `user_role_rel` ( `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', diff --git a/docs/how-to-use-relational-backend-storage.md b/docs/how-to-use-relational-backend-storage.md index 3d6900cd13..c0e2416936 100644 --- a/docs/how-to-use-relational-backend-storage.md +++ b/docs/how-to-use-relational-backend-storage.md @@ -43,8 +43,8 @@ ${GRAVITINO_HOME}/scripts/mysql/ ``` The script name is like `schema-{version}-mysql.sql`, and the `version` depends on your Gravitino version. -For example, if your Gravitino version is `0.6.0`, then you can choose the **latest version** script -file that is equal or smaller than `0.6.0`, you can choose the `schema-0.5.0-mysql.sql` script. +For example, if your Gravitino version is `0.6.0`, then you can choose the **latest version** script. +If you used a legacy script, you can use `upgrade-{old version}-to-{new version}-mysql.sql` to upgrade the schema. ### Step 2: Initialize the database diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 051e99f3ab..0ab5d7804a 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -157,7 +157,7 @@ private static void setMySQLBackend() { new File( gravitinoHome + String.format( - "/scripts/mysql/schema-%s-mysql.sql", ConfigConstants.VERSION_0_5_0)), + "/scripts/mysql/schema-%s-mysql.sql", ConfigConstants.VERSION_0_6_0)), "UTF-8"); String[] initMySQLBackendSqls = mysqlContent.split(";"); initMySQLBackendSqls = ArrayUtils.addFirst(initMySQLBackendSqls, "use " + META_DATA + ";"); diff --git a/meta/src/main/proto/gravitino_meta.proto b/meta/src/main/proto/gravitino_meta.proto index e26b4fef41..61ae666b33 100644 --- a/meta/src/main/proto/gravitino_meta.proto +++ b/meta/src/main/proto/gravitino_meta.proto @@ -118,13 +118,17 @@ message Group { AuditInfo audit_info = 5; } +message SecurableObject { + string full_name = 1; + string type = 2; + repeated string privilege_names = 3; + repeated string privilege_conditions = 4; +} + message Role { uint64 id = 1; string name = 2; - repeated string privileges = 3; - repeated string privilege_conditions = 4; - string securable_object_full_name = 5; - string securable_object_type = 6; - map properties = 7; - AuditInfo audit_info = 8; + repeated SecurableObject securable_objects = 3; + map properties = 4; + AuditInfo audit_info = 5; } diff --git a/scripts/mysql/schema-0.6.0-mysql.sql b/scripts/mysql/schema-0.6.0-mysql.sql new file mode 100644 index 0000000000..0f699017db --- /dev/null +++ b/scripts/mysql/schema-0.6.0-mysql.sql @@ -0,0 +1,196 @@ +-- +-- Copyright 2024 Datastrato Pvt Ltd. +-- This software is licensed under the Apache License version 2. +-- + +CREATE TABLE IF NOT EXISTS `metalake_meta` ( + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', + `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake deleted at', + PRIMARY KEY (`metalake_id`), + UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; + +CREATE TABLE IF NOT EXISTS `catalog_meta` ( + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', + `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider', + `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog deleted at', + PRIMARY KEY (`catalog_id`), + UNIQUE KEY `uk_mid_cn_del` (`metalake_id`, `catalog_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'catalog metadata'; + +CREATE TABLE IF NOT EXISTS `schema_meta` ( + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `schema_name` VARCHAR(128) NOT NULL COMMENT 'schema name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_comment` VARCHAR(256) DEFAULT '' COMMENT 'schema comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'schema audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'schema deleted at', + PRIMARY KEY (`schema_id`), + UNIQUE KEY `uk_cid_sn_del` (`catalog_id`, `schema_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'schema metadata'; + +CREATE TABLE IF NOT EXISTS `table_meta` ( + `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id', + `table_name` VARCHAR(128) NOT NULL COMMENT 'table name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'table audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'table deleted at', + PRIMARY KEY (`table_id`), + UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `table_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table metadata'; + +CREATE TABLE IF NOT EXISTS `fileset_meta` ( + `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', + `fileset_name` VARCHAR(128) NOT NULL COMMENT 'fileset name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `type` VARCHAR(64) NOT NULL COMMENT 'fileset type', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'fileset audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', + PRIMARY KEY (`fileset_id`), + UNIQUE KEY `uk_sid_fn_del` (`schema_id`, `fileset_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'fileset metadata'; + +CREATE TABLE IF NOT EXISTS `fileset_version_info` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', + `version` INT UNSIGNED NOT NULL COMMENT 'fileset info version', + `fileset_comment` VARCHAR(256) DEFAULT '' COMMENT 'fileset comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset properties', + `storage_location` MEDIUMTEXT NOT NULL COMMENT 'fileset storage location', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_fid_ver_del` (`fileset_id`, `version`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`), + KEY `idx_sid` (`schema_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'fileset version info'; + +CREATE TABLE IF NOT EXISTS `topic_meta` ( + `topic_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'topic id', + `topic_name` VARCHAR(128) NOT NULL COMMENT 'topic name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `comment` VARCHAR(256) DEFAULT '' COMMENT 'topic comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'topic properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'topic audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'topic deleted at', + PRIMARY KEY (`topic_id`), + UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `topic_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'topic metadata'; + +CREATE TABLE IF NOT EXISTS `user_meta` ( + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `user_name` VARCHAR(128) NOT NULL COMMENT 'username', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted at', + PRIMARY KEY (`user_id`), + UNIQUE KEY `uk_mid_us_del` (`metalake_id`, `user_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user metadata'; + +CREATE TABLE IF NOT EXISTS `role_meta` ( + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted at', + PRIMARY KEY (`role_id`), + UNIQUE KEY `uk_mid_rn_del` (`metalake_id`, `role_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'role metadata'; + +CREATE TABLE IF NOT EXISTS `role_meta_securable_object` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `entity_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'The entity id of securable object', + `type` VARCHAR(128) NOT NULL COMMENT 'securable object type', + `privilege_names` VARCHAR(256) NOT NULL COMMENT 'securable object privilege names', + `privilege_conditions` VARCHAR(256) NOT NULL COMMENT 'securable object privilege conditions', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable objectcurrent version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable object deleted at', + PRIMARY KEY (`id`), + KEY `idx_obj_rid` (`role_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'securable object meta'; + +CREATE TABLE IF NOT EXISTS `user_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_ui_ri_del` (`user_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`role_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user role relation'; + +CREATE TABLE IF NOT EXISTS `group_meta` ( + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `group_name` VARCHAR(128) NOT NULL COMMENT 'group name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted at', + PRIMARY KEY (`group_id`), + UNIQUE KEY `uk_mid_gr_del` (`metalake_id`, `group_name`, `deleted_at`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group metadata'; + +CREATE TABLE IF NOT EXISTS `group_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_gi_ri_del` (`group_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`group_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group role relation'; \ No newline at end of file diff --git a/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql b/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql new file mode 100644 index 0000000000..4ab49771cb --- /dev/null +++ b/scripts/mysql/upgrade-0.5.0-to-0.6.0-mysql.sql @@ -0,0 +1,22 @@ +-- +-- Copyright 2024 Datastrato Pvt Ltd. +-- This software is licensed under the Apache License version 2. +-- +ALTER TABLE `role_meta` MODIFY COLUMN `securable_object_full_name` VARCHAR(256) NOT NULL DEFAULT '' COMMENT 'deprecated'; +ALTER TABLE `role_meta` MODIFY COLUMN `securable_object_type` VARCHAR(32) NOT NULL DEFAULT '' COMMENT 'deprecated'; +ALTER TABLE `role_meta` MODIFY COLUMN `privileges` VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'deprecated'; +ALTER TABLE `role_meta` MODIFY COLUMN `privilege_conditions` VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'deprecated'; + +CREATE TABLE IF NOT EXISTS `role_meta_securable_object` ( + `id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'securable object id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `entity_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'The entity id of securable object', + `type` VARCHAR(128) NOT NULL COMMENT 'securable object type', + `privilege_names` VARCHAR(256) NOT NULL COMMENT 'securable object privilege names', + `privilege_conditions` VARCHAR(256) NOT NULL COMMENT 'securable object privilege conditions', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable objectcurrent version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable object deleted at', + PRIMARY KEY (`id`), + KEY `idx_obj_rid` (`role_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'securable object meta'; diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java index 147fa42aba..3d455b041e 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/RoleOperations.java @@ -18,7 +18,8 @@ import com.datastrato.gravitino.dto.util.DTOConverters; import com.datastrato.gravitino.metrics.MetricNames; import com.datastrato.gravitino.server.web.Utils; -import com.google.common.base.Preconditions; +import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; @@ -68,32 +69,29 @@ public Response getRole(@PathParam("metalake") String metalake, @PathParam("role @ResponseMetered(name = "create-role", absolute = true) public Response createRole(@PathParam("metalake") String metalake, RoleCreateRequest request) { try { - // TODO: Supports multiple securable objects. There will be some limited support for multiple - // securable objects in the future. - // The securable objects in the same role should under the same catalog. - // If a role contains a metalake securable object, the role can't contain any other securable - // object. - Preconditions.checkArgument( - request.getSecurableObjects() != null && request.getSecurableObjects().length == 1, - "The size of securable objects must be 1"); - return Utils.doAs( httpRequest, () -> { - SecurableObject securableObject = - SecurableObjects.parse( - request.getSecurableObjects()[0].fullName(), - request.getSecurableObjects()[0].type(), - request.getSecurableObjects()[0].privileges().stream() - .map( - privilege -> { - if (privilege.condition().equals(Privilege.Condition.ALLOW)) { - return Privileges.allow(privilege.name()); - } else { - return Privileges.deny(privilege.name()); - } - }) - .collect(Collectors.toList())); + List securableObjects = + Arrays.stream(request.getSecurableObjects()) + .map( + securableObjectDTO -> + SecurableObjects.parse( + securableObjectDTO.fullName(), + securableObjectDTO.type(), + securableObjectDTO.privileges().stream() + .map( + privilege -> { + if (privilege + .condition() + .equals(Privilege.Condition.ALLOW)) { + return Privileges.allow(privilege.name()); + } else { + return Privileges.deny(privilege.name()); + } + }) + .collect(Collectors.toList()))) + .collect(Collectors.toList()); return Utils.ok( new RoleResponse( @@ -102,7 +100,7 @@ public Response createRole(@PathParam("metalake") String metalake, RoleCreateReq metalake, request.getName(), request.getProperties(), - securableObject)))); + securableObjects)))); }); } catch (Exception e) { diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java index c32e201e85..e17977b0fb 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestRoleOperations.java @@ -101,12 +101,17 @@ protected void configure() { public void testCreateRole() { SecurableObject securableObject = SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.UseCatalog.allow())); + SecurableObject anotherSecurableObject = + SecurableObjects.ofCatalog( + "another_catalog", Lists.newArrayList(Privileges.DropCatalog.deny())); RoleCreateRequest req = new RoleCreateRequest( "role", Collections.emptyMap(), - new SecurableObjectDTO[] {DTOConverters.toDTO(securableObject)}); + new SecurableObjectDTO[] { + DTOConverters.toDTO(securableObject), DTOConverters.toDTO(anotherSecurableObject) + }); Role role = buildRole("role1"); when(manager.createRole(any(), any(), any(), any())).thenReturn(role); @@ -125,6 +130,19 @@ public void testCreateRole() { RoleDTO roleDTO = roleResponse.getRole(); Assertions.assertEquals("role1", roleDTO.name()); + Assertions.assertEquals( + SecurableObjects.ofCatalog( + "another_catalog", Lists.newArrayList(Privileges.DropCatalog.deny())) + .fullName(), + roleDTO.securableObjects().get(1).fullName()); + Assertions.assertEquals(1, roleDTO.securableObjects().get(1).privileges().size()); + Assertions.assertEquals( + Privileges.DropCatalog.deny().name(), + roleDTO.securableObjects().get(1).privileges().get(0).name()); + Assertions.assertEquals( + Privileges.UseCatalog.deny().condition(), + roleDTO.securableObjects().get(1).privileges().get(0).condition()); + Assertions.assertEquals( SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.UseCatalog.allow())) .fullName(), @@ -267,11 +285,15 @@ public void testGetRole() { private Role buildRole(String role) { SecurableObject catalog = SecurableObjects.ofCatalog("catalog", Lists.newArrayList(Privileges.UseCatalog.allow())); + SecurableObject anotherSecurableObject = + SecurableObjects.ofCatalog( + "another_catalog", Lists.newArrayList(Privileges.DropCatalog.deny())); + return RoleEntity.builder() .withId(1L) .withName(role) .withProperties(Collections.emptyMap()) - .withSecurableObject(catalog) + .withSecurableObjects(Lists.newArrayList(catalog, anotherSecurableObject)) .withAuditInfo( AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build()) .build();