Merge branch 'master' into Datahub-7733

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -1634,6 +1634,9 @@ private void configurePolicyResolvers(final RuntimeWiring.Builder builder) {
         })).dataFetcher("resolvedRoles", new LoadableTypeBatchResolver<>(dataHubRoleType, (env) -> {
             final ActorFilter filter = env.getSource();
             return filter.getRoles();
+        })).dataFetcher("resolvedOwnershipTypes", new LoadableTypeBatchResolver<>(ownershipType, (env) -> {
+            final ActorFilter filter = env.getSource();
+            return filter.getResourceOwnersTypes();
         })));
     }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyInfoPolicyMapper.java

@@ -1,14 +1,15 @@
 package com.linkedin.datahub.graphql.resolvers.policy.mappers;
 
+import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.Urn;
-import com.linkedin.datahub.graphql.generated.ActorFilter;
 import com.linkedin.datahub.graphql.generated.Policy;
 import com.linkedin.datahub.graphql.generated.PolicyMatchCondition;
 import com.linkedin.datahub.graphql.generated.PolicyMatchCriterion;
 import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue;
 import com.linkedin.datahub.graphql.generated.PolicyMatchFilter;
 import com.linkedin.datahub.graphql.generated.PolicyState;
 import com.linkedin.datahub.graphql.generated.PolicyType;
+import com.linkedin.datahub.graphql.generated.ActorFilter;
 import com.linkedin.datahub.graphql.generated.ResourceFilter;
 import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
 import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
@@ -53,6 +54,10 @@ private ActorFilter mapActors(final DataHubActorFilter actorFilter) {
     result.setAllGroups(actorFilter.isAllGroups());
     result.setAllUsers(actorFilter.isAllUsers());
     result.setResourceOwners(actorFilter.isResourceOwners());
+    UrnArray resourceOwnersTypes = actorFilter.getResourceOwnersTypes();
+    if (resourceOwnersTypes != null) {
+      result.setResourceOwnersTypes(resourceOwnersTypes.stream().map(Urn::toString).collect(Collectors.toList()));
+    }
     if (actorFilter.hasGroups()) {
       result.setGroups(actorFilter.getGroups().stream().map(Urn::toString).collect(Collectors.toList()));
     }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyUpdateInputInfoMapper.java

@@ -51,6 +51,9 @@ private DataHubActorFilter mapActors(final ActorFilterInput actorInput) {
     result.setAllGroups(actorInput.getAllGroups());
     result.setAllUsers(actorInput.getAllUsers());
     result.setResourceOwners(actorInput.getResourceOwners());
+    if (actorInput.getResourceOwnersTypes() != null) {
+      result.setResourceOwnersTypes(new UrnArray(actorInput.getResourceOwnersTypes().stream().map(this::createUrn).collect(Collectors.toList())));
+    }
     if (actorInput.getGroups() != null) {
       result.setGroups(new UrnArray(actorInput.getGroups().stream().map(this::createUrn).collect(Collectors.toList())));
     }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/policy/DataHubPolicyMapper.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.graphql.types.policy;
 
+import com.linkedin.common.UrnArray;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.data.DataMap;
 import com.linkedin.datahub.graphql.generated.ActorFilter;
@@ -67,6 +68,11 @@ private ActorFilter mapActors(final DataHubActorFilter actorFilter) {
     result.setAllGroups(actorFilter.isAllGroups());
     result.setAllUsers(actorFilter.isAllUsers());
     result.setResourceOwners(actorFilter.isResourceOwners());
+    // Change here is not executed at the moment - leaving it for the future
+    UrnArray resourceOwnersTypes = actorFilter.getResourceOwnersTypes();
+    if (resourceOwnersTypes != null) {
+      result.setResourceOwnersTypes(resourceOwnersTypes.stream().map(Urn::toString).collect(Collectors.toList()));
+    }
     if (actorFilter.hasGroups()) {
       result.setGroups(actorFilter.getGroups().stream().map(Urn::toString).collect(Collectors.toList()));
     }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -8007,6 +8007,16 @@ type ActorFilter {
   """
   resourceOwners: Boolean!
 
+  """
+  Set of OwnershipTypes to apply the policy to (if resourceOwners field is set to True)
+  """
+  resourceOwnersTypes: [String!]
+
+  """
+  Set of OwnershipTypes to apply the policy to (if resourceOwners field is set to True), resolved.
+  """
+  resolvedOwnershipTypes: [OwnershipTypeEntity!]
+
   """
   Whether the filter should apply to all users
   """
@@ -8135,6 +8145,11 @@ input ActorFilterInput {
   """
   resourceOwners: Boolean!
 
+  """
+  Set of OwnershipTypes to apply the policy to (if resourceOwners field is set to True)
+  """
+  resourceOwnersTypes: [String!]
+
   """
   Whether the filter should apply to all users
   """

datahub-web-react/src/app/permissions/policy/ManagePolicies.tsx

@@ -109,6 +109,7 @@ const toPolicyInput = (policy: Omit<Policy, 'urn'>): PolicyUpdateInput => {
             allUsers: policy.actors.allUsers,
             allGroups: policy.actors.allGroups,
             resourceOwners: policy.actors.resourceOwners,
+            resourceOwnersTypes: policy.actors.resourceOwnersTypes,
         },
     };
     if (policy.resources !== null && policy.resources !== undefined) {

datahub-web-react/src/app/permissions/policy/PolicyActorForm.tsx

@@ -1,10 +1,12 @@
 import React from 'react';
 import { Form, Select, Switch, Tag, Typography } from 'antd';
 import styled from 'styled-components';
+import { Maybe } from 'graphql/jsutils/Maybe';
 
 import { useEntityRegistry } from '../../useEntityRegistry';
 import { ActorFilter, CorpUser, EntityType, PolicyType, SearchResult } from '../../../types.generated';
 import { useGetSearchResultsLazyQuery } from '../../../graphql/search.generated';
+import { useListOwnershipTypesQuery } from '../../../graphql/ownership.generated';
 import { CustomAvatar } from '../../shared/avatar';
 
 type Props = {
@@ -36,6 +38,10 @@ const SearchResultContent = styled.div`
     align-items: center;
 `;
 
+const OwnershipWrapper = styled.div`
+    margin-top: 12px;
+`;
+
 /**
  * Component used to construct the "actors" portion of a DataHub
  * access Policy by populating an ActorFilter object.
@@ -46,12 +52,37 @@ export default function PolicyActorForm({ policyType, actors, setActors }: Props
     // Search for actors while building policy.
     const [userSearch, { data: userSearchData }] = useGetSearchResultsLazyQuery();
     const [groupSearch, { data: groupSearchData }] = useGetSearchResultsLazyQuery();
-
+    const { data: ownershipData } = useListOwnershipTypesQuery({
+        variables: {
+            input: {},
+        },
+    });
+    const ownershipTypes =
+        ownershipData?.listOwnershipTypes?.ownershipTypes.filter((type) => type.urn !== 'urn:li:ownershipType:none') ||
+        [];
+    const ownershipTypesMap = Object.fromEntries(ownershipTypes.map((type) => [type.urn, type.info?.name]));
     // Toggle the "Owners" switch
     const onToggleAppliesToOwners = (value: boolean) => {
         setActors({
             ...actors,
             resourceOwners: value,
+            resourceOwnersTypes: value ? actors.resourceOwnersTypes : null,
+        });
+    };
+
+    const onSelectOwnershipTypeActor = (newType: string) => {
+        const newResourceOwnersTypes: Maybe<string[]> = [...(actors.resourceOwnersTypes || []), newType];
+        setActors({
+            ...actors,
+            resourceOwnersTypes: newResourceOwnersTypes,
+        });
+    };
+
+    const onDeselectOwnershipTypeActor = (type: string) => {
+        const newResourceOwnersTypes: Maybe<string[]> = actors.resourceOwnersTypes?.filter((u: string) => u !== type);
+        setActors({
+            ...actors,
+            resourceOwnersTypes: newResourceOwnersTypes?.length ? newResourceOwnersTypes : null,
         });
     };
 
@@ -175,6 +206,7 @@ export default function PolicyActorForm({ policyType, actors, setActors }: Props
     // Select dropdown values.
     const usersSelectValue = actors.allUsers ? ['All'] : actors.users || [];
     const groupsSelectValue = actors.allGroups ? ['All'] : actors.groups || [];
+    const ownershipTypesSelectValue = actors.resourceOwnersTypes || [];
 
     const tagRender = (props) => {
         // eslint-disable-next-line react/prop-types
@@ -215,6 +247,36 @@ export default function PolicyActorForm({ policyType, actors, setActors }: Props
                         selected privileges.
                     </Typography.Paragraph>
                     <Switch size="small" checked={actors.resourceOwners} onChange={onToggleAppliesToOwners} />
+                    {actors.resourceOwners && (
+                        <OwnershipWrapper>
+                            <Typography.Paragraph>
+                                List of types of ownership which will be used to match owners. If empty, the policies
+                                will applied to any type of ownership.
+                            </Typography.Paragraph>
+                            <Select
+                                value={ownershipTypesSelectValue}
+                                mode="multiple"
+                                placeholder="Ownership types"
+                                onSelect={(asset: any) => onSelectOwnershipTypeActor(asset)}
+                                onDeselect={(asset: any) => onDeselectOwnershipTypeActor(asset)}
+                                tagRender={(tagProps) => {
+                                    return (
+                                        <Tag closable={tagProps.closable} onClose={tagProps.onClose}>
+                                            {ownershipTypesMap[tagProps.value.toString()]}
+                                        </Tag>
+                                    );
+                                }}
+                            >
+                                {ownershipTypes.map((resOwnershipType) => {
+                                    return (
+                                        <Select.Option value={resOwnershipType.urn}>
+                                            {resOwnershipType?.info?.name}
+                                        </Select.Option>
+                                    );
+                                })}
+                            </Select>
+                        </OwnershipWrapper>
+                    )}
                 </Form.Item>
             )}
             <Form.Item label={<Typography.Text strong>Users</Typography.Text>}>

datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx

@@ -103,6 +103,22 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege
         );
     };
 
+    const resourceOwnersField = (actors) => {
+        if (!actors?.resourceOwners) {
+            return <PoliciesTag>No</PoliciesTag>;
+        }
+        if ((actors?.resolvedOwnershipTypes?.length ?? 0) > 0) {
+            return (
+                <div>
+                    {actors?.resolvedOwnershipTypes?.map((type) => (
+                        <PoliciesTag>{type.info.name}</PoliciesTag>
+                    ))}
+                </div>
+            );
+        }
+        return <PoliciesTag>Yes - All owners</PoliciesTag>;
+    };
+
     return (
         <Modal title={policy?.name} visible={visible} onCancel={onClose} closable width={800} footer={actionButtons}>
             <PolicyContainer>
@@ -180,7 +196,7 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege
                 <div>
                     <Typography.Title level={5}>Applies to Owners</Typography.Title>
                     <ThinDivider />
-                    <PoliciesTag>{policy?.actors?.resourceOwners ? 'True' : 'False'}</PoliciesTag>
+                    {resourceOwnersField(policy?.actors)}
                 </div>
                 <div>
                     <Typography.Title level={5}>Applies to Users</Typography.Title>

datahub-web-react/src/graphql/policy.graphql

@@ -34,6 +34,13 @@ query listPolicies($input: ListPoliciesInput!) {
                 allUsers
                 allGroups
                 resourceOwners
+                resourceOwnersTypes
+                resolvedOwnershipTypes {
+                    urn
+                    info {
+                        name
+                    }
+                }
                 resolvedUsers {
                     username
                     urn

metadata-ingestion/tests/integration/s3/test_s3.py

@@ -56,7 +56,7 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names):
             pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/"
         )
         for root, _dirs, files in os.walk(test_resources_dir):
-            for file in files:
+            for file in sorted(files):
                 full_path = os.path.join(root, file)
                 rel_path = os.path.relpath(full_path, test_resources_dir)
                 bkt.upload_file(full_path, rel_path)

metadata-models/src/main/pegasus/com/linkedin/policy/DataHubActorFilter.pdl

@@ -1,6 +1,7 @@
 namespace com.linkedin.policy
 
 import com.linkedin.common.Urn
+import com.linkedin.ownership.OwnershipTypeInfo
 
 /**
  * Information used to filter DataHub actors.
@@ -23,6 +24,11 @@ record DataHubActorFilter {
    */
   resourceOwners: boolean = false
 
+  /**
+   * Define type of ownership for the policy
+  */
+  resourceOwnersTypes: optional array[Urn]
+
   /**
    * Whether the filter should apply to all users.
    */

metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java

@@ -2,15 +2,19 @@
 
 import com.datahub.authentication.Authentication;
 import com.google.common.collect.ImmutableSet;
+import com.linkedin.common.Owner;
+import com.linkedin.common.Ownership;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.StringArray;
 import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
 import com.linkedin.entity.EnvelopedAspectMap;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.identity.GroupMembership;
 import com.linkedin.identity.NativeGroupMembership;
 import com.linkedin.identity.RoleMembership;
+import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.policy.DataHubActorFilter;
 import com.linkedin.policy.DataHubPolicyInfo;
@@ -28,6 +32,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javax.annotation.Nullable;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -307,11 +312,34 @@ private boolean isOwnerMatch(
     if (!actorFilter.isResourceOwners() || !requestResource.isPresent()) {
       return false;
     }
-    return isActorOwner(actor, requestResource.get(), context);
+    List<Urn> ownershipTypes = actorFilter.getResourceOwnersTypes();
+    return isActorOwner(actor, requestResource.get(), ownershipTypes, context);
   }
 
-  private boolean isActorOwner(Urn actor, ResolvedResourceSpec resourceSpec, PolicyEvaluationContext context) {
-    Set<String> owners = resourceSpec.getOwners();
+  private Set<String> getOwnersForType(ResourceSpec resourceSpec, List<Urn> ownershipTypes) {
+    Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource());
+    EnvelopedAspect ownershipAspect;
+    try {
+      EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn,
+              Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME), _systemAuthentication);
+      if (response == null || !response.getAspects().containsKey(Constants.OWNERSHIP_ASPECT_NAME)) {
+        return Collections.emptySet();
+      }
+      ownershipAspect = response.getAspects().get(Constants.OWNERSHIP_ASPECT_NAME);
+    } catch (Exception e) {
+      log.error("Error while retrieving ownership aspect for urn {}", entityUrn, e);
+      return Collections.emptySet();
+    }
+    Ownership ownership = new Ownership(ownershipAspect.getValue().data());
+    Stream<Owner> ownersStream = ownership.getOwners().stream();
+    if (ownershipTypes != null) {
+      ownersStream = ownersStream.filter(owner -> ownershipTypes.contains(owner.getTypeUrn()));
+    }
+    return ownersStream.map(owner -> owner.getOwner().toString()).collect(Collectors.toSet());
+  }
+
+  private boolean isActorOwner(Urn actor, ResolvedResourceSpec resourceSpec, List<Urn> ownershipTypes, PolicyEvaluationContext context) {
+    Set<String> owners = this.getOwnersForType(resourceSpec.getSpec(), ownershipTypes);
     if (isUserOwner(actor, owners)) {
       return true;
     }