diff --git a/google/cloud/forseti/common/gcp_api/cloud_resource_manager.py b/google/cloud/forseti/common/gcp_api/cloud_resource_manager.py index 5828558129..c205ebd65d 100755 --- a/google/cloud/forseti/common/gcp_api/cloud_resource_manager.py +++ b/google/cloud/forseti/common/gcp_api/cloud_resource_manager.py @@ -50,6 +50,7 @@ def __init__(self, self._organizations = None self._folders = None self._folders_v1 = None + self._liens = None super(CloudResourceManagerRepositoryClient, self).__init__( 'cloudresourcemanager', versions=['v1', 'v2'], @@ -91,6 +92,15 @@ def folders_v1(self): self._folders_v1 = self._init_repository( _ResourceManagerFolderV1Repository, version='v1') return self._folders_v1 + + @property + def liens(self): + """Returns a _ResourceManagerLiensRepository instance.""" + if not self._liens: + self._liens = self._init_repository( + _ResourceManagerLiensRepository) + return self._liens + # pylint: enable=missing-return-doc, missing-return-type-doc @@ -226,6 +236,22 @@ def __init__(self, **kwargs): max_results_field='pageSize', component='folders', **kwargs) +class _ResourceManagerLiensRepository( + repository_mixins.ListQueryMixin, + _base_repository.GCPRepository): + """Implementation of Cloud Resource Manager Liens repository.""" + + def __init__(self, **kwargs): + """Constructor. + + Args: + **kwargs (dict): The args to pass into GCPRepository.__init__() + """ + super(_ResourceManagerLiensRepository, self).__init__( + list_key_field='parent', max_results_field='pageSize', + component='liens', **kwargs) + + class CloudResourceManagerClient(object): """Resource Manager Client.""" @@ -517,6 +543,28 @@ def get_folders(self, parent=None, show_deleted=False): resource_name = 'All Folders' raise api_errors.ApiExecutionError(resource_name, e) + def get_project_liens(self, project_id): + """Get all liens for this project. + + Args: + project_id (str): the id of the project. + + Returns: + list: A list of Lien dicts as returned by the API. + + Raises: + ApiExecutionError: An error has occurred when executing the API. + """ + project_id = self.repository.projects.get_name(project_id) + try: + paged_results = self.repository.liens.list( + project_id) + flattened_results = api_helpers.flatten_list_results( + paged_results, 'liens') + return flattened_results + except (errors.HttpError, HttpLib2Error) as e: + raise api_errors.ApiExecutionError(project_id, e) + def get_folder_iam_policies(self, folder_id): """Get all the iam policies of a folder. diff --git a/google/cloud/forseti/common/gcp_type/resource.py b/google/cloud/forseti/common/gcp_type/resource.py index b8c3bb19a5..29e87e20e9 100755 --- a/google/cloud/forseti/common/gcp_type/resource.py +++ b/google/cloud/forseti/common/gcp_type/resource.py @@ -32,6 +32,7 @@ class ResourceType(object): BILLING_ACCOUNT = resources.BillingAccount.type() FOLDER = resources.Folder.type() PROJECT = resources.Project.type() + LIEN = resources.Lien.type() # Groups GROUP = resources.GsuiteGroup.type() @@ -73,6 +74,7 @@ class ResourceType(object): BUCKET, GROUP, FORWARDING_RULE, + LIEN, LOG_SINK, ]) diff --git a/google/cloud/forseti/services/inventory/base/gcp.py b/google/cloud/forseti/services/inventory/base/gcp.py index da499c9598..475879eebd 100644 --- a/google/cloud/forseti/services/inventory/base/gcp.py +++ b/google/cloud/forseti/services/inventory/base/gcp.py @@ -375,6 +375,19 @@ def iter_folders(self, parent_id): for folder in self.crm.get_folders(parent_id): yield folder + @create_lazy('crm', _create_crm) + def iter_project_liens(self, project_id): + """Iterate Liens from GCP API. + + Args: + project_id (str): id of the parent project of the lien. + + Yields: + dict: Generator of liens + """ + for lien in self.crm.get_project_liens(project_id): + yield lien + @create_lazy('appengine', _create_appengine) def fetch_gae_app(self, projectid): """Fetch the AppEngine App. diff --git a/google/cloud/forseti/services/inventory/base/resources.py b/google/cloud/forseti/services/inventory/base/resources.py index 8be6b2e151..2ab994155e 100644 --- a/google/cloud/forseti/services/inventory/base/resources.py +++ b/google/cloud/forseti/services/inventory/base/resources.py @@ -1210,6 +1210,27 @@ def type(): return 'instancetemplate' +class Lien(Resource): + """The Resource implementation for Lien""" + + def key(self): + """Get key of this resource + + Returns: + str: key of this resource + """ + return self['name'].split('/')[-1] + + @staticmethod + def type(): + """Get type of this resource + + Returns: + str: 'lien' + """ + return 'lien' + + class Network(Resource): """The Resource implementation for Network""" @@ -2045,6 +2066,19 @@ def iter(self): yield FACTORIES['gsuite_group_member'].create_new(data) +class ProjectLienIterator(ResourceIterator): + """The Resource iterator implementation for Project Liens.""" + + def iter(self): + """Yields: + Resource: Lien created + """ + if self.resource.enumerable(): + for data in self.client.iter_project_liens( + project_id=self.resource['projectId']): + yield FACTORIES['lien'].create_new(data) + + class ProjectSinkIterator(ResourceIterator): """The Resource iterator implementation for Project Sink""" @@ -2143,6 +2177,7 @@ def iter(self): NetworkIterator, SnapshotIterator, SubnetworkIterator, + ProjectLienIterator, ProjectRoleIterator, ProjectSinkIterator ]}), @@ -2340,6 +2375,12 @@ def iter(self): 'contains': [ ]}), + 'lien': ResourceFactory({ + 'dependsOn': ['project'], + 'cls': Lien, + 'contains': [ + ]}), + 'sink': ResourceFactory({ 'dependsOn': ['organization', 'folder', 'project'], 'cls': Sink, diff --git a/google/cloud/forseti/services/model/importer/importer.py b/google/cloud/forseti/services/model/importer/importer.py index 7a33fc44dc..539865cce8 100644 --- a/google/cloud/forseti/services/model/importer/importer.py +++ b/google/cloud/forseti/services/model/importer/importer.py @@ -158,6 +158,7 @@ def run(self): 'instancegroupmanager', 'instancetemplate', 'instance', + 'lien', 'firewall', 'backendservice', 'forwardingrule', @@ -601,6 +602,9 @@ def _store_resource(self, resource, last_res_type=None): 'instance': (None, self._convert_instance, None), + 'lien': (None, + self._convert_lien, + None), 'firewall': (None, self._convert_firewall, None), @@ -1004,6 +1008,25 @@ def _convert_instance(self, instance): self.session.add(resource) self._add_to_cache(resource, instance.id) + def _convert_lien(self, lien): + """Convert a lien to a database object. + + Args: + lien (object): Lien to store. + """ + data = lien.get_resource_data() + parent, full_res_name, type_name = self._full_resource_name(lien) + resource = self.dao.TBL_RESOURCE( + full_name=full_res_name, + type_name=type_name, + name=lien.get_resource_id(), + type=lien.get_resource_type(), + display_name=data.get('name', ''), + data=lien.get_resource_data_raw(), + parent=parent) + + self.session.add(resource) + def _convert_firewall(self, firewall): """Convert a firewall to a database object. diff --git a/tests/common/gcp_api/cloud_resource_manager_test.py b/tests/common/gcp_api/cloud_resource_manager_test.py index bea6f38939..308e44fc91 100644 --- a/tests/common/gcp_api/cloud_resource_manager_test.py +++ b/tests/common/gcp_api/cloud_resource_manager_test.py @@ -402,6 +402,13 @@ def test_get_org_policy_errors(self): resource_id, fake_crm_responses.TEST_ORG_POLICY_CONSTRAINT) + def test_get_liens(self): + """Test get liens.""" + http_mocks.mock_http_response(fake_crm_responses.GET_LIENS) + response = self.crm_api_client.get_project_liens( + fake_crm_responses.FAKE_PROJECT_ID) + self.assertEqual(response, fake_crm_responses.EXPECTED_LIENS) + if __name__ == '__main__': unittest.main() diff --git a/tests/common/gcp_api/test_data/fake_crm_responses.py b/tests/common/gcp_api/test_data/fake_crm_responses.py index 64dd133559..d7b8af484a 100644 --- a/tests/common/gcp_api/test_data/fake_crm_responses.py +++ b/tests/common/gcp_api/test_data/fake_crm_responses.py @@ -160,6 +160,32 @@ } """ +GET_LIENS = """ +{ + "liens": [ + { + "name": "liens/test-lien1", + "parent": "projects/forseti-system-test", + "restrictions": [ + "resourcemanager.projects.delete" + ], + "origin": "testing", + "createTime": "2018-09-05T14:45:46.534Z" + } + ] +} +""" + +EXPECTED_LIENS = [{ + "name": "liens/test-lien1", + "parent": "projects/forseti-system-test", + "restrictions": [ + "resourcemanager.projects.delete" + ], + "origin": "testing", + "createTime": "2018-09-05T14:45:46.534Z" +}] + LIST_ORG_POLICIES = """ { "policies": [ diff --git a/tests/services/inventory/crawling_test.py b/tests/services/inventory/crawling_test.py index aaa4a344e5..e0bc7ce4be 100644 --- a/tests/services/inventory/crawling_test.py +++ b/tests/services/inventory/crawling_test.py @@ -138,6 +138,7 @@ def test_crawling_to_memory_storage(self): 'instancegroupmanager': {'resource': 2}, 'instancetemplate': {'resource': 2}, 'kubernetes_cluster': {'resource': 1, 'service_config': 1}, + 'lien': {'resource': 1}, 'network': {'resource': 2}, 'organization': {'iam_policy': 1, 'resource': 1}, 'project': {'billing_info': 4, 'enabled_apis': 4, 'iam_policy': 4, @@ -225,6 +226,7 @@ def test_crawling_from_project(self): 'instancegroupmanager': {'resource': 2}, 'instancetemplate': {'resource': 2}, 'kubernetes_cluster': {'resource': 1, 'service_config': 1}, + 'lien': {'resource': 1}, 'network': {'resource': 1}, 'project': {'billing_info': 1, 'enabled_apis': 1, 'iam_policy': 1, 'resource': 1}, @@ -284,6 +286,7 @@ def test_crawling_no_org_access(self): 'instancegroupmanager': {'resource': 2}, 'instancetemplate': {'resource': 2}, 'kubernetes_cluster': {'resource': 1, 'service_config': 1}, + 'lien': {'resource': 1}, 'network': {'resource': 2}, 'organization': {'resource': 1}, 'project': {'billing_info': 4, 'enabled_apis': 4, 'iam_policy': 4, diff --git a/tests/services/inventory/gcp_api_mocks.py b/tests/services/inventory/gcp_api_mocks.py index 547116385c..f546ffe9b2 100644 --- a/tests/services/inventory/gcp_api_mocks.py +++ b/tests/services/inventory/gcp_api_mocks.py @@ -236,6 +236,9 @@ def _mock_crm_get_projects(parent_type, parent_id): def _mock_crm_get_iam_policies(folderid): return results.CRM_GET_IAM_POLICIES[folderid] + def _mock_crm_get_project_liens(projectid): + return results.CRM_GET_PROJECT_LIENS[projectid] + def _mock_permission_denied(parentid): response = httplib2.Response( {'status': '403', 'content-type': 'application/json'}) @@ -259,6 +262,7 @@ def _mock_permission_denied(parentid): mock_crm.get_projects.side_effect = _mock_crm_get_projects mock_crm.get_folder_iam_policies.side_effect = _mock_crm_get_iam_policies mock_crm.get_project_iam_policies.side_effect = _mock_crm_get_iam_policies + mock_crm.get_project_liens.side_effect = _mock_crm_get_project_liens return crm_patcher diff --git a/tests/services/inventory/test_data/mock_gcp_results.py b/tests/services/inventory/test_data/mock_gcp_results.py index 9fea6f3071..7d6a58a61b 100644 --- a/tests/services/inventory/test_data/mock_gcp_results.py +++ b/tests/services/inventory/test_data/mock_gcp_results.py @@ -46,6 +46,7 @@ GCE_IMAGE_ID_PREFIX = "117" GCE_DISK_ID_PREFIX = "118" SNAPSHOT_ID_PREFIX = "119" +LIEN_ID_PREFIX = "120" # Fields: id, email, name AD_USER_TEMPLATE = """ @@ -564,6 +565,18 @@ "project4": json.loads(CRM_PROJECT_IAM_POLICY_DUP_MEMBER.format(id=4)), } +CRM_GET_PROJECT_LIENS = { + "project1": [{ + "name": "liens/" + LIEN_ID_PREFIX, + "parent": "projects/project1", + "restrictions": [ + "resourcemanager.projects.delete" + ], + "origin": "testing", + "createTime": "2018-09-05T14:45:46.534Z", + }], +} + GCP_PERMISSION_DENIED_TEMPLATE = """ {{ "error": {{ diff --git a/tests/services/model/importer/test_data/forseti-test.db b/tests/services/model/importer/test_data/forseti-test.db index b176e45fbe..6364383d26 100644 Binary files a/tests/services/model/importer/test_data/forseti-test.db and b/tests/services/model/importer/test_data/forseti-test.db differ