Add project Liens to inventory

google/cloud/forseti/common/gcp_api/cloud_resource_manager.py

@@ -50,6 +50,7 @@ def __init__(self,
         self._organizations = None
         self._folders = None
         self._folders_v1 = None
+        self._liens = None
 
         super(CloudResourceManagerRepositoryClient, self).__init__(
             'cloudresourcemanager', versions=['v1', 'v2'],
@@ -91,6 +92,15 @@ def folders_v1(self):
             self._folders_v1 = self._init_repository(
                 _ResourceManagerFolderV1Repository, version='v1')
         return self._folders_v1
+
+    @property
+    def liens(self):
+        """Returns a _ResourceManagerLiensRepository instance."""
+        if not self._liens:
+            self._liens = self._init_repository(
+                _ResourceManagerLiensRepository)
+        return self._liens
+
     # pylint: enable=missing-return-doc, missing-return-type-doc
 
 
@@ -226,6 +236,22 @@ def __init__(self, **kwargs):
             max_results_field='pageSize', component='folders', **kwargs)
 
 
+class _ResourceManagerLiensRepository(
+        repository_mixins.ListQueryMixin,
+        _base_repository.GCPRepository):
+    """Implementation of Cloud Resource Manager Liens repository."""
+
+    def __init__(self, **kwargs):
+        """Constructor.
+
+        Args:
+            **kwargs (dict): The args to pass into GCPRepository.__init__()
+        """
+        super(_ResourceManagerLiensRepository, self).__init__(
+            list_key_field='parent', max_results_field='pageSize',
+            component='liens', **kwargs)
+
+
 class CloudResourceManagerClient(object):
     """Resource Manager Client."""
 
@@ -517,6 +543,28 @@ def get_folders(self, parent=None, show_deleted=False):
                 resource_name = 'All Folders'
             raise api_errors.ApiExecutionError(resource_name, e)
 
+    def get_project_liens(self, project_id):
+        """Get all liens for this project.
+
+        Args:
+            project_id (str): the id of the project.
+
+        Returns:
+            list: A list of Lien dicts as returned by the API.
+
+        Raises:
+            ApiExecutionError: An error has occurred when executing the API.
+        """
+        project_id = self.repository.projects.get_name(project_id)
+        try:
+            paged_results = self.repository.liens.list(
+                project_id)
+            flattened_results = api_helpers.flatten_list_results(
+                paged_results, 'liens')
+            return flattened_results
+        except (errors.HttpError, HttpLib2Error) as e:
+            raise api_errors.ApiExecutionError(project_id, e)
+
     def get_folder_iam_policies(self, folder_id):
         """Get all the iam policies of a folder.
 

google/cloud/forseti/common/gcp_type/resource.py

@@ -32,6 +32,7 @@ class ResourceType(object):
     BILLING_ACCOUNT = resources.BillingAccount.type()
     FOLDER = resources.Folder.type()
     PROJECT = resources.Project.type()
+    LIEN = resources.Lien.type()
 
     # Groups
     GROUP = resources.GsuiteGroup.type()
@@ -73,6 +74,7 @@ class ResourceType(object):
         BUCKET,
         GROUP,
         FORWARDING_RULE,
+        LIEN,
         LOG_SINK,
     ])
 

google/cloud/forseti/services/inventory/base/gcp.py

@@ -375,6 +375,19 @@ def iter_folders(self, parent_id):
         for folder in self.crm.get_folders(parent_id):
             yield folder
 
+    @create_lazy('crm', _create_crm)
+    def iter_project_liens(self, project_id):
+        """Iterate Liens from GCP API.
+
+        Args:
+            project_id (str): id of the parent project of the lien.
+
+        Yields:
+            dict: Generator of liens
+        """
+        for lien in self.crm.get_project_liens(project_id):
+            yield lien
+
     @create_lazy('appengine', _create_appengine)
     def fetch_gae_app(self, projectid):
         """Fetch the AppEngine App.

google/cloud/forseti/services/inventory/base/resources.py

@@ -1210,6 +1210,27 @@ def type():
         return 'instancetemplate'
 
 
+class Lien(Resource):
+    """The Resource implementation for Lien"""
+
+    def key(self):
+        """Get key of this resource
+
+        Returns:
+            str: key of this resource
+        """
+        return self['name'].split('/')[-1]
+
+    @staticmethod
+    def type():
+        """Get type of this resource
+
+        Returns:
+            str: 'lien'
+        """
+        return 'lien'
+
+
 class Network(Resource):
     """The Resource implementation for Network"""
 
@@ -2045,6 +2066,19 @@ def iter(self):
                 yield FACTORIES['gsuite_group_member'].create_new(data)
 
 
+class ProjectLienIterator(ResourceIterator):
+    """The Resource iterator implementation for Project Liens."""
+
+    def iter(self):
+        """Yields:
+            Resource: Lien created
+        """
+        if self.resource.enumerable():
+            for data in self.client.iter_project_liens(
+                    project_id=self.resource['projectId']):
+                yield FACTORIES['lien'].create_new(data)
+
+
 class ProjectSinkIterator(ResourceIterator):
     """The Resource iterator implementation for Project Sink"""
 
@@ -2143,6 +2177,7 @@ def iter(self):
             NetworkIterator,
             SnapshotIterator,
             SubnetworkIterator,
+            ProjectLienIterator,
             ProjectRoleIterator,
             ProjectSinkIterator
         ]}),
@@ -2340,6 +2375,12 @@ def iter(self):
         'contains': [
         ]}),
 
+    'lien': ResourceFactory({
+        'dependsOn': ['project'],
+        'cls': Lien,
+        'contains': [
+        ]}),
+
     'sink': ResourceFactory({
         'dependsOn': ['organization', 'folder', 'project'],
         'cls': Sink,

google/cloud/forseti/services/model/importer/importer.py

@@ -158,6 +158,7 @@ def run(self):
             'instancegroupmanager',
             'instancetemplate',
             'instance',
+            'lien',
             'firewall',
             'backendservice',
             'forwardingrule',
@@ -603,6 +604,9 @@ def _store_resource(self, resource, last_res_type=None):
             'instance': (None,
                          self._convert_instance,
                          None),
+            'lien': (None,
+                     self._convert_lien,
+                     None),
             'firewall': (None,
                          self._convert_firewall,
                          None),
@@ -1006,6 +1010,25 @@ def _convert_instance(self, instance):
         self.session.add(resource)
         self._add_to_cache(resource, instance.id)
 
+    def _convert_lien(self, lien):
+        """Convert a lien to a database object.
+
+        Args:
+            lien (object): Lien to store.
+        """
+        data = lien.get_resource_data()
+        parent, full_res_name, type_name = self._full_resource_name(lien)
+        resource = self.dao.TBL_RESOURCE(
+            full_name=full_res_name,
+            type_name=type_name,
+            name=lien.get_resource_id(),
+            type=lien.get_resource_type(),
+            display_name=data.get('name', ''),
+            data=lien.get_resource_data_raw(),
+            parent=parent)
+
+        self.session.add(resource)
+
     def _convert_firewall(self, firewall):
         """Convert a firewall to a database object.
 

tests/common/gcp_api/cloud_resource_manager_test.py

@@ -402,6 +402,13 @@ def test_get_org_policy_errors(self):
                     resource_id,
                     fake_crm_responses.TEST_ORG_POLICY_CONSTRAINT)
 
+    def test_get_liens(self):
+        """Test get liens."""
+        http_mocks.mock_http_response(fake_crm_responses.GET_LIENS)
+        response = self.crm_api_client.get_project_liens(
+            fake_crm_responses.FAKE_PROJECT_ID)
+        self.assertEqual(response, fake_crm_responses.EXPECTED_LIENS)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/common/gcp_api/test_data/fake_crm_responses.py

@@ -160,6 +160,32 @@
 }
 """
 
+GET_LIENS = """
+{
+  "liens": [
+    {
+      "name": "liens/test-lien1",
+      "parent": "projects/forseti-system-test",
+      "restrictions": [
+        "resourcemanager.projects.delete"
+      ],
+      "origin": "testing",
+      "createTime": "2018-09-05T14:45:46.534Z"
+    }
+  ]
+}
+"""
+
+EXPECTED_LIENS = [{
+    "name": "liens/test-lien1",
+    "parent": "projects/forseti-system-test",
+    "restrictions": [
+        "resourcemanager.projects.delete"
+    ],
+    "origin": "testing",
+    "createTime": "2018-09-05T14:45:46.534Z"
+}]
+
 LIST_ORG_POLICIES = """
 {
  "policies": [

tests/services/inventory/crawling_test.py

@@ -138,6 +138,7 @@ def test_crawling_to_memory_storage(self):
             'instancegroupmanager': {'resource': 2},
             'instancetemplate': {'resource': 2},
             'kubernetes_cluster': {'resource': 1, 'service_config': 1},
+            'lien': {'resource': 1},
             'network': {'resource': 2},
             'organization': {'iam_policy': 1, 'resource': 1},
             'project': {'billing_info': 4, 'enabled_apis': 4, 'iam_policy': 4,
@@ -225,6 +226,7 @@ def test_crawling_from_project(self):
             'instancegroupmanager': {'resource': 2},
             'instancetemplate': {'resource': 2},
             'kubernetes_cluster': {'resource': 1, 'service_config': 1},
+            'lien': {'resource': 1},
             'network': {'resource': 1},
             'project': {'billing_info': 1, 'enabled_apis': 1, 'iam_policy': 1,
                         'resource': 1},
@@ -284,6 +286,7 @@ def test_crawling_no_org_access(self):
             'instancegroupmanager': {'resource': 2},
             'instancetemplate': {'resource': 2},
             'kubernetes_cluster': {'resource': 1, 'service_config': 1},
+            'lien': {'resource': 1},
             'network': {'resource': 2},
             'organization': {'resource': 1},
             'project': {'billing_info': 4, 'enabled_apis': 4, 'iam_policy': 4,

tests/services/inventory/gcp_api_mocks.py

@@ -235,6 +235,9 @@ def _mock_crm_get_projects(parent_type, parent_id):
     def _mock_crm_get_iam_policies(folderid):
         return results.CRM_GET_IAM_POLICIES[folderid]
 
+    def _mock_crm_get_project_liens(projectid):
+        return results.CRM_GET_PROJECT_LIENS[projectid]
+
     def _mock_permission_denied(parentid):
         response = httplib2.Response(
             {'status': '403', 'content-type': 'application/json'})
@@ -258,6 +261,7 @@ def _mock_permission_denied(parentid):
     mock_crm.get_projects.side_effect = _mock_crm_get_projects
     mock_crm.get_folder_iam_policies.side_effect = _mock_crm_get_iam_policies
     mock_crm.get_project_iam_policies.side_effect = _mock_crm_get_iam_policies
+    mock_crm.get_project_liens.side_effect = _mock_crm_get_project_liens
 
     return crm_patcher
 

tests/services/inventory/test_data/mock_gcp_results.py

@@ -45,6 +45,7 @@
 GCE_IMAGE_ID_PREFIX = "117"
 GCE_DISK_ID_PREFIX = "118"
 SNAPSHOT_ID_PREFIX = "119"
+LIEN_ID_PREFIX = "120"
 
 # Fields: id, email, name
 AD_USER_TEMPLATE = """
@@ -563,6 +564,18 @@
     "project4": json.loads(CRM_PROJECT_IAM_POLICY_DUP_MEMBER.format(id=4)),
 }
 
+CRM_GET_PROJECT_LIENS = {
+    "project1": [{
+        "name": "liens/" + LIEN_ID_PREFIX,
+        "parent": "projects/project1",
+        "restrictions": [
+            "resourcemanager.projects.delete"
+        ],
+        "origin": "testing",
+        "createTime": "2018-09-05T14:45:46.534Z",
+    }],
+}
+
 GCP_PERMISSION_DENIED_TEMPLATE = """
 {{
  "error": {{