Add organization policies to inventory crawler and model. (#2105)

* Add organization policies to inventory crawler and model.

google/cloud/forseti/services/inventory/base/gcp.py

@@ -376,6 +376,14 @@ def fetch_crm_project_iam_policy(self, project_number):
             project_number (str): id of the project to query.
         """
 
+    @abc.abstractmethod
+    def iter_crm_folder_org_policies(self, folder_id):
+        """Folder organization policies from gcp API call.
+
+        Args:
+            folder_id (str): id of the folder to get policy.
+        """
+
     @abc.abstractmethod
     def iter_crm_folders(self, parent_id):
         """Iterate Folders from GCP API.
@@ -384,6 +392,14 @@ def iter_crm_folders(self, parent_id):
             parent_id (str): id of the parent of the folder.
         """
 
+    @abc.abstractmethod
+    def iter_crm_organization_org_policies(self, org_id):
+        """Organization organization policies from gcp API call.
+
+        Args:
+            org_id (str): id of the organization to get policy.
+        """
+
     @abc.abstractmethod
     def iter_crm_project_liens(self, project_number):
         """Iterate Liens from GCP API.
@@ -392,6 +408,14 @@ def iter_crm_project_liens(self, project_number):
             project_number (str): id of the parent project of the lien.
         """
 
+    @abc.abstractmethod
+    def iter_crm_project_org_policies(self, project_number):
+        """Project organization policies from gcp API call.
+
+        Args:
+            project_number (str): id of the parent project of the policy.
+        """
+
     @abc.abstractmethod
     def iter_crm_projects(self, parent_type, parent_id):
         """Iterate Projects from GCP API.
@@ -1301,6 +1325,19 @@ def fetch_crm_project_iam_policy(self, project_number):
         """
         return self.crm.get_project_iam_policies(project_number)
 
+    @create_lazy('crm', _create_crm)
+    def iter_crm_folder_org_policies(self, folder_id):
+        """Folder organization policies from gcp API call.
+
+        Args:
+            folder_id (str): id of the folder to get policy.
+
+        Yields:
+            dict: Generator of org policies.
+        """
+        for org_policy in self.crm.get_folder_org_policies(folder_id):
+            yield org_policy
+
     @create_lazy('crm', _create_crm)
     def iter_crm_folders(self, parent_id):
         """Iterate Folders from GCP API.
@@ -1314,6 +1351,19 @@ def iter_crm_folders(self, parent_id):
         for folder in self.crm.get_folders(parent_id):
             yield folder
 
+    @create_lazy('crm', _create_crm)
+    def iter_crm_organization_org_policies(self, org_id):
+        """Organization organization policies from gcp API call.
+
+        Args:
+            org_id (str): id of the organization to get policy.
+
+        Yields:
+            dict: Generator of org policies.
+        """
+        for org_policy in self.crm.get_org_org_policies(org_id):
+            yield org_policy
+
     @create_lazy('crm', _create_crm)
     def iter_crm_project_liens(self, project_number):
         """Iterate Liens from GCP API.
@@ -1327,6 +1377,19 @@ def iter_crm_project_liens(self, project_number):
         for lien in self.crm.get_project_liens(project_number):
             yield lien
 
+    @create_lazy('crm', _create_crm)
+    def iter_crm_project_org_policies(self, project_number):
+        """Project organization policies from gcp API call.
+
+        Args:
+            project_number (str): id of the parent project of the policy.
+
+        Yields:
+            dict: Generator of org policies.
+        """
+        for org_policy in self.crm.get_project_org_policies(project_number):
+            yield org_policy
+
     @create_lazy('crm', _create_crm)
     def iter_crm_projects(self, parent_type, parent_id):
         """Iterate Projects from GCP API.

google/cloud/forseti/services/inventory/base/resources.py

@@ -543,6 +543,21 @@ def key(self):
         return self['name'].split('/', 1)[-1]
 
 
+class ResourceManagerOrgPolicy(resource_class_factory('crm_org_policy', None)):
+    """The Resource implementation for Resource Manager Organization Policy."""
+
+    def key(self):
+        """Get key of this resource.
+
+        Returns:
+            str: key of this resource
+        """
+        unique_key = '/'.join([self.parent().type(),
+                               self.parent().key(),
+                               self['constraint']])
+        return '%u' % ctypes.c_size_t(hash(unique_key)).value
+
+
 class ResourceManagerFolder(resource_class_factory('folder', None)):
     """The Resource implementation for Folder."""
 
@@ -1265,6 +1280,20 @@ class ResourceManagerFolderIterator(resource_iter_class_factory(
     """The Resource iterator implementation for Resource Manager Folder."""
 
 
+class ResourceManagerFolderOrgPolicyIterator(resource_iter_class_factory(
+        api_method_name='iter_crm_folder_org_policies',
+        resource_name='crm_org_policy',
+        api_method_arg_key='name')):
+    """The Resource iterator implementation for CRM Folder Org Policies."""
+
+
+class ResourceManagerOrganizationOrgPolicyIterator(resource_iter_class_factory(
+        api_method_name='iter_crm_organization_org_policies',
+        resource_name='crm_org_policy',
+        api_method_arg_key='name')):
+    """The Resource iterator for CRM Organization Org Policies."""
+
+
 # Project iterator requires looking up parent type, so cannot use class factory.
 class ResourceManagerProjectIterator(ResourceIterator):
     """The Resource iterator implementation for Resource Manager Project."""
@@ -1283,6 +1312,13 @@ def iter(self):
             yield FACTORIES['project'].create_new(data)
 
 
+class ResourceManagerProjectOrgPolicyIterator(resource_iter_class_factory(
+        api_method_name='iter_crm_project_org_policies',
+        resource_name='crm_org_policy',
+        api_method_arg_key='projectNumber')):
+    """The Resource iterator implementation for CRM Project Org Policies."""
+
+
 # AppEngine iterators do not support using the class factory.
 class AppEngineAppIterator(ResourceIterator):
     """The Resource iterator implementation for AppEngineApp"""
@@ -1747,6 +1783,7 @@ class StorageObjectIterator(resource_iter_class_factory(
             IamOrganizationCuratedRoleIterator,
             IamOrganizationRoleIterator,
             LoggingOrganizationSinkIterator,
+            ResourceManagerOrganizationOrgPolicyIterator,
             ResourceManagerFolderIterator,
             ResourceManagerProjectIterator,
         ]}),
@@ -1756,6 +1793,7 @@ class StorageObjectIterator(resource_iter_class_factory(
         'cls': ResourceManagerFolder,
         'contains': [
             LoggingFolderSinkIterator,
+            ResourceManagerFolderOrgPolicyIterator,
             ResourceManagerFolderIterator,
             ResourceManagerProjectIterator,
         ]}),
@@ -1801,6 +1839,7 @@ class StorageObjectIterator(resource_iter_class_factory(
             KubernetesClusterIterator,
             LoggingProjectSinkIterator,
             ResourceManagerProjectLienIterator,
+            ResourceManagerProjectOrgPolicyIterator,
             SpannerInstanceIterator,
             StorageBucketIterator,
         ]}),
@@ -1988,6 +2027,11 @@ class StorageObjectIterator(resource_iter_class_factory(
         'cls': ResourceManagerLien,
         'contains': []}),
 
+    'crm_org_policy': ResourceFactory({
+        'dependsOn': ['folder', 'organization', 'project'],
+        'cls': ResourceManagerOrgPolicy,
+        'contains': []}),
+
     'dns_managedzone': ResourceFactory({
         'dependsOn': ['project'],
         'cls': DnsManagedZone,

google/cloud/forseti/services/model/importer/importer.py

@@ -152,6 +152,7 @@ def run(self):
             'bucket',
             'dataset',
             'compute_project',
+            'crm_org_policy',
             'disk',
             'image',
             'instancegroup',
@@ -584,6 +585,9 @@ def _store_resource(self, resource, last_res_type=None):
             'compute_project': (None,
                                 self._convert_computeproject,
                                 None),
+            'crm_org_policy': (None,
+                               self._convert_crm_org_policy,
+                               None),
             'disk': (None,
                      self._convert_disk,
                      None),
@@ -852,6 +856,27 @@ def _convert_computeproject(self, computeproject):
         self.session.add(resource)
         self._add_to_cache(resource, computeproject.id)
 
+    def _convert_crm_org_policy(self, org_policy):
+        """Convert an org policy to a database object.
+
+        Args:
+            org_policy (object): org policy to store.
+        """
+        data = org_policy.get_resource_data()
+        parent, full_res_name, type_name = self._full_resource_name(
+            org_policy)
+        resource = self.dao.TBL_RESOURCE(
+            full_name=full_res_name,
+            type_name=type_name,
+            name=org_policy.get_resource_id(),
+            type=org_policy.get_resource_type(),
+            display_name=data.get('constraint', ''),
+            email='',
+            data=org_policy.get_resource_data_raw(),
+            parent=parent)
+
+        self.session.add(resource)
+
     def _convert_iam_policy(self, iam_policy):
         """Convert an IAM policy to a database object.
 

tests/services/inventory/crawling_test.py

@@ -133,6 +133,7 @@ def test_crawling_to_memory_storage(self):
             'bucket': {'gcs_policy': 2, 'iam_policy': 2, 'resource': 2},
             'cloudsqlinstance': {'resource': 1},
             'compute_project': {'resource': 2},
+            'crm_org_policy': {'resource': 5},
             'dataset': {'dataset_policy': 1, 'resource': 1},
             'disk': {'resource': 4},
             'firewall': {'resource': 7},
@@ -230,6 +231,7 @@ def test_crawling_from_project(self):
         expected_counts = {
             'backendservice': {'resource': 1},
             'compute_project': {'resource': 1},
+            'crm_org_policy': {'resource': 1},
             'disk': {'resource': 3},
             'firewall': {'resource': 3},
             'forwardingrule': {'resource': 1},
@@ -276,8 +278,8 @@ def test_crawling_no_org_access(self):
             result_counts = self._get_resource_counts_from_storage(storage)
 
         # The crawl should be the same as test_crawling_to_memory_storage, but
-        # without organization iam_policy (needs Org access) or gsuite_*
-        # resources (needs directoryCustomerId from Organization).
+        # without organization iam_policy, org_policy (needs Org access) or
+        # gsuite_* resources (needs directoryCustomerId from Organization).
         expected_counts = {
             'appengine_app': {'resource': 2},
             'appengine_instance': {'resource': 3},
@@ -288,6 +290,7 @@ def test_crawling_no_org_access(self):
             'bucket': {'gcs_policy': 2, 'iam_policy': 2, 'resource': 2},
             'cloudsqlinstance': {'resource': 1},
             'compute_project': {'resource': 2},
+            'crm_org_policy': {'resource': 3},
             'dataset': {'dataset_policy': 1, 'resource': 1},
             'disk': {'resource': 4},
             'firewall': {'resource': 7},
@@ -403,6 +406,7 @@ def test_cai_crawl_to_memory(self):
             'compute_targetsslproxy': {'resource': 1},
             'compute_targettcpproxy': {'resource': 1},
             'compute_urlmap': {'resource': 1},
+            'crm_org_policy': {'resource': 5},
             'dataset': {'dataset_policy': 1, 'resource': 1},
             'disk': {'resource': 4},
             'dns_managedzone': {'resource': 1},

tests/services/inventory/gcp_api_mocks.py

@@ -258,6 +258,9 @@ def _mock_crm_get_iam_policies(folderid):
     def _mock_crm_get_project_liens(projectid):
         return results.CRM_GET_PROJECT_LIENS.get(projectid, [])
 
+    def _mock_crm_get_org_policies(resourceid):
+        return results.CRM_GET_ORG_POLICIES.get(resourceid, [])
+
     def _mock_permission_denied(parentid):
         response = httplib2.Response(
             {'status': '403', 'content-type': 'application/json'})
@@ -272,16 +275,20 @@ def _mock_permission_denied(parentid):
     if has_org_access:
         mock_crm.get_organization.side_effect = _mock_crm_get_organization
         mock_crm.get_org_iam_policies.side_effect = _mock_crm_get_iam_policies
+        mock_crm.get_org_org_policies.side_effect = _mock_crm_get_org_policies
     else:
         mock_crm.get_organization.side_effect = _mock_permission_denied
         mock_crm.get_org_iam_policies.side_effect = _mock_permission_denied
+        mock_crm.get_org_org_policies.side_effect = _mock_permission_denied
     mock_crm.get_folder.side_effect = _mock_crm_get_folder
     mock_crm.get_folders.side_effect = _mock_crm_get_folders
     mock_crm.get_project.side_effect = _mock_crm_get_project
     mock_crm.get_projects.side_effect = _mock_crm_get_projects
     mock_crm.get_folder_iam_policies.side_effect = _mock_crm_get_iam_policies
+    mock_crm.get_folder_org_policies.side_effect = _mock_crm_get_org_policies
     mock_crm.get_project_iam_policies.side_effect = _mock_crm_get_iam_policies
     mock_crm.get_project_liens.side_effect = _mock_crm_get_project_liens
+    mock_crm.get_project_org_policies.side_effect = _mock_crm_get_org_policies
 
     return crm_patcher
 

tests/services/inventory/test_data/mock_gcp_results.py

@@ -574,6 +574,61 @@
     PROJECT_ID_PREFIX + "4": json.loads(CRM_PROJECT_IAM_POLICY_DUP_MEMBER.format(id=4)),
 }
 
+CRM_ORG_ORG_POLICIES = """
+[
+  {
+    "constraint": "constraints/compute.disableSerialPortAccess",
+    "booleanPolicy": {
+     "enforced": true
+    }
+  },
+  {
+    "constraint": "constraints/compute.storageResourceUseRestrictions",
+    "list_policy": {
+      "all_values": "ALLOW"
+    }
+  }
+]
+"""
+
+CRM_FOLDER_ORG_POLICIES = """
+[
+  {
+    "constraint": "constraints/storage.requireIamAclsOnly",
+    "boolean_policy": {
+      "enforced": true
+    }
+  }
+]
+"""
+
+
+CRM_PROJECT_ORG_POLICIES = """
+[
+  {
+    "constraint": "constraints/compute.trustedImageProjects",
+    "list_policy": {
+      "allowed_values": [
+        "is:projects/my-good-images",
+        "is:projects/my-other-images",
+        "is:projects/trusted-cloud-images"
+      ]
+    }
+  }
+]
+"""
+
+CRM_GET_ORG_POLICIES = {
+    ORGANIZATION_ID: json.loads(CRM_ORG_ORG_POLICIES),
+    "folders/" + FOLDER_ID_PREFIX + "1": json.loads(CRM_FOLDER_ORG_POLICIES),
+    "folders/" + FOLDER_ID_PREFIX + "2": {},
+    "folders/" + FOLDER_ID_PREFIX + "3": {},
+    PROJECT_ID_PREFIX + "1": json.loads(CRM_PROJECT_ORG_POLICIES),
+    PROJECT_ID_PREFIX + "2": json.loads(CRM_PROJECT_ORG_POLICIES),
+    PROJECT_ID_PREFIX + "3": {},
+    PROJECT_ID_PREFIX + "4": {},
+}
+
 CRM_GET_PROJECT_LIENS = {
     PROJECT_ID_PREFIX + "1": [{
         "name": "liens/" + LIEN_ID_PREFIX,