Add project_id to Big Query resources (#2393)

* + Add project_id
+ Add datasets with project id

* Updates

* del unused variable.

* re-enable crawler to run in parallel in unit test.

* pylint updates

* updates

* pylint updates

google/cloud/forseti/services/inventory/base/cai_gcp_client.py

@@ -109,47 +109,61 @@ def session(self):
         self._local.cai_session = db.create_readonly_session(engine=self.engine)
         return self._local.cai_session
 
-    def fetch_bigquery_iam_policy(self, project_number, dataset_id):
+    def fetch_bigquery_iam_policy(self, project_id, project_number, dataset_id):
         """Gets IAM policy of a bigquery dataset from Cloud Asset data.
 
         Args:
+            project_id (str): id of the project to query.
             project_number (str): number of the project to query.
             dataset_id (str): id of the dataset to query.
 
         Returns:
             dict: Dataset IAM Policy.
         """
+        bigquery_name_fmt = '//bigquery.googleapis.com/projects/{}/datasets/{}'
+
+        # Try fetching with project id, if that returns nothing, fall back to
+        # project number.
         resource = self.dao.fetch_cai_asset(
             ContentTypes.iam_policy,
             'google.cloud.bigquery.Dataset',
-            '//bigquery.googleapis.com/projects/{}/datasets/{}'.format(
-                project_number, dataset_id),
+            bigquery_name_fmt.format(project_id, dataset_id),
             self.session)
+
+        if not resource:
+            resource = self.dao.fetch_cai_asset(
+                ContentTypes.iam_policy,
+                'google.cloud.bigquery.Dataset',
+                bigquery_name_fmt.format(project_number, dataset_id),
+                self.session)
+
         if resource:
             return resource
+
         return {}
 
-    def fetch_bigquery_dataset_policy(self, project_number, dataset_id):
+    def fetch_bigquery_dataset_policy(self, project_id, project_number,
+                                      dataset_id):
         """Dataset policy Iterator for a dataset from Cloud Asset data.
 
         Args:
+            project_id (str): id of the project to query.
             project_number (str): number of the project to query.
             dataset_id (str): id of the dataset to query.
 
         Returns:
             dict: Dataset Policy.
         """
-        resource = self.dao.fetch_cai_asset(
-            ContentTypes.iam_policy,
-            'google.cloud.bigquery.Dataset',
-            '//bigquery.googleapis.com/projects/{}/datasets/{}'.format(
-                project_number, dataset_id),
-            self.session)
+
+        resource = self.fetch_bigquery_iam_policy(
+            project_id, project_number, dataset_id)
+
         if resource:
             return iam_helpers.convert_iam_to_bigquery_policy(resource)
+
         # Fall back to live API if the data isn't in the CAI cache.
         return super(CaiApiClientImpl, self).fetch_bigquery_dataset_policy(
-            project_number, dataset_id)
+            project_id, project_number, dataset_id)
 
     def iter_bigquery_datasets(self, project_number):
         """Iterate Datasets from Cloud Asset data.
@@ -160,6 +174,7 @@ def iter_bigquery_datasets(self, project_number):
         Yields:
             dict: Generator of datasets.
         """
+
         resources = list(self.dao.iter_cai_assets(
             ContentTypes.resource,
             'google.cloud.bigquery.Dataset',

google/cloud/forseti/services/inventory/base/gcp.py

@@ -41,19 +41,22 @@ class ApiClient(object):
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod
-    def fetch_bigquery_dataset_policy(self, project_number, dataset_id):
+    def fetch_bigquery_dataset_policy(self, project_id,
+                                      project_number, dataset_id):
         """Dataset policy Iterator for a dataset from gcp API call.
 
         Args:
+            project_id (str): id of the project to query.
             project_number (str): number of the project to query.
             dataset_id (str): id of the dataset to query.
         """
 
     @abc.abstractmethod
-    def fetch_bigquery_iam_policy(self, project_number, dataset_id):
+    def fetch_bigquery_iam_policy(self, project_id, project_number, dataset_id):
         """Gets IAM policy of a bigquery dataset from gcp API call.
 
         Args:
+            project_id (str): id of the project to query.
             project_number (str): number of the project to query.
             dataset_id (str): id of the dataset to query.
         """
@@ -1012,22 +1015,27 @@ def _create_storage(self):
         return storage.StorageClient(self.config)
 
     @create_lazy('bigquery', _create_bq)
-    def fetch_bigquery_dataset_policy(self, project_number, dataset_id):
+    def fetch_bigquery_dataset_policy(self, project_id,
+                                      project_number, dataset_id):
         """Dataset policy Iterator for a dataset from gcp API call.
 
         Args:
+            project_id (str): id of the project to query.
             project_number (str): number of the project to query.
             dataset_id (str): id of the dataset to query.
 
         Returns:
             dict: Dataset Policy.
         """
+        del project_id
+
         return self.bigquery.get_dataset_access(project_number, dataset_id)
 
-    def fetch_bigquery_iam_policy(self, project_number, dataset_id):
+    def fetch_bigquery_iam_policy(self, project_id, project_number, dataset_id):
         """Gets IAM policy of a bigquery dataset from gcp API call.
 
         Args:
+            project_id (str): id of the project to query.
             project_number (str): number of the project to query.
             dataset_id (str): id of the dataset to query.
 

google/cloud/forseti/services/inventory/base/resources.py

@@ -875,6 +875,7 @@ def get_iam_policy(self, client=None):
         """
         try:
             iam_policy = client.fetch_bigquery_iam_policy(
+                self.parent()['projectId'],
                 self.parent()['projectNumber'],
                 self['datasetReference']['datasetId'])
             dataset_policy = iam_helpers.convert_iam_to_bigquery_policy(
@@ -898,6 +899,7 @@ def get_dataset_policy(self, client=None):
         """
         try:
             dataset_policy = client.fetch_bigquery_dataset_policy(
+                self.parent()['projectId'],
                 self.parent()['projectNumber'],
                 self['datasetReference']['datasetId'])
             iam_policy = iam_helpers.convert_bigquery_policy_to_iam(

tests/services/inventory/crawling_test.py

@@ -286,7 +286,6 @@ def test_crawling_no_org_access(self):
         expected_counts.pop('gsuite_user')
         expected_counts.pop('gsuite_user_member')
 
-
         self.assertEqual(expected_counts, result_counts)
 
     def test_crawling_with_apis_disabled(self):
@@ -387,7 +386,7 @@ def test_cai_crawl_to_memory(self):
             'compute_targetsslproxy': {'resource': 1},
             'compute_targettcpproxy': {'resource': 1},
             'compute_urlmap': {'resource': 1},
-            'dataset': {'dataset_policy': 1, 'iam_policy': 1, 'resource': 2},
+            'dataset': {'dataset_policy': 2, 'iam_policy': 2, 'resource': 3},
             'dns_managedzone': {'resource': 1},
             'dns_policy': {'resource': 1},
             'kms_cryptokey': {'iam_policy': 1, 'resource': 1},
@@ -442,7 +441,7 @@ def test_crawl_cai_api_polling_disabled(self):
             'compute_targetsslproxy': {'resource': 1},
             'compute_targettcpproxy': {'resource': 1},
             'compute_urlmap': {'resource': 1},
-            'dataset': {'dataset_policy': 1, 'iam_policy': 1, 'resource': 2},
+            'dataset': {'dataset_policy': 2, 'iam_policy': 2, 'resource': 3},
             'disk': {'resource': 4},
             'dns_managedzone': {'resource': 1},
             'dns_policy': {'resource': 1},

tests/services/inventory/test_data/additional_cai_iam_policies.dump

@@ -2,8 +2,10 @@
 # enabled. These resources are merged into mock_cai_iam_policies.dump by the
 # update_cai_dumps.py script. Any lines that start with a '#' are ignored.
 #
-# Bigquery Dataset
+# Bigquery Dataset with project number
 {"name":"//bigquery.googleapis.com/projects/1042/datasets/bq_test_ds","asset_type":"google.cloud.bigquery.Dataset","iam_policy":{"bindings":[{"role":"roles/bigquery.dataEditor","members":["group:test-group@forseti.testing","projectEditor:project2"]},{"role":"roles/bigquery.dataOwner","members":["projectOwner:project2","user:user@forseti.testing"]},{"role":"roles/bigquery.dataViewer","members":["allAuthenticatedUsers","domain:forseti.testing","projectViewer:project2"]}]}}
+# Bigquery Dataset with project id
+{"name":"//bigquery.googleapis.com/projects/project3/datasets/bq_test_ds1","asset_type":"google.cloud.bigquery.Dataset","iam_policy":{"bindings":[{"role":"roles/bigquery.dataEditor","members":["group:test-group@forseti.testing","projectEditor:project3"]},{"role":"roles/bigquery.dataOwner","members":["projectOwner:project3","user:user@forseti.testing"]},{"role":"roles/bigquery.dataViewer","members":["allAuthenticatedUsers","domain:forseti.testing","projectViewer:project3"]}]}}
 # KMS CryptoKey
 {"name":"//cloudkms.googleapis.com/projects/project2/locations/us/keyRings/test-kr/cryptoKeys/test-key","asset_type":"google.cloud.kms.CryptoKey","iam_policy":{"etag":"ACAB","bindings":[{"role":"roles/cloudkms.cryptoKeyEncrypterDecrypter","members":["user:user@forseti.testing"]}]}}
 # KMS KeyRing

tests/services/inventory/test_data/additional_cai_resources.dump

@@ -2,7 +2,9 @@
 # enabled. These resources are merged into mock_cai_resources.dump by the
 # update_cai_dumps.py script. Any lines that start with a '#' are ignored.
 #
-# Bigquery Dataset
+# Bigquery Dataset with project id
+{"name":"//bigquery.googleapis.com/projects/project3/datasets/bq_test_ds1","asset_type":"google.cloud.bigquery.Dataset", "resource":{"version":"v2","discovery_document_uri":"https://bigquery.googleapis.com/$discovery/rest","discovery_name":"Dataset","parent":"//cloudresourcemanager.googleapis.com/projects/1043","data":{"creationTime":"1540220016171","datasetReference":{"datasetId":"bq_test_ds1","projectId":"project3"},"location":"us","id":"project3:bq_test_ds1","kind":"bigquery#dataset","lastModifiedTime":"1540220016171"}}}
+# Bigquery Dataset with project name
 {"name":"//bigquery.googleapis.com/projects/1042/datasets/bq_test_ds","asset_type":"google.cloud.bigquery.Dataset", "resource":{"version":"v2","discovery_document_uri":"https://bigquery.googleapis.com/$discovery/rest","discovery_name":"Dataset","parent":"//cloudresourcemanager.googleapis.com/projects/1042","data":{"creationTime":"1540220016171","datasetReference":{"datasetId":"bq_test_ds","projectId":"project2"},"location":"us","id":"project2:bq_test_ds","kind":"bigquery#dataset","lastModifiedTime":"1540220016171"}}}
 # Compute Autoscalar (id 900x)
 {"name":"//compute.googleapis.com/projects/project2/zones/us-central1-b/autoscalers/gae-default-20161128t212211","asset_type":"google.compute.Autoscaler","resource":{"version":"v1","discovery_document_uri":"https://www.googleapis.com/discovery/v1/apis/compute/v1/rest","discovery_name":"Autoscaler","parent":"//cloudresourcemanager.googleapis.com/projects/1042","data":{"autoscalingPolicy":{"coolDownPeriodSec":120,"cpuUtilization":{"utilizationTarget":0.5},"maxNumReplicas":20,"minNumReplicas":2},"creationTimestamp":"2016-11-28T21:27:35.916-08:00","id":"9001","name":"gae-default-20161128t212211","selfLink":"https://www.googleapis.com/compute/v1/projects/project2/zones/us-central1-b/autoscalers/gae-default-20161128t212211","status":"ACTIVE","target":"https://www.googleapis.com/replicapool/v1beta1/projects/project2/zones/us-central1-b/pools/gae-default-20161128t212211","zone":"https://www.googleapis.com/compute/v1/projects/project2/zones/us-central1-b"}}}

tests/services/inventory/test_data/mock_cai_iam_policies.dump

@@ -13,6 +13,7 @@
 {"asset_type":"google.cloud.resourcemanager.Project","iam_policy":{"bindings":[{"members":["serviceAccount:4@cloudservices.gserviceaccount.com","serviceAccount:4-compute@developer.gserviceaccount.com","serviceAccount:4@cloudservices.gserviceaccount.com"],"role":"roles/editor"},{"members":["group:c_grp@forseti.test","user:a_user@forseti.test"],"role":"roles/owner"}]},"name":"//cloudresourcemanager.googleapis.com/projects/1044"}
 {"asset_type":"google.cloud.resourcemanager.Project","iam_policy":{"auditConfigs":[{"auditLogConfigs":[{"logType":"ADMIN_READ"},{"logType":"DATA_WRITE"},{"logType":"DATA_READ"}],"service":"allServices"},{"auditLogConfigs":[{"exemptedMembers":["user:gcp-reader-12345@p1234.iam.gserviceaccount.com"],"logType":"ADMIN_READ"}],"service":"cloudsql.googleapis.com"}],"bindings":[{"members":["serviceAccount:2@cloudservices.gserviceaccount.com","serviceAccount:2-compute@developer.gserviceaccount.com"],"role":"roles/editor"},{"members":["group:c_grp@forseti.test","user:a_user@forseti.test"],"role":"roles/owner"}]},"name":"//cloudresourcemanager.googleapis.com/projects/1042"}
 {"name":"//bigquery.googleapis.com/projects/1042/datasets/bq_test_ds","asset_type":"google.cloud.bigquery.Dataset","iam_policy":{"bindings":[{"role":"roles/bigquery.dataEditor","members":["group:test-group@forseti.testing","projectEditor:project2"]},{"role":"roles/bigquery.dataOwner","members":["projectOwner:project2","user:user@forseti.testing"]},{"role":"roles/bigquery.dataViewer","members":["allAuthenticatedUsers","domain:forseti.testing","projectViewer:project2"]}]}}
+{"name":"//bigquery.googleapis.com/projects/project3/datasets/bq_test_ds1","asset_type":"google.cloud.bigquery.Dataset","iam_policy":{"bindings":[{"role":"roles/bigquery.dataEditor","members":["group:test-group@forseti.testing","projectEditor:project3"]},{"role":"roles/bigquery.dataOwner","members":["projectOwner:project3","user:user@forseti.testing"]},{"role":"roles/bigquery.dataViewer","members":["allAuthenticatedUsers","domain:forseti.testing","projectViewer:project3"]}]}}
 {"name":"//cloudkms.googleapis.com/projects/project2/locations/us/keyRings/test-kr/cryptoKeys/test-key","asset_type":"google.cloud.kms.CryptoKey","iam_policy":{"etag":"ACAB","bindings":[{"role":"roles/cloudkms.cryptoKeyEncrypterDecrypter","members":["user:user@forseti.testing"]}]}}
 {"name":"//cloudkms.googleapis.com/projects/project2/locations/us/keyRings/test-kr","asset_type":"google.cloud.kms.KeyRing","iam_policy":{"etag":"ACAB","bindings":[{"role":"roles/cloudkms.admin","members":["user:user@forseti.testing"]}]}}
 {"name":"//pubsub.googleapis.com/projects/project2/topics/test-topic0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123","asset_type":"google.pubsub.Topic","iam_policy":{"etag":"BwV5a9uxYDM=","bindings":[{"role":"roles/pubsub.publisher","members":["user:user@forseti.testing"]}]}}