Add groups raw json data. (#189)

forseti-security · Apr 7, 2017 · 1f8dd45 · 1f8dd45
1 parent 88d0422
commit 1f8dd45
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 3 deletions.
diff --git a/google/cloud/security/common/data_access/csv_writer.py b/google/cloud/security/common/data_access/csv_writer.py
@@ -22,7 +22,8 @@
 
 GROUPS_FIELDNAMES = [
     'group_id',
-    'group_email'
+    'group_email',
+    'raw_group'
 ]
 
 ORG_IAM_POLICIES_FIELDNAMES = [

diff --git a/google/cloud/security/common/data_access/sql_queries/create_tables.py b/google/cloud/security/common/data_access/sql_queries/create_tables.py
@@ -79,6 +79,7 @@
         `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
         `group_id` varchar(255) DEFAULT NULL,
         `group_email` varchar(255) DEFAULT NULL,
+        `raw_group` json DEFAULT NULL,
         PRIMARY KEY (`id`)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 """
diff --git a/google/cloud/security/inventory/pipelines/load_groups_pipeline.py b/google/cloud/security/inventory/pipelines/load_groups_pipeline.py
@@ -14,6 +14,8 @@
 
 """Pipeline to load GSuite Account Groups into Inventory."""
 
+import json
+
 from google.cloud.security.common.gcp_api import errors as api_errors
 from google.cloud.security.common.util import metadata_server
 from google.cloud.security.inventory import errors as inventory_errors
@@ -78,10 +80,11 @@ def _transform(self, groups_map):
         """
         for group in groups_map:
             yield {'group_id': group.get('id'),
-                   'group_email': group.get('email')}
+                   'group_email': group.get('email'),
+                   'raw_group': json.dumps(group)}
 
     def _retrieve(self):
-        """Retrieve the org IAM policies from GCP.
+        """Retrieve the groups from GCP.
 
         Returns:
             A list of group objects returned from the API.

diff --git a/tests/inventory/pipelines/test_data/fake_groups.py b/tests/inventory/pipelines/test_data/fake_groups.py
@@ -56,13 +56,16 @@
     {
         'group_id': '11111',
         'group_email': 'aaaaa@foo.com',
+        'raw_group': '{"nonEditableAliases": ["aaaaa@foo.com"], "kind": "admin#directory#group", "etag": "\\"pCd5iosDe_tWdPv4ke8sAYzlGK8/oWZC62Ysx9kAKLlW23uoKQlYu3k\\"", "name": "aaaaa", "adminCreated": true, "description": "", "directMembersCount": "1", "email": "aaaaa@foo.com", "id": "11111"}',
     },
     {
         'group_id': '22222',
         'group_email': 'bbbbb@foo.com',
+        'raw_group': '{"nonEditableAliases": ["bbbbb@foo.com"], "kind": "admin#directory#group", "etag": "\\"pCd5iosDe_tWdPv4ke8sAYzlGK8/cglP2U9YgiKA9zjJ-DvxjotnaLU\\"", "name": "bbbbb", "adminCreated": false, "description": "", "directMembersCount": "1", "email": "bbbbb@foo.com", "id": "22222"}',
     },
     {
         'group_id': '33333',
         'group_email': 'ccccc@foo.com',
+        'raw_group': '{"nonEditableAliases": ["ccccc@foo.com"], "kind": "admin#directory#group", "etag": "\\"pCd5iosDe_tWdPv4ke8sAYzlGK8/kQ2NdfLnWQTiAs-FCSEKJRaipxw\\"", "name": "CCCCC Users", "adminCreated": true, "description": "Members of this group will be allowed to perform bar.", "directMembersCount": "4", "email": "ccccc@foo.com", "id": "33333"}',
     }
 ]