CAI Integration - Installer (#2061)

* updates

* updated gustil iam ch command

* Included cloudasset api in the enabling process.

* updates

* turn on cai by default

* Added missing comma

* variable naming updates

* updates

* updates

* fixed indentations

* updates

* Addressed PR comments

* importing bucket_cai.py in deployment template

* Updated to use context properties.

* Using object instead of string for lifecycle field

configs/server/forseti_conf_server.yaml.in

@@ -62,8 +62,8 @@ inventory:
           period: 1.1
 
     cai:
-        enabled: False
-        gcs_path: ""
+        enabled: {CAI_ENABLED}
+        gcs_path: gs://{FORSETI_CAI_BUCKET}
 
     # Number of days to retain inventory data:
     #  -1 : (default) keep all previous data forever

deployment-templates/deploy-forseti-server.yaml.in

@@ -25,6 +25,8 @@ imports:
   name: forseti-instance-server.py
 - path: storage/bucket.py
   name: bucket.py
+- path: storage/bucket_cai.py
+  name: bucket_cai.py
 
 resources:
 
@@ -42,6 +44,11 @@ resources:
   type: bucket.py
   properties:
     location: {BUCKET_LOCATION}
+- name: {FORSETI_CAI_BUCKET}
+  type: bucket_cai.py
+  properties:
+    location: {BUCKET_LOCATION}
+    retention_days: 14
 
 # Compute Engine
 - name: forseti-instance-server

deployment-templates/storage/bucket_cai.py

@@ -0,0 +1,39 @@
+# Copyright 2017 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Creates a Cloud Storage bucket template for Forseti Security."""
+
+def GenerateConfig(context):
+    """Generate configuration."""
+    resources = []
+
+    resources.append({
+        'name': context.env['name'],
+        'type': 'storage.v1.bucket',
+        'properties': {
+            'project': context.env['project'],
+            'lifecycle':
+                {
+                    "rule":
+                        [{
+                            "action": {"type": "Delete" },
+                            "condition": {
+                                "age": context.properties['retention_days']}
+                        }]
+                },
+            'location': context.properties['location'],
+        }
+    })
+
+    return {'resources': resources}

install/gcp/installer/forseti_server_installer.py

@@ -100,6 +100,7 @@ def deploy(self, deployment_tpl_path, conf_file_path, bucket_name):
                 self.target_id,
                 self.project_id,
                 self.gcp_service_acct_email,
+                self._get_cai_bucket_name(),
                 self.user_can_grant_roles)
 
             # Waiting for VM to be initialized.
@@ -174,6 +175,7 @@ def get_deployment_values(self):
             'CLOUDSQL_REGION': self.config.cloudsql_region,
             'CLOUDSQL_INSTANCE_NAME': self.config.cloudsql_instance,
             'FORSETI_BUCKET': bucket_name[len('gs://'):],
+            'FORSETI_CAI_BUCKET': self._get_cai_bucket_name(),
             'BUCKET_LOCATION': self.config.bucket_location,
             'GCP_SERVER_SERVICE_ACCOUNT': self.gcp_service_acct_email,
             'FORSETI_SERVER_REGION': self.config.cloudsql_region,
@@ -194,10 +196,12 @@ def get_configuration_values(self):
         """
         bucket_name = self.generate_bucket_name()
         return {
+            'CAI_ENABLED': 'organizations' in self.resource_root_id,
             'EMAIL_RECIPIENT': self.config.notification_recipient_email,
             'EMAIL_SENDER': self.config.notification_sender_email,
             'SENDGRID_API_KEY': self.config.sendgrid_api_key,
             'FORSETI_BUCKET': bucket_name[len('gs://'):],
+            'FORSETI_CAI_BUCKET': self._get_cai_bucket_name(),
             'DOMAIN_SUPER_ADMIN_EMAIL': self.config.gsuite_superadmin_email,
             'ROOT_RESOURCE_ID': self.resource_root_id,
         }
@@ -320,6 +324,14 @@ def post_install_instructions(self, deploy_success,
 
         return instructions
 
+    def _get_cai_bucket_name(self):
+        """Get CAI bucket name.
+
+        Returns:
+            str: CAI bucket name.
+        """
+        return 'forseti-cai-export-{}'.format(self.config.identifier)
+
     @staticmethod
     def _get_gcs_path(resources):
         """Get gcs path from resources.

install/gcp/installer/util/constants.py

@@ -86,14 +86,15 @@ class DeploymentStatus(Enum):
 
 # Roles
 GCP_READ_IAM_ROLES = [
+    'roles/appengine.appViewer',
+    'roles/bigquery.dataViewer',
     'roles/browser',
+    'roles/cloudasset.viewer',
+    'roles/cloudsql.viewer',
     'roles/compute.networkViewer',
     'roles/iam.securityReviewer',
-    'roles/appengine.appViewer',
-    'roles/bigquery.dataViewer',
     'roles/servicemanagement.quotaViewer',
     'roles/serviceusage.serviceUsageConsumer',
-    'roles/cloudsql.viewer'
 ]
 
 GCP_WRITE_IAM_ROLES = [
@@ -112,6 +113,10 @@ class DeploymentStatus(Enum):
     'roles/logging.logWriter'
 ]
 
+FORSETI_CAI_BUCKET_ROLES = [
+    'objectAdmin'
+]
+
 SVC_ACCT_ROLES = [
     'roles/iam.serviceAccountTokenCreator'
 ]
@@ -129,6 +134,8 @@ class DeploymentStatus(Enum):
      'service': 'appengine.googleapis.com'},
     {'name': 'BigQuery',
      'service': 'bigquery-json.googleapis.com'},
+    {'name': 'Cloud Asset API',
+     'service': 'cloudasset.googleapis.com'},
     {'name': 'Cloud Billing',
      'service': 'cloudbilling.googleapis.com'},
     {'name': 'Cloud Resource Manager',

install/gcp/installer/util/gcloud.py

@@ -16,6 +16,7 @@
 
 from __future__ import print_function
 import json
+import os.path
 import re
 import sys
 
@@ -213,6 +214,7 @@ def grant_server_svc_acct_roles(enable_write,
                                 target_id,
                                 project_id,
                                 gcp_service_account,
+                                cai_bucket_name,
                                 user_can_grant_roles):
     """Grant the following IAM roles to GCP service account.
 
@@ -225,16 +227,17 @@ def grant_server_svc_acct_roles(enable_write,
         Cloud SQL Client, Storage Object Viewer, Storage Object Creator
 
     Args:
-        enable_write (bool): Whether or not to enable write access
-        access_target (str): Access target, either org, folder or project
-        target_id (str): Id of the access_target
-        project_id (str): GCP Project Id
-        gcp_service_account (str): GCP service account email
+        enable_write (bool): Whether or not to enable write access.
+        access_target (str): Access target, either org, folder or project.
+        target_id (str): Id of the access_target.
+        project_id (str): GCP Project Id.
+        gcp_service_account (str): GCP service account email.
+        cai_bucket_name (str): The name of the CAI bucket.
         user_can_grant_roles (bool): Whether or not user has
-                                        access to grant roles
+            access to grant roles.
 
     Returns:
-        bool: Whether or not a role script has been generated
+        bool: Whether or not a role script has been generated.
     """
 
     utils.print_banner('Assigning Roles To The GCP Service Account',
@@ -249,10 +252,84 @@ def grant_server_svc_acct_roles(enable_write,
         'service_accounts': constants.SVC_ACCT_ROLES,
     }
 
-    return _grant_svc_acct_roles(
+    has_role_script_bucket = _grant_bucket_roles(
+        gcp_service_account,
+        cai_bucket_name,
+        constants.FORSETI_CAI_BUCKET_ROLES,
+        user_can_grant_roles)
+
+    has_role_script_rest = _grant_svc_acct_roles(
         target_id, project_id, gcp_service_account,
         user_can_grant_roles, roles)
 
+    return has_role_script_bucket or has_role_script_rest
+
+
+def _grant_bucket_roles(gcp_service_account,
+                        bucket_name,
+                        roles_to_grant,
+                        user_can_grant_roles):
+    """Grant GCS bucket level roles to GCP service account.
+
+    Note: This is only supported through gsutil library and not in
+    gcloud, that's why we need this one off method to grant the role.
+
+    Command structure:
+    gsutil iam ch [MEMBER_TYPE]:[MEMBER_NAME]:[ROLE] gs://[BUCKET_NAME]
+
+    Args:
+        gcp_service_account (str): GCP service account email.
+        bucket_name (str): Name of the bucket to grant the role on.
+        user_can_grant_roles (bool): Whether or not user has
+            access to grant roles.
+        roles_to_grant (list): List of roles to grant.
+    Returns:
+        bool: Whether or not a role script has been generated.
+    """
+    failed_commands = []
+
+    for role in roles_to_grant:
+        member = 'serviceAccount:{}:{}'.format(gcp_service_account, role)
+        bucket = 'gs://{}'.format(bucket_name)
+        bucket_role_cmd = ['gsutil', 'iam', 'ch', member, bucket]
+
+        if user_can_grant_roles:
+            print('Assigning {} on {}... '.format(member, bucket), end='')
+            sys.stdout.flush()
+            return_code, _, err = utils.run_command(bucket_role_cmd)
+            if return_code:
+                # Role not assigned, save the commands and write the command to
+                # the role script later.
+                failed_commands.append('%s\n' % ' '.join(bucket_role_cmd))
+                print(err)
+            else:
+                print('assigned')
+        else:
+            failed_commands.append('%s\n' % ' '.join(bucket_role_cmd))
+
+    if failed_commands:
+        file_name = 'grant_forseti_roles.sh'
+        _generate_script_file(file_name, failed_commands, 'a+')
+        print(constants.MESSAGE_CREATE_ROLE_SCRIPT)
+        return True
+    return False
+
+
+def _generate_script_file(file_name, role_commands, file_mode='wt'):
+    """Generate a shell script file that contains all the role commands.
+
+    Args:
+        file_name (str): File name of the shell script.
+        role_commands (list): A list of role assignment commands.
+        file_mode (str): File mode.
+    """
+    need_shebang = not os.path.isfile(file_name)
+    with open(file_name, file_mode) as roles_script:
+        if need_shebang:
+            roles_script.write('#!/bin/bash\n\n')
+        for role_command in role_commands:
+            roles_script.write(role_command)
+
 
 def _grant_svc_acct_roles(target_id,
                           project_id,
@@ -278,11 +355,9 @@ def _grant_svc_acct_roles(target_id,
 
     if grant_roles_cmds:
         print(constants.MESSAGE_CREATE_ROLE_SCRIPT)
-
-        with open('grant_forseti_roles.sh', 'wt') as roles_script:
-            roles_script.write('#!/bin/bash\n\n')
-            for cmd in grant_roles_cmds:
-                roles_script.write('%s\n' % ' '.join(cmd))
+        failed_commands = ['%s\n' % ' '.join(cmd) for cmd in grant_roles_cmds]
+        file_name = 'grant_forseti_roles.sh'
+        _generate_script_file(file_name, failed_commands, 'a+')
         return True
     return False