Merge pull request #152 from nccgroup/feature/120-gcp-sa-multi-projec…

…t-scanning

Feature/120 gcp sa multi project scanning

ScoutSuite/__main__.py

@@ -82,6 +82,8 @@ def main(passed_args=None):
     if cloud_provider.provider_code == 'azure':
         report_file_name = 'azure'
 
+    # TODO move this to after authentication, so that the report can be more specific to what's being scanned.
+    # For example if scanning with a GCP service account, the SA email can only be known after authenticating...
     # Create a new report
     report = Scout2Report(args.provider, report_file_name, args.report_dir, args.timestamp)
 

ScoutSuite/providers/gcp/configs/base.py

@@ -156,6 +156,12 @@ def _get_targets(self, response_attribute, api_client, method, list_params, igno
                 try:
 
                     if self.library_type == 'cloud_client_library':
+
+                        # TODO this should be more modular
+                        # this is only for stackdriverlogging
+                        if self.service == 'stackdriverlogging':
+                            api_client.project = list_params_combination.pop('project')
+
                         response = method(**list_params_combination)
 
                         # TODO this should be more modular

ScoutSuite/providers/gcp/configs/services.py

@@ -20,6 +20,7 @@
 class GCPServicesConfig(BaseServicesConfig):
 
     def __init__(self, metadata=None, thread_config=4, projects=None, **kwargs):
+
         projects = [] if projects is None else projects
 
         self.cloudresourcemanager = CloudResourceManager(thread_config=thread_config)

ScoutSuite/providers/gcp/provider.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-[d['value'] for d in l]
 
 import os
-
 import warnings
+
 import google.auth
 import googleapiclient
 from opinel.utils.console import printError, printException
@@ -18,6 +18,7 @@ def __init__(self, api_client_credentials, cloud_client_credentials):
         self.api_client_credentials = api_client_credentials
         self.cloud_client_credentials = cloud_client_credentials
 
+
 class GCPProvider(BaseProvider):
     """
     Implements provider for GCP
@@ -68,38 +69,45 @@ def authenticate(self, key_file=None, user_account=None, service_account=None, *
             self.credentials, project_id = google.auth.default()
             if self.credentials:
 
+                # Project passed through the CLI
                 if self.project_id:
-                    # service_account credentials with project_id will follow this path
                     self.projects = self._get_projects(parent_type='project',
                                                        parent_id=self.project_id)
-                    self.aws_account_id = self.project_id # FIXME this is for AWS
-                    self.profile = self.project_id # FIXME this is for AWS
-
-                elif self.organization_id:
-                    self.projects = self._get_projects(parent_type='organization',
-                                                       parent_id=self.organization_id)
-                    self.aws_account_id = self.organization_id # FIXME this is for AWS
-                    self.profile = self.organization_id # FIXME this is for AWS
+                    self.aws_account_id = self.project_id  # FIXME this is for AWS
+                    self.profile = self.project_id  # FIXME this is for AWS
 
+                # Folder passed through the CLI
                 elif self.folder_id:
                     self.projects = self._get_projects(parent_type='folder',
                                                        parent_id=self.folder_id)
-                    self.aws_account_id = self.folder_id # FIXME this is for AWS
-                    self.profile = self.folder_id # FIXME this is for AWS
+                    self.aws_account_id = self.folder_id  # FIXME this is for AWS
+                    self.profile = self.folder_id  # FIXME this is for AWS
 
-                elif self.service_account: # We know that project_id hasn't been provided and that we have a service account
-                    self.projects = self._get_projects(parent_type='service-account',
-                                                       parent_id=self.project_id)
-                    self.aws_account_id = self.credentials.service_account_email # FIXME this is for AWS
-                    self.profile = self.credentials.service_account_email # FIXME this is for AWS
+                # Organization passed through the CLI
+                elif self.organization_id:
+                    self.projects = self._get_projects(parent_type='organization',
+                                                       parent_id=self.organization_id)
+                    self.aws_account_id = self.organization_id  # FIXME this is for AWS
+                    self.profile = self.organization_id  # FIXME this is for AWS
 
-                else:
-                    # FIXME this will fail if no default project is set in gcloud config. This is caused because html.py is looking for a profile to build the report
-                    self.project_id = project_id
+                # Project inferred from default configuration
+                if project_id:
                     self.projects = self._get_projects(parent_type='project',
-                                                       parent_id=self.project_id)
-                    self.aws_account_id = self.project_id # FIXME this is for AWS
-                    self.profile = self.project_id # FIXME this is for AWS
+                                                       parent_id=project_id)
+                    self.aws_account_id = project_id  # FIXME this is for AWS
+                    self.profile = project_id  # FIXME this is for AWS
+
+                # All projects to which the user / Service Account has access to
+                else:
+                    # self.project_id = project_id
+                    self.projects = self._get_projects(parent_type='all',
+                                                       parent_id=None)
+                    if service_account and hasattr(self.credentials, 'service_account_email'):
+                        self.aws_account_id = self.credentials.service_account_email  # FIXME this is for AWS
+                    else:
+                        self.aws_account_id = 'GCP'  # FIXME this is for AWS
+
+                    self.profile = 'GCP'  # FIXME this is for AWS
 
                 # TODO this shouldn't be done here? but it has to in order to init with projects...
                 self.services.set_projects(projects=self.projects)
@@ -118,7 +126,6 @@ def authenticate(self, key_file=None, user_account=None, service_account=None, *
             printException(e)
             return False
 
-
     def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None):
         """
         TODO description
@@ -141,12 +148,12 @@ def _get_projects(self, parent_type, parent_id):
         details.
         """
 
-        if parent_type not in ['project', 'organization', 'folder', 'service-account']:
+        if parent_type not in ['project', 'organization', 'folder', 'all']:
             return None
 
         projects = []
 
-        #FIXME can't currently be done with API client library as it consumes v1 which doesn't support folders
+        # FIXME can't currently be done with API client library as it consumes v1 which doesn't support folders
         """
 
         resource_manager_client = resource_manager.Client(credentials=self.credentials)
@@ -159,7 +166,8 @@ def _get_projects(self, parent_type, parent_id):
         """
 
         resource_manager_client_v1 = gcp_connect_service(service='cloudresourcemanager', credentials=self.credentials)
-        resource_manager_client_v2 = gcp_connect_service(service='cloudresourcemanager-v2', credentials=self.credentials)
+        resource_manager_client_v2 = gcp_connect_service(service='cloudresourcemanager-v2',
+                                                         credentials=self.credentials)
 
         if parent_type == 'project':
             project_response = resource_manager_client_v1.projects().list(filter='id:%s' % parent_id).execute()
@@ -168,15 +176,15 @@ def _get_projects(self, parent_type, parent_id):
                     if project['lifecycleState'] == "ACTIVE":
                         projects.append(project)
 
-        elif parent_type == 'service-account':
+        elif parent_type == 'all':
             project_response = resource_manager_client_v1.projects().list().execute()
             if 'projects' in project_response.keys():
                 for project in project_response['projects']:
                     if project['lifecycleState'] == "ACTIVE":
                         projects.append(project)
         else:
 
-            # get parent children projectss
+            # get parent children projects
             request = resource_manager_client_v1.projects().list(filter='parent.id:%s' % parent_id)
             while request is not None:
                 response = request.execute()
@@ -190,7 +198,8 @@ def _get_projects(self, parent_type, parent_id):
                                                                           previous_response=response)
 
             # get parent children projects in children folders recursively
-            folder_response = resource_manager_client_v2.folders().list(parent='%ss/%s' % (parent_type, parent_id)).execute()
+            folder_response = resource_manager_client_v2.folders().list(
+                parent='%ss/%s' % (parent_type, parent_id)).execute()
             if 'folders' in folder_response.keys():
                 for folder in folder_response['folders']:
                     projects.extend(self._get_projects("folder", folder['name'].strip(u'folders/')))
@@ -216,7 +225,6 @@ def _match_instances_and_snapshots(self):
                                                            key=lambda x: x['creation_timestamp']) \
                         if instance_disk['snapshots'] else None
 
-
     def _match_networks_and_instances(self):
         """
         For each network, math instances in that network

ScoutSuite/providers/gcp/services/stackdriverlogging.py

@@ -7,7 +7,7 @@
 
 class StackdriverLoggingConfig(GCPBaseConfig):
     targets = (
-        ('sinks', 'Sinks', 'list_sinks', {}, False),
+        ('sinks', 'Sinks', 'list_sinks', {'project': '{{project_placeholder}}'}, False),
     )
 
     def __init__(self, thread_config):

ScoutSuite/providers/gcp/utils.py

@@ -26,7 +26,9 @@ def gcp_connect_service(service, credentials=None, region_name=None):
             return discovery.build('cloudresourcemanager', 'v2', cache_discovery=False, cache=MemoryCache())
 
         elif service == 'cloudstorage':
-            return storage.Client()
+            # return storage.Client()
+            return storage.Client(project='placeholder')  # need a project value to instantiate the client event though
+                                                          # it won't be used afterwards
 
         elif service == 'cloudsql':
             return discovery.build('sqladmin', 'v1beta4', cache_discovery=False, cache=MemoryCache())
@@ -35,7 +37,9 @@ def gcp_connect_service(service, credentials=None, region_name=None):
             return discovery.build('iam', 'v1', cache_discovery=False, cache=MemoryCache())
 
         if service == 'stackdriverlogging':
-            return stackdriver_logging.Client()
+            # return stackdriver_logging.LoggingServiceV2Client()
+            return stackdriver_logging.Client(project='placeholder')  # need a project value to instantiate the client
+                                                                      # event though it won't be used afterwards
 
         if service == 'stackdrivermonitoring':
             return monitoring_v3.MetricServiceClient()

requirements.txt

@@ -7,15 +7,17 @@ boto3>=1.9.60
 opinel>=3.3.4,<4.0.0
 
 # GCP Provider
-grpcio
+grpcio>=1.18.0
 ## Cloud Libraries
-google-cloud-resource-manager
-google-cloud-storage
-google-cloud-container
-google-cloud-logging
-google-cloud-monitoring
+google-cloud-container>=0.2.1
+google-cloud-core>=0.29.1
+google-cloud-iam>=0.1.0
+google-cloud-logging>=1.10.0
+google-cloud-monitoring>=0.31.1
+google-cloud-resource-manager>=0.28.3
+google-cloud-storage>=1.13.2
 ## API Client Libraries
-google_api_python_client
+google-api-python-client>=1.7.8
 oauth2client>=4.1.3
 
 # Azure Provider