Skip to content
This repository has been archived by the owner on Jun 5, 2023. It is now read-only.

Commit

Permalink
Revert "Run External Project Access Scanner Separately" (#2261)
Browse files Browse the repository at this point in the history
  • Loading branch information
@joecheuk
joecheuk committed Nov 15, 2018
1 parent 0ef4e15 commit 18c56c4
Show file tree
Hide file tree
Showing 11 changed files with 86 additions and 152 deletions.
1 change: 0 additions & 1 deletion google/cloud/forseti/common/data_access/violation_map.py
View file
Expand Up @@ -49,5 +49,4 @@
'RETENTION_VIOLATION': 'retention_violations',
'SERVICE_ACCOUNT_KEY_VIOLATION': (
'service_account_key_violations'),
'EXTERNAL_PROJECT_ACCESS_VIOLATION': 'external_project_access_violations'
}
10 changes: 4 additions & 6 deletions google/cloud/forseti/scanner/scanner.py
View file
Expand Up @@ -77,10 +77,7 @@ def mark_scanner_index_complete(
session.flush()


def run(model_name=None,
progress_queue=None,
service_config=None,
scanner_name=None):
def run(model_name=None, progress_queue=None, service_config=None):
"""Run the scanners.
Entry point when the scanner is run as a library.
Expand All @@ -89,12 +86,13 @@ def run(model_name=None,
model_name (str): The name of the data model.
progress_queue (Queue): The progress queue.
service_config (ServiceConfig): Forseti 2.0 service configs.
scanner_name (str): name of the scanner that runs separately.
Returns:
int: Status code.
"""
global_configs = service_config.get_global_config()
scanner_configs = service_config.get_scanner_config()

with service_config.scoped_session() as session:
service_config.violation_access = scanner_dao.ViolationAccess(session)
model_description = (
Expand All @@ -104,7 +102,7 @@ def run(model_name=None,
scanner_index_id = init_scanner_index(session, inventory_index_id)
runnable_scanners = scanner_builder.ScannerBuilder(
global_configs, scanner_configs, service_config, model_name,
None, scanner_name).build()
None).build()

succeeded = []
failed = []
Expand Down
138 changes: 57 additions & 81 deletions google/cloud/forseti/scanner/scanner_builder.py
View file
Expand Up @@ -14,14 +14,12 @@

"""Builds the scanners to run."""

# pylint: disable=line-too-long

import importlib
import inspect

from google.cloud.forseti.common.util import logger
from google.cloud.forseti.scanner import scanner_requirements_map
from google.cloud.forseti.scanner.scanners import external_project_access_scanner # noqa=E501


LOGGER = logger.get_logger(__name__)

Expand All @@ -30,7 +28,7 @@ class ScannerBuilder(object):
"""Scanner Builder."""

def __init__(self, global_configs, scanner_configs, service_config,
model_name, snapshot_timestamp, scanner_name=None):
model_name, snapshot_timestamp):
"""Initialize the scanner builder.
Args:
Expand All @@ -39,14 +37,12 @@ def __init__(self, global_configs, scanner_configs, service_config,
service_config (ServiceConfig): Service configuration.
model_name (str): name of the data model
snapshot_timestamp (str): The snapshot timestamp
scanner_name (str): Name of the specified scanner to run separately
"""
self.global_configs = global_configs
self.scanner_configs = scanner_configs
self.service_config = service_config
self.model_name = model_name
self.snapshot_timestamp = snapshot_timestamp
self.scanner_name = scanner_name

def build(self):
"""Build the enabled scanners to run.
Expand All @@ -55,80 +51,60 @@ def build(self):
list: Scanner instances that will be run.
"""
runnable_scanners = []
if self.scanner_name == 'external_project_access_scanner':
scanner = self._instantiate_scanner(
external_project_access_scanner.
ExternalProjectAccessScanner,
'external_project_access_rules.yaml')
runnable_scanners.append(scanner)
else:
for scanner in self.scanner_configs.get('scanners'):
if scanner.get('enabled'):
module_path = 'google.cloud.forseti.scanner.scanners.{}'

requirements_map = scanner_requirements_map.REQUIREMENTS_MAP
if not scanner.get('name') in requirements_map:
log_message = (
'Configured scanner is undefined '
'in scanner requirements map : %s')
LOGGER.error(log_message, scanner.get('name'))
continue

module_name = module_path.format(
scanner_requirements_map.REQUIREMENTS_MAP
.get(scanner.get('name'))
.get('module_name'))

try:
module = importlib.import_module(module_name)
except (ImportError, TypeError, ValueError):
LOGGER.exception('Unable to import %s\n', module_name)
continue

class_name = (
scanner_requirements_map.REQUIREMENTS_MAP
.get(scanner.get('name'))
.get('class_name'))
try:
scanner_class = getattr(module, class_name)
except AttributeError:
LOGGER.exception('Unable to instantiate %s', class_name)
continue

rules_filename = (scanner_requirements_map.REQUIREMENTS_MAP
.get(scanner.get('name'))
.get('rules_filename'))
scanner = self._instantiate_scanner(
scanner_class,
rules_filename)
runnable_scanners.append(scanner)
for scanner in self.scanner_configs.get('scanners'):
if scanner.get('enabled'):
module_path = 'google.cloud.forseti.scanner.scanners.{}'

requirements_map = scanner_requirements_map.REQUIREMENTS_MAP
if not scanner.get('name') in requirements_map:
log_message = (
'Configured scanner is undefined '
'in scanner requirements map : %s')
LOGGER.error(log_message, scanner.get('name'))
continue

module_name = module_path.format(
scanner_requirements_map.REQUIREMENTS_MAP
.get(scanner.get('name'))
.get('module_name'))

try:
module = importlib.import_module(module_name)
except (ImportError, TypeError, ValueError):
LOGGER.exception('Unable to import %s\n', module_name)
continue

class_name = (
scanner_requirements_map.REQUIREMENTS_MAP
.get(scanner.get('name'))
.get('class_name'))
try:
scanner_class = getattr(module, class_name)
except AttributeError:
LOGGER.exception('Unable to instantiate %s', class_name)
continue

# Simple way to find the path to folders directory no matter
# where forseti runs.
rules_path = self.scanner_configs.get('rules_path')
if rules_path is None:
scanner_path = inspect.getfile(scanner_class)
rules_path = scanner_path.split('/google/cloud/forseti')[0]
rules_path += '/rules'

rules_filename = (scanner_requirements_map.REQUIREMENTS_MAP
.get(scanner.get('name'))
.get('rules_filename'))
rules = '{}/{}'.format(rules_path, rules_filename)
LOGGER.info('Initializing the rules engine:\nUsing rules: %s',
rules)

scanner = scanner_class(self.global_configs,
self.scanner_configs,
self.service_config,
self.model_name,
self.snapshot_timestamp,
rules)
runnable_scanners.append(scanner)

return runnable_scanners

def _instantiate_scanner(self, scanner_class, rules_filename):
"""Make individual scanners.
Args:
scanner_class (class): the individual scanner class
rules_filename (str): file name of the scanner rule file
Returns:
scanner: the individual scanner instance
"""
rules_path = self.scanner_configs.get('rules_path')
if rules_path is None:
scanner_path = inspect.getfile(scanner_class)
rules_path = scanner_path.split('/google/cloud/forseti')[0]
rules_path += '/rules'
rules = '{}/{}'.format(rules_path, rules_filename)

LOGGER.info('Initializing the rules engine:\nUsing rules: %s',
rules)

scanner = scanner_class(self.global_configs,
self.scanner_configs,
self.service_config,
self.model_name,
self.snapshot_timestamp,
rules)
return scanner
22 changes: 12 additions & 10 deletions google/cloud/forseti/scanner/scanners/external_project_access_scanner.py
View file
Expand Up @@ -13,14 +13,11 @@
# limitations under the License.

"""External project access scanner."""

# pylint: disable=line-too-long

import time

from google.auth.exceptions import RefreshError
from google.cloud.forseti.common.util import logger
from google.cloud.forseti.common.gcp_api.errors import ApiExecutionError
from google.cloud.forseti.common.gcp_api import api_helpers # noqa=E501
from google.cloud.forseti.common.gcp_type import resource_util # noqa=E501
from google.cloud.forseti.common.gcp_api.cloud_resource_manager import CloudResourceManagerClient # noqa=E501
Expand All @@ -29,6 +26,8 @@
from google.cloud.forseti.scanner.audit import external_project_access_rules_engine as epa_rules_engine # noqa=E501
from google.cloud.forseti.scanner.scanners import base_scanner

# pylint: enable=line-too-long

LOGGER = logger.get_logger(__name__)

SCOPES = ['https://www.googleapis.com/auth/cloudplatformprojects.readonly']
Expand Down Expand Up @@ -177,12 +176,15 @@ def __init__(self, global_configs, scanner_configs, service_config,
model_name,
snapshot_timestamp,
rules)

self.inventory_configs = self.service_config.get_inventory_config()
self.rules_engine = (
epa_rules_engine.ExternalProjectAccessRulesEngine(
rules_file_path=self.rules,
snapshot_timestamp=self.snapshot_timestamp))

self.rules_engine.build_rule_book(self.inventory_configs)

self._ancestries = dict()

def _output_results(self, all_violations):
Expand Down Expand Up @@ -213,6 +215,7 @@ def _find_violations(self, ancestries_by_user):
self.rules_engine.find_violations(
user_mail, project_ancestry))
all_violations.extend(violations)

return all_violations

@staticmethod
Expand Down Expand Up @@ -268,17 +271,17 @@ def _get_crm_client(self, user_email):
return client

def _retrieve(self):
# pylint: disable=line-too-long
"""Retrieve the project ancestries for all users.
Returns:
dict: User project relationship.
{"user1@example.com": [[Project("1234"), Organization("1234567")],
[Project("12345"), Folder("ABCDEFG"),
Organization("1234567")]],
"user2@example.com": [[Project("1234"), Organization("34567")],
[Project("12345"), Folder("ABCDEFG"),
Organization("1234567")]]}
[Project("12345"), Folder("ABCDEFG"), Organization("1234567")]],
"user2@example.com": [[Project("1234"), Organization("1234567")],
[Project("12345"), Folder("ABCDEFG"), Organization("1234567")]]}
"""
# pylint: enable=line-too-long
# This dictionary is the result of the scan. The key
# is the user ID. The value is a list of lists of ancestries.
user_to_project_ancestries_map = {}
Expand All @@ -298,10 +301,9 @@ def _retrieve(self):
project_ids)

user_to_project_ancestries_map[user_email] = ancestries
except (RefreshError, ApiExecutionError):
except RefreshError:
LOGGER.debug('Unable to access project ancestry %s.',
user_email)

# TODO: Remove when instrumentation is implemented.
elapsed_time = time.time() - start_time

Expand Down
13 changes: 2 additions & 11 deletions google/cloud/forseti/services/cli.py
View file
Expand Up @@ -304,18 +304,10 @@ def define_scanner_parser(parent):
title='action',
dest='action')

run_scanner_parser = action_subparser.add_parser(
action_subparser.add_parser(
'run',
help='Run the scanner')

run_scanner_parser.add_argument(
'--scanner',
choices=['external_project_access_scanner'],
help='Run a specific scanner, '
'currently only applicable for '
'the external project access scanner'
)


def define_notifier_parser(parent):
"""Define the notifier service parser.
Expand Down Expand Up @@ -679,11 +671,10 @@ def run_scanner(client, config, output, _):
"""

client = client.scanner
scanner_name = config.scanner

def do_run():
"""Run a scanner."""
for progress in client.run(scanner_name):
for progress in client.run():
output.write(progress)

actions = {
Expand Down
7 changes: 2 additions & 5 deletions google/cloud/forseti/services/client.py
View file
Expand Up @@ -125,16 +125,13 @@ def is_available(self):
return echo == data

@require_model
def run(self, scanner_name):
def run(self):
"""Runs the scanner
Args:
scanner_name (String): name of the scanner specified to run
Returns:
proto: the returned proto message.
"""
request = scanner_pb2.RunRequest(handle=scanner_name)
request = scanner_pb2.RunRequest()
return self.stub.Run(request,
metadata=self.metadata())

Expand Down
1 change: 0 additions & 1 deletion google/cloud/forseti/services/scanner/scanner.proto
View file
Expand Up @@ -24,7 +24,6 @@ service Scanner {
}

message RunRequest {
string scanner_name = 1;
}

message Progress {
Expand Down

0 comments on commit 18c56c4

Please sign in to comment.