diff --git a/configs/forseti_conf.yaml.in b/configs/forseti_conf.yaml.in index e351b9c4ff..96a11b1996 100644 --- a/configs/forseti_conf.yaml.in +++ b/configs/forseti_conf.yaml.in @@ -193,6 +193,15 @@ notifier: # gcs_path should begin with "gs://" gcs_path: gs://{SCANNER_BUCKET}/scanner_violations + - resource: firewall_rule_violations + should_notify: false + pipelines: + # Upload violations to GCS. + - name: gcs_violations_pipeline + configuration: + # gcs_path should begin with "gs://" + gcs_path: gs://{SCANNER_BUCKET}/scanner_violations + - resource: forwarding_rule_violations should_notify: true pipelines: diff --git a/google/cloud/security/common/data_access/dao.py b/google/cloud/security/common/data_access/dao.py index 04ad0a8d80..25e50f28c8 100755 --- a/google/cloud/security/common/data_access/dao.py +++ b/google/cloud/security/common/data_access/dao.py @@ -274,7 +274,7 @@ def execute_sql_with_fetch(self, resource_name, sql, values): values (tuple): Tuple of string for sql placeholder values. Returns: - list: A list of tuples representing rows of sql query result. + list: A list of dict representing rows of sql query result. Raises: MySQLError: When an error has occured while executing the query. diff --git a/google/cloud/security/common/data_access/sql_queries/create_tables.py b/google/cloud/security/common/data_access/sql_queries/create_tables.py index 4dc811c642..dc43e2697a 100755 --- a/google/cloud/security/common/data_access/sql_queries/create_tables.py +++ b/google/cloud/security/common/data_access/sql_queries/create_tables.py @@ -530,10 +530,18 @@ `resource_id` varchar(255) NOT NULL, `rule_name` varchar(255) DEFAULT NULL, `rule_index` int DEFAULT NULL, - `violation_type` enum('UNSPECIFIED','ADDED','REMOVED', - 'BIGQUERY_VIOLATION', 'BUCKET_VIOLATION', - 'IAP_VIOLATION', 'CLOUD_SQL_VIOLATION', - 'GROUP_VIOLATION', 'FORWARDING_RULE_VIOLATION', + `violation_type` enum('UNSPECIFIED', + 'ADDED','REMOVED', + 'BIGQUERY_VIOLATION', + 'BUCKET_VIOLATION', + 'CLOUD_SQL_VIOLATION', + 'FIREWALL_BLACKLIST_VIOLATION', + 'FIREWALL_MATCHES_VIOLATION', + 'FIREWALL_REQUIRED_VIOLATION', + 'FIREWALL_WHITELIST_VIOLATION', + 'FORWARDING_RULE_VIOLATION', + 'GROUP_VIOLATION', + 'IAP_VIOLATION', 'INSTANCE_NETWORK_INTERFACE_VIOLATION') NOT NULL, `violation_data` json DEFAULT NULL, PRIMARY KEY (`id`) diff --git a/google/cloud/security/common/data_access/sql_queries/select_data.py b/google/cloud/security/common/data_access/sql_queries/select_data.py index 5314d38e9f..aa199df940 100755 --- a/google/cloud/security/common/data_access/sql_queries/select_data.py +++ b/google/cloud/security/common/data_access/sql_queries/select_data.py @@ -166,44 +166,13 @@ WHERE project_number = %s; """ -# TODO: reduce these sql to a generic statement -SELECT_POLICY_VIOLATIONS = """ - SELECT * FROM violations_{0} - WHERE violation_type in ('ADDED', 'REMOVED'); -""" - -SELECT_IAP_VIOLATIONS = """ - SELECT * FROM violations_{0} - WHERE violation_type = 'IAP_VIOLATION'; -""" - -SELECT_BIGQUERY_ACL_VIOLATIONS = """ - SELECT * FROM violations_{0} - WHERE violation_type = 'BIGQUERY_VIOLATION'; -""" - -SELECT_BUCKETS_ACL_VIOLATIONS = """ - SELECT * FROM violations_{0} - WHERE violation_type = 'BUCKET_VIOLATION'; -""" -SELECT_CLOUDSQL_VIOLATIONS = """ - SELECT * FROM violations_{0} - WHERE violation_type = 'CLOUD_SQL_VIOLATION'; -""" - -SELECT_FORWARDING_RULE_VIOLATION = """ - SELECT * FROM violations_{0} - WHERE violation_type = 'FORWARDING_RULE_VIOLATION'; -""" - -SELECT_GROUPS_VIOLATIONS = """ - SELECT * FROM violations_{0} - WHERE violation_type = 'GROUP_VIOLATION'; +SELECT_ALL_VIOLATIONS = """ + SELECT * FROM violations_{0}; """ -SELECT_INSTANCE_NETWORK_INTERFACE_VIOLATIONS = """ +SELECT_VIOLATIONS_BY_TYPE = """ SELECT * FROM violations_{0} - WHERE violation_type = 'INSTANCE_NETWORK_INTERFACE_VIOLATION'; + WHERE violation_type = %s; """ BACKEND_SERVICES = """ diff --git a/google/cloud/security/common/data_access/violation_dao.py b/google/cloud/security/common/data_access/violation_dao.py index 1b26308e45..702829649c 100644 --- a/google/cloud/security/common/data_access/violation_dao.py +++ b/google/cloud/security/common/data_access/violation_dao.py @@ -14,14 +14,21 @@ """Provides the data access object (DAO) for Organizations.""" +import json + +from collections import defaultdict from collections import namedtuple + import MySQLdb from google.cloud.security.common.data_access import dao from google.cloud.security.common.data_access import errors as db_errors from google.cloud.security.common.data_access import violation_map as vm +from google.cloud.security.common.data_access.sql_queries import load_data +from google.cloud.security.common.data_access.sql_queries import select_data from google.cloud.security.common.util import log_util + LOGGER = log_util.get_logger(__name__) @@ -34,13 +41,12 @@ class ViolationDao(dao.Dao): frozen_violation_attribute_list = frozenset(violation_attribute_list) Violation = namedtuple('Violation', frozen_violation_attribute_list) - def insert_violations(self, violations, resource_name, + def insert_violations(self, violations, snapshot_timestamp=None): """Import violations into database. Args: violations (iterator): An iterator of RuleViolations. - resource_name (str): String that defines a resource. snapshot_timestamp (str): The snapshot timestamp to associate these violations with. @@ -53,6 +59,8 @@ def insert_violations(self, violations, resource_name, MySQLError: is raised when the snapshot table can not be created. """ + resource_name = 'violations' + try: # Make sure to have a reasonable timestamp to use. if not snapshot_timestamp: @@ -89,7 +97,7 @@ def insert_violations(self, violations, resource_name, try: self.execute_sql_with_commit( resource_name, - vm.VIOLATION_INSERT_MAP[resource_name](snapshot_table), + load_data.INSERT_VIOLATION.format(snapshot_table), formatted_violation) inserted_rows += 1 except MySQLdb.Error, e: @@ -99,19 +107,28 @@ def insert_violations(self, violations, resource_name, return (inserted_rows, violation_errors) - def get_all_violations(self, timestamp, resource_name): + def get_all_violations(self, timestamp, violation_type=None): """Get all the violations. Args: timestamp (str): The timestamp of the snapshot. - resource_name (str): String that defines a resource. + violation_type (str): The violation type. Returns: - tuple: A tuple of the violations as dict. + list: A list of dict of the violations data. """ - violations_sql = vm.VIOLATION_SELECT_MAP[resource_name](timestamp) + if not violation_type: + resource_name = 'all_violations' + query = select_data.SELECT_ALL_VIOLATIONS + params = () + else: + resource_name = violation_type + query = select_data.SELECT_VIOLATIONS_BY_TYPE + params = (violation_type,) + + violations_sql = query.format(timestamp) rows = self.execute_sql_with_fetch( - resource_name, violations_sql, ()) + resource_name, violations_sql, params) return rows @@ -128,3 +145,29 @@ def _format_violation(violation, resource_name): """ formatted_output = vm.VIOLATION_MAP[resource_name](violation) return formatted_output + + +def map_by_resource(violation_rows): + """Create a map of violation types to violations of that resource. + + Args: + violation_rows (list): A list of dict of violation data. + + Returns: + dict: A dict of violation types mapped to the list of corresponding + violation types, i.e. { resource => [violation_data...] }. + """ + v_by_type = defaultdict(list) + + for v_data in violation_rows: + try: + v_data['violation_data'] = json.loads(v_data['violation_data']) + except ValueError: + LOGGER.warn('Invalid violation data, unable to parse json for %s', + v_data['violation_data']) + + v_resource = vm.VIOLATION_RESOURCES.get(v_data['violation_type']) + if v_resource: + v_by_type[v_resource].append(v_data) + + return dict(v_by_type) diff --git a/google/cloud/security/common/data_access/violation_map.py b/google/cloud/security/common/data_access/violation_map.py index e5328f0b21..ee2409c9cc 100644 --- a/google/cloud/security/common/data_access/violation_map.py +++ b/google/cloud/security/common/data_access/violation_map.py @@ -14,11 +14,8 @@ """Provides violations map""" -# pylint: disable=line-too-long from google.cloud.security.common.data_access import violation_format as vf -from google.cloud.security.common.data_access.sql_queries import load_data -from google.cloud.security.common.data_access.sql_queries import select_data as sd -# pylint: enable=line-too-long + VIOLATION_MAP = { 'violations': vf.format_violation, @@ -27,24 +24,19 @@ 'groups_violations': vf.format_groups_violation, } -# TODO: Now that all violations are going into the same table, a map is not -# need anymore. -VIOLATION_INSERT_MAP = { - 'violations': load_data.INSERT_VIOLATION.format, - 'bigquery_acl_violations': load_data.INSERT_VIOLATION.format, - 'buckets_acl_violations': load_data.INSERT_VIOLATION.format, - 'cloudsql_acl_violations': load_data.INSERT_VIOLATION.format, - 'groups_violations': load_data.INSERT_VIOLATION.format -} - -VIOLATION_SELECT_MAP = { - 'bigquery_acl_violations': sd.SELECT_BIGQUERY_ACL_VIOLATIONS.format, - 'buckets_acl_violations': sd.SELECT_BUCKETS_ACL_VIOLATIONS.format, - 'cloudsql_acl_violations': sd.SELECT_CLOUDSQL_VIOLATIONS.format, - 'forwarding_rule_violations': sd.SELECT_FORWARDING_RULE_VIOLATION.format, - 'groups_violations': sd.SELECT_GROUPS_VIOLATIONS.format, - 'policy_violations': sd.SELECT_POLICY_VIOLATIONS.format, - 'iap_violations': sd.SELECT_IAP_VIOLATIONS.format, - 'instance_network_interface_violations': ( - sd.SELECT_INSTANCE_NETWORK_INTERFACE_VIOLATIONS.format), +VIOLATION_RESOURCES = { + 'ADDED': 'policy_violations', + 'REMOVED': 'policy_violations', + 'BIGQUERY_VIOLATION': 'bigquery_acl_violations', + 'BUCKET_VIOLATION': 'buckets_acl_violations', + 'CLOUD_SQL_VIOLATION': 'cloudsql_acl_violations', + 'FORWARDING_RULE_VIOLATION': 'forwarding_rule_violations', + 'FIREWALL_BLACKLIST_VIOLATION': 'firewall_rule_violations', + 'FIREWALL_MATCHES_VIOLATION': 'firewall_rule_violations', + 'FIREWALL_REQUIRED_VIOLATION': 'firewall_rule_violations', + 'FIREWALL_WHITELIST_VIOLATION': 'firewall_rule_violations', + 'GROUP_VIOLATION': 'groups_violations', + 'IAP_VIOLATION': 'iap_violations', + 'INSTANCE_NETWORK_INTERFACE_VIOLATION': ( + 'instance_network_interface_violations'), } diff --git a/google/cloud/security/notifier/notifier.py b/google/cloud/security/notifier/notifier.py index 6a27557d3b..74c9d59b7b 100644 --- a/google/cloud/security/notifier/notifier.py +++ b/google/cloud/security/notifier/notifier.py @@ -37,7 +37,6 @@ from google.cloud.security.notifier.pipelines.base_notification_pipeline import BaseNotificationPipeline from google.cloud.security.notifier.pipelines import email_inventory_snapshot_summary_pipeline as inv_summary from google.cloud.security.notifier.pipelines import email_scanner_summary_pipeline as scanner_summary -from google.cloud.security.scanner.scanners.scanners_map import SCANNER_VIOLATION_MAP # pylint: enable=line-too-long @@ -181,15 +180,14 @@ def main(_): # get violations v_dao = violation_dao.ViolationDao(global_configs) violations = {} - for mapped_scanner_violation in SCANNER_VIOLATION_MAP: - try: - violations[mapped_scanner_violation] = v_dao.get_all_violations( - timestamp, mapped_scanner_violation) - except db_errors.MySQLError, e: - # even if an error is raised we still want to continue execution - # this is because if we don't have violations the Mysql table - # is not present and an error is thrown - LOGGER.error('get_all_violations error: %s', e.message) + try: + violations = violation_dao.map_by_resource( + v_dao.get_all_violations(timestamp)) + except db_errors.MySQLError, e: + # even if an error is raised we still want to continue execution + # this is because if we don't have violations the Mysql table + # is not present and an error is thrown + LOGGER.error('get_all_violations error: %s', e.message) for retrieved_v in violations: LOGGER.info('retrieved %d violations for resource \'%s\'', @@ -199,8 +197,8 @@ def main(_): pipelines = [] for resource in notifier_configs['resources']: if violations.get(resource['resource']) is None: - LOGGER.error('The resource name \'%s\' is invalid, skipping', - resource['resource']) + LOGGER.warn('The resource name \'%s\' has no violations, ' + 'skipping', resource['resource']) continue if not violations[resource['resource']]: LOGGER.debug('No violations for: %s', resource['resource']) diff --git a/google/cloud/security/scanner/scanners/base_scanner.py b/google/cloud/security/scanner/scanners/base_scanner.py index 960d2ae725..7616ff5712 100644 --- a/google/cloud/security/scanner/scanners/base_scanner.py +++ b/google/cloud/security/scanner/scanners/base_scanner.py @@ -54,26 +54,24 @@ def run(self): """Runs the pipeline.""" pass - def _output_results_to_db(self, resource_name, violations): + def _output_results_to_db(self, violations): """Output scanner results to DB. Args: - resource_name (str): Resource name. violations (list): A list of violations. Returns: list: Violations that encountered an error during insert. """ - resource_name = 'violations' (inserted_row_count, violation_errors) = (0, []) try: vdao = violation_dao.ViolationDao(self.global_configs) (inserted_row_count, violation_errors) = vdao.insert_violations( violations, - resource_name=resource_name, snapshot_timestamp=self.snapshot_timestamp) except db_errors.MySQLError as err: - LOGGER.error('Error importing violations to database: %s', err) + LOGGER.error('Error importing violations to database: %s\n%s', + err, violations) # TODO: figure out what to do with the errors. For now, just log it. LOGGER.debug('Inserted %s rows with %s errors', diff --git a/google/cloud/security/scanner/scanners/bigquery_scanner.py b/google/cloud/security/scanner/scanners/bigquery_scanner.py index 073c8b4b42..47a9a4c96c 100644 --- a/google/cloud/security/scanner/scanners/bigquery_scanner.py +++ b/google/cloud/security/scanner/scanners/bigquery_scanner.py @@ -82,10 +82,8 @@ def _output_results(self, all_violations): Args: all_violations (list): A list of BigQuery violations. """ - resource_name = 'violations' - all_violations = self._flatten_violations(all_violations) - self._output_results_to_db(resource_name, all_violations) + self._output_results_to_db(all_violations) def _find_violations(self, bigquery_data): """Find violations in the policies. diff --git a/google/cloud/security/scanner/scanners/bucket_rules_scanner.py b/google/cloud/security/scanner/scanners/bucket_rules_scanner.py index 64f4fc92b3..fffa6028d9 100644 --- a/google/cloud/security/scanner/scanners/bucket_rules_scanner.py +++ b/google/cloud/security/scanner/scanners/bucket_rules_scanner.py @@ -80,10 +80,8 @@ def _output_results(self, all_violations): Args: all_violations (list): All violations """ - resource_name = 'violations' - all_violations = self._flatten_violations(all_violations) - self._output_results_to_db(resource_name, all_violations) + self._output_results_to_db(all_violations) def _find_violations(self, bucket_data): """Find violations in the policies. diff --git a/google/cloud/security/scanner/scanners/cloudsql_rules_scanner.py b/google/cloud/security/scanner/scanners/cloudsql_rules_scanner.py index d015047493..361631e932 100644 --- a/google/cloud/security/scanner/scanners/cloudsql_rules_scanner.py +++ b/google/cloud/security/scanner/scanners/cloudsql_rules_scanner.py @@ -80,10 +80,8 @@ def _output_results(self, all_violations): Args: all_violations (list): A list of violations. """ - resource_name = 'violations' - all_violations = self._flatten_violations(all_violations) - self._output_results_to_db(resource_name, all_violations) + self._output_results_to_db(all_violations) def _find_violations(self, cloudsql_data): """Find violations in the policies. diff --git a/google/cloud/security/scanner/scanners/forwarding_rule_scanner.py b/google/cloud/security/scanner/scanners/forwarding_rule_scanner.py index 2dfce9075c..f7f8e150c8 100644 --- a/google/cloud/security/scanner/scanners/forwarding_rule_scanner.py +++ b/google/cloud/security/scanner/scanners/forwarding_rule_scanner.py @@ -87,10 +87,8 @@ def _output_results(self, all_violations): Args: all_violations (list): All violations """ - resource_name = 'violations' - all_violations = self._flatten_violations(all_violations) - self._output_results_to_db(resource_name, all_violations) + self._output_results_to_db(all_violations) def _retrieve(self): """Runs the data collection. diff --git a/google/cloud/security/scanner/scanners/fw_rules_scanner.py b/google/cloud/security/scanner/scanners/fw_rules_scanner.py index b2a814aef9..d4eac3300e 100644 --- a/google/cloud/security/scanner/scanners/fw_rules_scanner.py +++ b/google/cloud/security/scanner/scanners/fw_rules_scanner.py @@ -96,8 +96,7 @@ def _output_results(self, all_violations, resource_counts): rule_indices = self.rules_engine.rule_book.rule_indices all_violations = list(self._flatten_violations(all_violations, rule_indices)) - violation_errors = self._output_results_to_db(resource_name, - all_violations) + violation_errors = self._output_results_to_db(all_violations) # Write the CSV for all the violations. # TODO: Move this into the base class? The IAP scanner version of this diff --git a/google/cloud/security/scanner/scanners/groups_scanner.py b/google/cloud/security/scanner/scanners/groups_scanner.py index dac8aa8e48..b21fcc19da 100644 --- a/google/cloud/security/scanner/scanners/groups_scanner.py +++ b/google/cloud/security/scanner/scanners/groups_scanner.py @@ -86,10 +86,8 @@ def _output_results(self, all_violations): Args: all_violations (list): A list of nodes that are in violation. """ - resource_name = 'violations' - all_violations = self._flatten_violations(all_violations) - self._output_results_to_db(resource_name, all_violations) + self._output_results_to_db(all_violations) # pylint: disable=too-many-branches @staticmethod diff --git a/google/cloud/security/scanner/scanners/iam_rules_scanner.py b/google/cloud/security/scanner/scanners/iam_rules_scanner.py index fd765e9e60..27fd74935e 100644 --- a/google/cloud/security/scanner/scanners/iam_rules_scanner.py +++ b/google/cloud/security/scanner/scanners/iam_rules_scanner.py @@ -94,8 +94,7 @@ def _output_results(self, all_violations, resource_counts): resource_name = 'violations' all_violations = list(self._flatten_violations(all_violations)) - violation_errors = self._output_results_to_db(resource_name, - all_violations) + violation_errors = self._output_results_to_db(all_violations) # Write the CSV for all the violations. # TODO: Move this into the base class? The IAP scanner version of this diff --git a/google/cloud/security/scanner/scanners/iap_scanner.py b/google/cloud/security/scanner/scanners/iap_scanner.py index f7a2fb441f..24def6f8fb 100644 --- a/google/cloud/security/scanner/scanners/iap_scanner.py +++ b/google/cloud/security/scanner/scanners/iap_scanner.py @@ -421,8 +421,7 @@ def _output_results(self, all_violations, resource_counts): all_violations = list(self._flatten_violations(all_violations)) LOGGER.debug('Writing violations: %r', all_violations) - violation_errors = self._output_results_to_db(resource_name, - all_violations) + violation_errors = self._output_results_to_db(all_violations) # Write the CSV for all the violations. # TODO: Move this into base class? It's cargo-culted from the IAM diff --git a/google/cloud/security/scanner/scanners/instance_network_interface_scanner.py b/google/cloud/security/scanner/scanners/instance_network_interface_scanner.py index 1a2bfadd22..4fcd5ece3c 100644 --- a/google/cloud/security/scanner/scanners/instance_network_interface_scanner.py +++ b/google/cloud/security/scanner/scanners/instance_network_interface_scanner.py @@ -82,10 +82,8 @@ def _output_results(self, all_violations): Args: all_violations (list): All violations """ - resource_name = 'violations' - all_violations = self._flatten_violations(all_violations) - self._output_results_to_db(resource_name, all_violations) + self._output_results_to_db(all_violations) # pylint: disable=invalid-name def get_instance_networks_interfaces(self): diff --git a/google/cloud/security/scanner/scanners/scanners_map.py b/google/cloud/security/scanner/scanners/scanners_map.py deleted file mode 100644 index 14add8de4e..0000000000 --- a/google/cloud/security/scanner/scanners/scanners_map.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2017 The Forseti Security Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Map for mapping violations.""" - -# TODO: After groups violations are going into the violations table, -# this map is not needed anymore. -SCANNER_VIOLATION_MAP = { - 'bigquery_acl_violations': 'violations', - 'buckets_acl_violations': 'violations', - 'cloudsql_acl_violations': 'violations', - 'forwarding_rule_violations': 'violations', - 'groups_violations': 'violations', - 'instance_network_interface_violations': 'violations', - 'policy_violations': 'violations', - 'iap_violations': 'violations', -} diff --git a/tests/common/data_access/test_data/fake_violation_dao_data.py b/tests/common/data_access/test_data/fake_violation_dao_data.py new file mode 100644 index 0000000000..14802e9a3c --- /dev/null +++ b/tests/common/data_access/test_data/fake_violation_dao_data.py @@ -0,0 +1,195 @@ +# Copyright 2017 The Forseti Security Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test data for violation dao tests.""" + +ROWS_MAP_BY_RESOURCE_1 = [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_1', + 'rule_name': 'fake rule 1', + 'rule_index': 0, + 'violation_type': 'ADDED', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_2', + 'rule_name': 'fake rule 2', + 'rule_index': 0, + 'violation_type': 'REMOVED', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_3', + 'rule_name': 'fake rule 3', + 'rule_index': 0, + 'violation_type': 'BIGQUERY_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_4', + 'rule_name': 'fake rule 4', + 'rule_index': 0, + 'violation_type': 'BUCKET_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_5', + 'rule_name': 'fake rule 5', + 'rule_index': 0, + 'violation_type': 'CLOUD_SQL_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_6', + 'rule_name': 'fake rule 6', + 'rule_index': 0, + 'violation_type': 'FORWARDING_RULE_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_7', + 'rule_name': 'fake rule 7', + 'rule_index': 0, + 'violation_type': 'FIREWALL_BLACKLIST_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_8', + 'rule_name': 'fake rule 8', + 'rule_index': 0, + 'violation_type': 'FIREWALL_MATCHES_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_9', + 'rule_name': 'fake rule 9', + 'rule_index': 0, + 'violation_type': 'FIREWALL_REQUIRED_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_10', + 'rule_name': 'fake rule 10', + 'rule_index': 0, + 'violation_type': 'FIREWALL_WHITELIST_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_11', + 'rule_name': 'fake rule 11', + 'rule_index': 0, + 'violation_type': 'GROUP_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_12', + 'rule_name': 'fake rule 12', + 'rule_index': 0, + 'violation_type': 'IAP_VIOLATION', + 'violation_data': '{}'}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_13', + 'rule_name': 'fake rule 13', + 'rule_index': 0, + 'violation_type': 'INSTANCE_NETWORK_INTERFACE_VIOLATION', + 'violation_data': '{}'}, +] + +EXPECTED_MAP_BY_RESOURCE_1 = { + 'policy_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_1', + 'rule_name': 'fake rule 1', + 'rule_index': 0, + 'violation_type': 'ADDED', + 'violation_data': {}}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_2', + 'rule_name': 'fake rule 2', + 'rule_index': 0, + 'violation_type': 'REMOVED', + 'violation_data': {}}, + ], + 'bigquery_acl_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_3', + 'rule_name': 'fake rule 3', + 'rule_index': 0, + 'violation_type': 'BIGQUERY_VIOLATION', + 'violation_data': {}}, + ], + 'buckets_acl_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_4', + 'rule_name': 'fake rule 4', + 'rule_index': 0, + 'violation_type': 'BUCKET_VIOLATION', + 'violation_data': {}}, + ], + 'cloudsql_acl_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_5', + 'rule_name': 'fake rule 5', + 'rule_index': 0, + 'violation_type': 'CLOUD_SQL_VIOLATION', + 'violation_data': {}}, + ], + 'forwarding_rule_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_6', + 'rule_name': 'fake rule 6', + 'rule_index': 0, + 'violation_type': 'FORWARDING_RULE_VIOLATION', + 'violation_data': {}}, + ], + 'firewall_rule_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_7', + 'rule_name': 'fake rule 7', + 'rule_index': 0, + 'violation_type': 'FIREWALL_BLACKLIST_VIOLATION', + 'violation_data': {}}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_8', + 'rule_name': 'fake rule 8', + 'rule_index': 0, + 'violation_type': 'FIREWALL_MATCHES_VIOLATION', + 'violation_data': {}}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_9', + 'rule_name': 'fake rule 9', + 'rule_index': 0, + 'violation_type': 'FIREWALL_REQUIRED_VIOLATION', + 'violation_data': {}}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_10', + 'rule_name': 'fake rule 10', + 'rule_index': 0, + 'violation_type': 'FIREWALL_WHITELIST_VIOLATION', + 'violation_data': {}}, + ], + 'groups_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_11', + 'rule_name': 'fake rule 11', + 'rule_index': 0, + 'violation_type': 'GROUP_VIOLATION', + 'violation_data': {}}, + ], + 'iap_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_12', + 'rule_name': 'fake rule 12', + 'rule_index': 0, + 'violation_type': 'IAP_VIOLATION', + 'violation_data': {}}, + ], + 'instance_network_interface_violations': [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_13', + 'rule_name': 'fake rule 13', + 'rule_index': 0, + 'violation_type': 'INSTANCE_NETWORK_INTERFACE_VIOLATION', + 'violation_data': {}}, + ], +} diff --git a/tests/common/data_access/violation_dao_test.py b/tests/common/data_access/violation_dao_test.py index 0ad54d07da..017255b62c 100644 --- a/tests/common/data_access/violation_dao_test.py +++ b/tests/common/data_access/violation_dao_test.py @@ -22,8 +22,10 @@ from google.cloud.security.common.data_access import _db_connector from google.cloud.security.common.data_access import errors from google.cloud.security.common.data_access import violation_dao +from google.cloud.security.common.data_access.sql_queries import select_data from google.cloud.security.common.gcp_type import iam_policy as iam from google.cloud.security.scanner.audit import rules +from tests.common.data_access.test_data import fake_violation_dao_data as fake_data class ViolationDaoTest(ForsetiTestCase): @@ -134,8 +136,7 @@ def test_insert_violations_no_timestamp(self): return_value=self.fake_table_name) self.dao.conn = conn_mock self.dao.execute_sql_with_commit = commit_mock - self.dao.insert_violations(self.fake_flattened_violations, - self.resource_name) + self.dao.insert_violations(self.fake_flattened_violations) # Assert snapshot is retrieved because no snapshot timestamp was # provided to the method call. @@ -169,7 +170,6 @@ def test_insert_violations_with_timestamp(self): self.dao.get_latest_snapshot_timestamp = mock.MagicMock() self.dao.insert_violations( self.fake_flattened_violations, - self.resource_name, fake_custom_timestamp) self.dao.get_latest_snapshot_timestamp.assert_not_called() @@ -231,6 +231,68 @@ def insert_violation_side_effect(*args, **kwargs): self.assertEqual(expected, actual) self.assertEquals(1, violation_dao.LOGGER.error.call_count) + def test_get_all_violations_no_type(self): + """Test get_all_violations() with no type.""" + expected = [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_1', + 'rule_name': 'fake rule name', + 'rule_index': 0, + 'violation_type': 'type1', + 'violation_data': {}}, + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_2', + 'rule_name': 'fake rule name', + 'rule_index': 0, + 'violation_type': 'type2', + 'violation_data': {}}, + ] + self.dao.conn = mock.MagicMock() + self.dao.get_latest_snapshot_timestamp = mock.MagicMock() + self.dao.execute_sql_with_fetch = mock.MagicMock( + return_value=expected) + violations = self.dao.get_all_violations(self.fake_snapshot_timestamp) + + self.dao.execute_sql_with_fetch.assert_called_once_with( + 'all_violations', + select_data.SELECT_ALL_VIOLATIONS.format( + self.fake_snapshot_timestamp), + ()) + self.assertEqual(expected, violations) + + def test_get_all_violations_by_type(self): + """Test get_all_violations() with no type.""" + expected = [ + {'resource_type': 'fake_type', + 'resource_id': 'fake_id_1', + 'rule_name': 'fake rule name', + 'rule_index': 0, + 'violation_type': 'type1', + 'violation_data': {}}, + ] + violation_type = 'type1' + self.dao.conn = mock.MagicMock() + self.dao.get_latest_snapshot_timestamp = mock.MagicMock() + self.dao.execute_sql_with_fetch = mock.MagicMock( + return_value=expected) + violations = self.dao.get_all_violations( + self.fake_snapshot_timestamp, violation_type) + + self.dao.execute_sql_with_fetch.assert_called_once_with( + violation_type, + select_data.SELECT_VIOLATIONS_BY_TYPE.format( + self.fake_snapshot_timestamp), + (violation_type,)) + self.assertEqual(expected, violations) + + def test_map_by_type(self): + """Test violation_dao.map_by_resource() util method.""" + actual = violation_dao.map_by_resource( + fake_data.ROWS_MAP_BY_RESOURCE_1) + + self.assertEqual( + fake_data.EXPECTED_MAP_BY_RESOURCE_1, actual) + if __name__ == '__main__': unittest.main() diff --git a/tests/scanner/scanners/fw_rules_scanner_test.py b/tests/scanner/scanners/fw_rules_scanner_test.py index 884293c91a..af5cc9e175 100644 --- a/tests/scanner/scanners/fw_rules_scanner_test.py +++ b/tests/scanner/scanners/fw_rules_scanner_test.py @@ -179,7 +179,7 @@ def test_output_results_local_no_email( self.scanner._output_results(violations, '88888') mock_output_results_to_db.assert_called_once_with( - self.scanner, 'violations', flattened_violations) + self.scanner, flattened_violations) mock_write_csv.assert_called_once_with( resource_name='violations', data=flattened_violations, @@ -265,7 +265,7 @@ def test_output_results_gcs_email( } for i, v in enumerate(violations)] mock_output_results_to_db.assert_called_once_with( - self.scanner, 'violations', flattened_violations) + self.scanner, flattened_violations) mock_write_csv.assert_called_once_with( resource_name='violations', data=flattened_violations, @@ -293,7 +293,7 @@ def test_output_results_gcs_email( 'all_violations': flattened_violations, 'resource_counts': '88888', 'violation_errors': mock_output_results_to_db( - self, 'violations', flattened_violations), + self, flattened_violations), } } mock_notifier.process.assert_called_once_with(expected_message) @@ -525,7 +525,7 @@ def test_run_no_email(self, mock_output_results_to_db): ] [sorted(v) for v in expected_violations] mock_output_results_to_db.assert_called_once_with( - scanner, 'violations', expected_violations) + scanner, expected_violations) def assert_rule_violation_lists_equal(self, expected, violations): sorted(expected, key=lambda k: k.resource_id)