Update Notifier to support CV violations with violation type = CV_{co…

…nstraint_name}. Update CV e2e test to assert the GCS violations file exists after a scan. (#3634)

Co-authored-by: red2k18 <rdevani@google.com>

google/cloud/forseti/common/data_access/violation_map.py

@@ -16,6 +16,7 @@
 
 from google.cloud.forseti.common.data_access import violation_format as vf
 
+CV_VIOLATION_TYPE = 'config_validator_violations'
 
 VIOLATION_MAP = {
     'violations': vf.format_violation,
@@ -30,7 +31,7 @@
     'BLACKLIST_VIOLATION': 'blacklist_violations',
     'BUCKET_VIOLATION': 'buckets_acl_violations',
     'CLOUD_SQL_VIOLATION': 'cloudsql_acl_violations',
-    'CONFIG_VALIDATOR_VIOLATION': 'config_validator_violations',
+    'CONFIG_VALIDATOR_VIOLATION': CV_VIOLATION_TYPE,
     'ENABLED_APIS_VIOLATION': 'enabled_apis_violations',
     'FORWARDING_RULE_VIOLATION': 'forwarding_rule_violations',
     'FIREWALL_BLACKLIST_VIOLATION': 'firewall_rule_violations',

google/cloud/forseti/services/scanner/dao.py

@@ -18,6 +18,7 @@
 from collections import defaultdict
 import hashlib
 import json
+import re
 
 from sqlalchemy import BigInteger
 from sqlalchemy import Column
@@ -38,6 +39,7 @@
 BASE = declarative_base()
 CURRENT_SCHEMA = 1
 SUCCESS_STATES = [IndexState.SUCCESS, IndexState.PARTIAL_SUCCESS]
+CV_VIOLATION_PATTERN = re.compile('^cv', re.I)
 
 
 class ScannerIndex(BASE):
@@ -350,9 +352,13 @@ def map_by_resource(violation_rows):
             v_data['resource_data'] = json.loads(
                 json.dumps(v_data['resource_data']))
 
-        v_resource = vm.VIOLATION_RESOURCES.get(v_data['violation_type'])
-        if v_resource:
-            v_by_type[v_resource].append(v_data)
+        violation_type = vm.VIOLATION_RESOURCES.get(v_data['violation_type'])
+        if not violation_type:
+            if bool(CV_VIOLATION_PATTERN.match(v_data['violation_type'])):
+                violation_type = vm.CV_VIOLATION_TYPE
+
+        if violation_type:
+            v_by_type[violation_type].append(v_data)
 
     return dict(v_by_type)
 

integration_tests/tests/forseti/controls/scanner-config_validator_gcs_policy.rb

@@ -54,20 +54,33 @@
   # Act
   describe command("forseti scanner run") do
     its('exit_status') { should eq 0 }
-    its('stdout') { should match (/Running ConfigValidatorScanner.../) }
-    its('stdout') { should match (/Scan completed/) }
-    its('stdout') { should match (/Scanner Index ID: (.*[0-9]*) is created/) }
+    its('stdout') { should match(/Running ConfigValidatorScanner.../) }
+    its('stdout') { should match(/Scan completed/) }
+    its('stdout') { should match(/Scanner Index ID: (.*[0-9]*) is created/) }
+  end
+
+  # Test notifier for the GCS export
+  describe command("forseti notifier run --inventory_index_id #{@inventory_id}") do
+    its('exit_status') { should eq 0 }
+    its('stdout') { should match(/Notification completed!/) }
+    its('stdout') { should match(/Retrieved ([0-9]*) violations for resource 'config_validator_violations'/) }
   end
 
   # Assert violations exist for Cloud SQL Location policy
   describe command("mysql -u #{db_user_name} -p#{db_password} --host 127.0.0.1 --database forseti_security --execute \"SELECT COUNT(*) FROM violations V JOIN forseti_security.scanner_index SI ON SI.id = V.scanner_index_id WHERE SI.inventory_index_id = #{@inventory_id} AND V.resource_id = '#{forseti_cloudsql_instance_name}' AND V.violation_type = CONCAT('CV_', V.rule_name);\"") do
     its('exit_status') { should eq 0 }
-    its('stdout') { should match (/1/) }
+    its('stdout') { should match(/1/) }
   end
 
   # Assert violations exist for Compute Zone policy
   describe command("mysql -u #{db_user_name} -p#{db_password} --host 127.0.0.1 --database forseti_security --execute \"SELECT COUNT(*) FROM violations V JOIN forseti_security.scanner_index SI ON SI.id = V.scanner_index_id WHERE SI.inventory_index_id = #{@inventory_id} AND V.resource_id = '#{forseti_server_vm_name}' AND V.resource_type = 'compute.googleapis.com/Instance' AND V.violation_type = CONCAT('CV_', V.rule_name);\"") do
     its('exit_status') { should eq 0 }
-    its('stdout') { should match (/1/) }
+    its('stdout') { should match(/1/) }
+  end
+
+  # Assert GCS violations file exists
+  gcs_file_path = "gs://#{forseti_server_storage_bucket}/scanner_violations/violations.config_validator_violations.#{@inventory_id}.*"
+  describe command("sudo gsutil ls #{gcs_file_path}") do
+    its('exit_status') { should eq 0 }
   end
 end

tests/services/scanner/scanner_dao_test.py

@@ -25,6 +25,7 @@
 from sqlalchemy.orm import sessionmaker
 
 from tests.services.scanner import scanner_base_db
+from tests.services.scanner.test_data import config_validator_violations
 from tests.services.util.db import create_test_engine_with_file
 from tests.unittest_utils import ForsetiTestCase
 from google.cloud.forseti.common.util import date_time
@@ -278,6 +279,12 @@ def test_get_latest_scanner_index_id_with_failure_state(self,
             scanner_dao.get_latest_scanner_index_id(
                 self.session, expected_id, IndexState.FAILURE))
 
+    @staticmethod
+    def test_map_by_resource_returns_cv_violations():
+        resource_map = scanner_dao.map_by_resource(
+            config_validator_violations.CONFIG_VALIDATOR_VIOLATIONS)
+        assert len(resource_map['config_validator_violations']) == 2
+
 
 class ScannerIndexTest(ForsetiTestCase):
     """Test scanner data access."""

tests/services/scanner/test_data/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Scanner test data."""

tests/services/scanner/test_data/config_validator_violations.py

@@ -0,0 +1,50 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test violation data."""
+
+CONFIG_VALIDATOR_VIOLATIONS = [
+    {
+        'id': 1,
+        'created_at_datetime': '2020-02-05T09:29:52Z',
+        'full_name': 'organization/123/project/test-project/bucket/test-bucket/',
+        'resource_data': '{}',
+        'resource_name': '//storage.googleapis.com/test-bucket',
+        'resource_id': 'test-bucket',
+        'resource_type': 'storage.googleapis.com/Bucket',
+        'rule_index': 0,
+        'rule_name': 'always_violates_all',
+        'scanner_index_id': 1,
+        'violation_data': '{"asset": {"ancestry_path": "organization/123/project/test-project/", "asset_type": "storage.googleapis.com/Bucket", "name": "//storage.googleapis.com/test-bucket", "resource": {"data": {}, "version": "v1"}}, "constraint": {"kind": "GCPAlwaysViolatesConstraintV1", "name": "always_violates_all"}}',
+        'violation_hash': '15fda93a6fdd32d867064677cf07686f79b',
+        'violation_message': 'always_violates_all violates on all resources.',
+        'violation_type': 'CV_always_violates_all'
+    },
+    {
+        'id': 2,
+        'created_at_datetime': '2020-02-05T09:29:52Z',
+        'full_name': 'organization/123/project/test-project/cloudsqlinstance/456/',
+        'resource_data': '{}',
+        'resource_name': '//cloudsql.googleapis.com/projects/test-project/instances/test-db',
+        'resource_id': 'test-db',
+        'resource_type': 'sqladmin.googleapis.com/Instance',
+        'rule_index': 0,
+        'rule_name': 'allow_some_sql_location',
+        'scanner_index_id': 1,
+        'violation_data': '{"location": "us-central1", "resource": "//cloudsql.googleapis.com/projects/test-project/instances/test-db"}',
+        'violation_hash': '7745b8e9c82ebd6297d6d733067dc23907c481feb674cd730d50e01a9c9d795dda8cba899d35620b03a396e3f3bd0796a90744812fd5b6586044bd0a8050a45e',
+        'violation_message': '//cloudsql.googleapis.com/projects/test-project/instances/test-db is in a disallowed location (us-central1).',
+        'violation_type': 'CV_allow_some_sql_location'
+    }
+]