Update tests for Google Provider 3.7 + reliability/improvements (#3682)

* Update Forseti test fixtures to pin Terraform provider versions to compatible versions with the Forseti TF repo. Fix explain and external project access scanner tests which recently started failing because the model was not set prior to the command execution. Fix ACL setup for the test resources, which was causing an error during Terraform destroy.

* Update build timeout to 3 hours. Minor updates for cv test, passing locally...

* Logs indicate the notifier process is running 20+ minutes after the inventory was created, and that the inventory is being purged before notifier is run. Testing to see if running the notifier within the same command as the scanner run helps.

* Update Forseti test fixtures to pin Terraform provider versions to compatible versions with the Forseti TF repo. Fix explain and external project access scanner tests which recently started failing because the model was not set prior to the command execution. Fix ACL setup for the test resources, which was causing an error during Terraform destroy.

* Update build timeout to 3 hours. Minor updates for cv test, passing locally...

* Logs indicate the notifier process is running 20+ minutes after the inventory was created, and that the inventory is being purged before notifier is run. Testing to see if running the notifier within the same command as the scanner run helps.

* Updated CV test to use specific commit from Policy Library, as recent changes break with thje older version of CV in the Forseti repo. Temporarily disabling client tests to get faster result from this test.

* Working on migrating CV test to pytest

* Separated tests for CloudSQL and Compute Zone policies

* Conftest updates

* Re-enable other tests, delete old CV test

* Update perf test to point to inventory dir. Update asserts for the pytest commands.

* Address PR comments, fix comments and merge master in

* Add session scope to fixtures

.kitchen.yml

@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -68,7 +68,7 @@ suites:
             - notifier-inventory-summary-email
             - notifier-temp-file-deletion
             - scanner-bucket-acl-scanner
-            - scanner-config-validator-gcs-policy
+            - scanner-config-validator
             - scanner-enabled-apis-scanner
             - scanner-enable-audit-logging-scanner
             - scanner-firewall-scanner

build/e2e.cloudbuild.yaml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-timeout: 7200s
+timeout: 10800s
 steps:
 - id: run_e2e_tests
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'

endtoend_tests/conftest.py

@@ -27,6 +27,8 @@ def pytest_addoption(parser):
     parser.addoption('--cai_dump_file_gcs_paths',
                      default='',
                      help='CAI dump file paths for inventory performance test')
+    parser.addoption('--cloudsql_instance_name',
+                     help='Cloud SQL instance name')
     parser.addoption('--cloudsql_password', help='Cloud SQL password')
     parser.addoption('--cloudsql_port',
                      default=CLOUDSQL_PORT,
@@ -37,6 +39,7 @@ def pytest_addoption(parser):
     parser.addoption('--forseti_server_config_path',
                      default=FORSETI_SERVER_CONFIG_PATH,
                      help='Path to Forseti server config')
+    parser.addoption('--forseti_server_vm_name', help='Forseti server VM name')
     parser.addoption('--root_resource_id',
                      help='Root resource id for inventory performance test')
 
@@ -46,7 +49,10 @@ def pytest_configure(config):
         'markers', 'e2e: mark test to run only on named environment'
     )
     config.addinivalue_line(
-        'markers', 'inventory: mark test to run only on named environment'
+        'markers', 'inventory: mark to run all inventory tests'
+    )
+    config.addinivalue_line(
+        'markers', 'scanner: mark to run all scanner tests'
     )
 
 
@@ -58,41 +64,51 @@ def cai_dump_file_gcs_paths(request):
     return path.split(',')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def cloudsql_connection(cloudsql_password, cloudsql_port, cloudsql_username):
     yield create_engine(f'mysql+pymysql://{cloudsql_username}:{cloudsql_password}@127.0.0.1:{cloudsql_port}')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
+def cloudsql_instance_name(request):
+    return request.config.getoption('--cloudsql_instance_name')
+
+
+@pytest.fixture(scope="session")
 def cloudsql_password(request):
     return request.config.getoption('--cloudsql_password')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def cloudsql_port(request):
     return request.config.getoption('--cloudsql_port')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def cloudsql_username(request):
     return request.config.getoption('--cloudsql_username')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def forseti_server_bucket_name(request):
     return request.config.getoption('--forseti_server_bucket_name')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def forseti_server_config_path(request):
     return request.config.getoption('--forseti_server_config_path')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
+def forseti_server_vm_name(request):
+    return request.config.getoption('--forseti_server_vm_name')
+
+
+@pytest.fixture(scope="session")
 def root_resource_id(request):
     return request.config.getoption('--root_resource_id')
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def test_id():
     return str(int(time.time()))

endtoend_tests/forseti/conftest.py

@@ -0,0 +1,46 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Forseti end-to-end test configuration"""
+
+import pytest
+import time
+from endtoend_tests.helpers.forseti_cli import ForsetiCli
+
+
+@pytest.fixture(scope="session")
+def forseti_cli():
+    return ForsetiCli()
+
+
+@pytest.fixture(scope="session")
+def forseti_inventory_readonly(forseti_cli):
+    inventory_id, result = forseti_cli.inventory_create()
+    yield inventory_id, result
+    forseti_cli.inventory_delete(inventory_id)
+
+
+@pytest.fixture(scope="session")
+def forseti_model_readonly(forseti_cli, forseti_inventory_readonly):
+    model_name = f'Test{str(int(time.time()))}'
+    result = forseti_cli.model_create(forseti_inventory_readonly[0],
+                                      model_name)
+    yield model_name, result
+    forseti_cli.model_delete(model_name)
+
+
+@pytest.fixture(scope="session")
+def forseti_scan_readonly(forseti_cli, forseti_model_readonly):
+    forseti_cli.model_use(forseti_model_readonly[0])
+    yield forseti_cli.scanner_run()

endtoend_tests/forseti/server/conftest.py

@@ -18,9 +18,8 @@
 from endtoend_tests.helpers.server_config import ServerConfig
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def server_config_helper(forseti_server_bucket_name, forseti_server_config_path):
     server_config = ServerConfig(forseti_server_config_path)
     yield server_config
     server_config.copy_from_gcs(forseti_server_bucket_name)
-

endtoend_tests/forseti/server/scanner/cv_cloudsql_location_test.py

@@ -0,0 +1,59 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Config Validator Cloud SQL Location test"""
+
+import pytest
+from sqlalchemy.sql import text
+
+
+class TestConfigValidatorCloudSqlLocation:
+    """Config Validator Cloud SQL Location test
+
+    Run a scan and assert the Cloud SQL Location policy produces a violation.
+    """
+
+    @pytest.mark.e2e
+    @pytest.mark.scanner
+    def test_cv_cloudsql_location(self,
+                                  cloudsql_connection,
+                                  cloudsql_instance_name,
+                                  forseti_scan_readonly):
+        """Config Validator Cloud SQL Location test
+
+        Args:
+            cloudsql_connection (object): SQLAlchemy connection for Forseti
+            cloudsql_instance_name (str): Cloud SQL instance name
+            forseti_scan_readonly (Tuple): Scanner id & scanner process result
+        """
+        # Arrange/Act
+        scanner_id, scanner_result = forseti_scan_readonly
+
+        # Assert violation found
+        violation_type = 'CV_sql_location_denylist'
+        query = text('SELECT '
+                     'COUNT(*) '
+                     'FROM forseti_security.violations '
+                     'WHERE '
+                     'scanner_index_id = :scanner_id '
+                     'AND resource_id = :cloudsql_instance_name '
+                     'AND violation_type = :violation_type')
+        violation_count = (
+                cloudsql_connection.execute(
+                    query,
+                    cloudsql_instance_name=cloudsql_instance_name,
+                    scanner_id=scanner_id,
+                    violation_type=violation_type)
+                .fetchone())
+        assert 1 == violation_count[0]

endtoend_tests/forseti/server/scanner/cv_compute_zone_test.py

@@ -0,0 +1,60 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Config Validator Cloud Zone test"""
+
+import pytest
+from sqlalchemy.sql import text
+
+
+class TestConfigValidatorComputeZone:
+    """Config Validator Cloud Zone test
+
+    Run a scan and assert the Cloud Zone policy produces a violation.
+    """
+
+    @pytest.mark.e2e
+    @pytest.mark.scanner
+    def test_cv_compute_zone(self,
+                             cloudsql_connection,
+                             forseti_scan_readonly,
+                             forseti_server_vm_name):
+        """Config Validator Cloud Zone test
+
+        Args:
+            cloudsql_connection (object): SQLAlchemy  connection for Forseti
+            forseti_scan_readonly (Tuple): Scanner id & scanner process result
+            forseti_server_vm_name (str): Forseti server VM name
+        """
+        # Arrange/Act
+        scanner_id, scanner_result = forseti_scan_readonly
+
+        # Assert violation found
+        violation_type = 'CV_compute_zone_denylist'
+        query = text('SELECT '
+                     'COUNT(*) '
+                     'FROM forseti_security.violations '
+                     'WHERE '
+                     'scanner_index_id = :scanner_id '
+                     'AND resource_id = :forseti_server_vm_name '
+                     'AND resource_type = \'compute.googleapis.com/Instance\' '
+                     'AND violation_type = :violation_type')
+        violation_count = (
+                cloudsql_connection.execute(
+                    query,
+                    forseti_server_vm_name=forseti_server_vm_name,
+                    scanner_id=scanner_id,
+                    violation_type=violation_type)
+                .fetchone())
+        assert 1 == violation_count[0]

endtoend_tests/forseti/server/scanner/cv_scan_test.py

@@ -0,0 +1,49 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Config Validator Scan test"""
+
+import pytest
+import re
+
+
+class TestConfigValidatorScan:
+    """Config Validator Scan test
+
+    Run a scan and assert CV scanner is run and there are no errors.
+    """
+
+    @pytest.mark.e2e
+    @pytest.mark.scanner
+    def test_cv_scan(self, forseti_scan_readonly):
+        """Config Validator Scan test
+
+        Args:
+            forseti_scan_readonly (Tuple): Scanner id & scanner process result
+        """
+        # Arrange/Act
+        scanner_id, scanner_result = forseti_scan_readonly
+
+        # Assert scanner id
+        assert scanner_id
+
+        # Assert CV scanner was run
+        regex = re.compile('Running ConfigValidatorScanner')
+        match = regex.search(str(scanner_result.stdout))
+        assert match
+
+        # Assert scan completed
+        regex = re.compile('Scan completed')
+        match = regex.search(str(scanner_result.stdout))
+        assert match