Changes to fix model creation for release 2.25.2 (#3781)

* Cherry pick model/e2e fixes from commit cf6e9d5. Update TF version used by e2e tests to avoid issue with TF versions in Forseti module. Fixcrawlertest (#3743). Fix e2e tests. Update forseticli.

* Update Forseti version.

.kitchen.yml

@@ -37,6 +37,7 @@ suites:
         - name: forseti-client
           backend: ssh
           controls:
+            - client-pytest
             - explain
             - inventory-cai-gcs-export
             - model-list
@@ -61,18 +62,18 @@ suites:
             - inventory-delete
             - inventory-get-list
             - inventory-performance
-            - inventory-purge
+#            - inventory-purge # TODO: Replace with pytest
             - logs-debug-level
             - model-create-get
             - model-delete
             - notifier-inventory-summary-email
             - notifier-temp-file-deletion
             - scanner-bucket-acl-scanner
-            - scanner-config-validator
             - scanner-enabled-apis-scanner
             - scanner-enable-audit-logging-scanner
             - scanner-firewall-scanner
             - scanner-run
+            - server-pytest
           hosts_output: forseti-server-vm-ip
           user: ubuntu
           key_files:

build/e2e.cloudbuild.yaml

@@ -37,4 +37,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.7.5'

endtoend_tests/conftest.py

@@ -34,26 +34,44 @@ def pytest_addoption(parser):
                      default=CLOUDSQL_PORT,
                      help='Cloud SQL port')
     parser.addoption('--cloudsql_username', help='Cloud SQL username')
+    parser.addoption('--forseti_client_service_account',
+                     help='Forseti client service account email')
     parser.addoption('--forseti_server_bucket_name',
                      help='Forseti server bucket name')
     parser.addoption('--forseti_server_config_path',
                      default=FORSETI_SERVER_CONFIG_PATH,
                      help='Path to Forseti server config')
+    parser.addoption('--forseti_server_service_account',
+                     help='Forseti server service account email')
     parser.addoption('--forseti_server_vm_name', help='Forseti server VM name')
+    parser.addoption('--organization_id', help='Org id being scanned')
+    parser.addoption('--project_id', help='Project id being scanned')
     parser.addoption('--root_resource_id',
                      help='Root resource id for inventory performance test')
 
 
 def pytest_configure(config):
+    config.addinivalue_line(
+        'markers', 'client: mark to run all client tests'
+    )
     config.addinivalue_line(
         'markers', 'e2e: mark test to run only on named environment'
     )
+    config.addinivalue_line(
+        'markers', 'explainer: mark to run all explainer tests'
+    )
     config.addinivalue_line(
         'markers', 'inventory: mark to run all inventory tests'
     )
+    config.addinivalue_line(
+        'markers', 'model: mark to run all model tests'
+    )
     config.addinivalue_line(
         'markers', 'scanner: mark to run all scanner tests'
     )
+    config.addinivalue_line(
+        'markers', 'server: mark to run all server tests'
+    )
 
 
 @pytest.fixture
@@ -66,7 +84,8 @@ def cai_dump_file_gcs_paths(request):
 
 @pytest.fixture(scope="session")
 def cloudsql_connection(cloudsql_password, cloudsql_port, cloudsql_username):
-    yield create_engine(f'mysql+pymysql://{cloudsql_username}:{cloudsql_password}@127.0.0.1:{cloudsql_port}')
+    yield create_engine(
+            f'mysql+pymysql://{cloudsql_username}:{cloudsql_password}@127.0.0.1:{cloudsql_port}')
 
 
 @pytest.fixture(scope="session")
@@ -89,6 +108,11 @@ def cloudsql_username(request):
     return request.config.getoption('--cloudsql_username')
 
 
+@pytest.fixture(scope="session")
+def forseti_client_service_account(request):
+    return request.config.getoption('--forseti_client_service_account')
+
+
 @pytest.fixture(scope="session")
 def forseti_server_bucket_name(request):
     return request.config.getoption('--forseti_server_bucket_name')
@@ -99,11 +123,26 @@ def forseti_server_config_path(request):
     return request.config.getoption('--forseti_server_config_path')
 
 
+@pytest.fixture(scope="session")
+def forseti_server_service_account(request):
+    return request.config.getoption('--forseti_server_service_account')
+
+
 @pytest.fixture(scope="session")
 def forseti_server_vm_name(request):
     return request.config.getoption('--forseti_server_vm_name')
 
 
+@pytest.fixture(scope="session")
+def organization_id(request):
+    return request.config.getoption('--organization_id')
+
+
+@pytest.fixture(scope="session")
+def project_id(request):
+    return request.config.getoption('--project_id')
+
+
 @pytest.fixture(scope="session")
 def root_resource_id(request):
     return request.config.getoption('--root_resource_id')

endtoend_tests/forseti/conftest.py

@@ -17,6 +17,7 @@
 import pytest
 import time
 from endtoend_tests.helpers.forseti_cli import ForsetiCli
+from endtoend_tests.helpers.server_config import ServerConfig
 
 
 @pytest.fixture(scope="session")
@@ -34,13 +35,20 @@ def forseti_inventory_readonly(forseti_cli):
 @pytest.fixture(scope="session")
 def forseti_model_readonly(forseti_cli, forseti_inventory_readonly):
     model_name = f'Test{str(int(time.time()))}'
-    result = forseti_cli.model_create(forseti_inventory_readonly[0],
-                                      model_name)
-    yield model_name, result
+    handle, result = forseti_cli.model_create(forseti_inventory_readonly[0],
+                                              model_name)
+    yield model_name, handle, result
     forseti_cli.model_delete(model_name)
 
 
 @pytest.fixture(scope="session")
 def forseti_scan_readonly(forseti_cli, forseti_model_readonly):
     forseti_cli.model_use(forseti_model_readonly[0])
     yield forseti_cli.scanner_run()
+
+
+@pytest.fixture(scope="session")
+def server_config_helper(forseti_server_bucket_name, forseti_server_config_path):
+    server_config = ServerConfig(forseti_server_config_path)
+    yield server_config
+    server_config.copy_from_gcs(forseti_server_bucket_name)

endtoend_tests/forseti/explain/access_by_resource_test.py

@@ -0,0 +1,78 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import re
+from endtoend_tests.helpers.forseti_cli import ForsetiCli
+
+
+class TestExplainerAccessByResource:
+    """Explainer access_by_resource tests.
+
+    TODO: Tests take a long time for explain commands, use different resources?
+    """
+
+    @pytest.mark.client
+    @pytest.mark.e2e
+    @pytest.mark.explainer
+    def test_access_by_resource_for_organization(self, forseti_cli: ForsetiCli,
+                                                 forseti_model_readonly,
+                                                 forseti_server_service_account,
+                                                 organization_id):
+        """Test access_by_resource for organization includes Forseti SA.
+
+        Args:
+            forseti_cli (ForsetiCli): Instance of the forseti cli helper
+            forseti_model_readonly (Tuple): Model name & process result
+            forseti_server_service_account (str): Server service account email
+            organization_id (str): Organization id being scanned
+        """
+        # Arrange
+        model_name, _, _ = forseti_model_readonly
+        forseti_cli.model_use(model_name=model_name)
+
+        # Act
+        result = forseti_cli.explainer_access_by_resource(
+            f'organization/{organization_id}')
+
+        # Assert
+        assert result.returncode == 0, f'Forseti stdout: {str(result.stdout)}'
+        assert 10 == len(re.findall(forseti_server_service_account, str(result.stdout)))
+
+    @pytest.mark.client
+    @pytest.mark.e2e
+    @pytest.mark.explainer
+    def test_access_by_resource_for_project(self, forseti_cli: ForsetiCli,
+                                            forseti_model_readonly,
+                                            forseti_server_service_account,
+                                            project_id):
+        """Test access_by_resource for project includes Forseti SA.
+
+        Args:
+            forseti_cli (ForsetiCli): Instance of the forseti cli helper
+            forseti_model_readonly (Tuple): Model name & process result
+            forseti_server_service_account (str): Server service account email
+            project_id (str): Project id being scanned
+        """
+        # Arrange
+        model_name, _, _ = forseti_model_readonly
+        forseti_cli.model_use(model_name=model_name)
+
+        # Act
+        result = forseti_cli.explainer_access_by_resource(
+            f'project/{project_id}')
+
+        # Assert
+        assert result.returncode == 0, f'Forseti stdout: {str(result.stdout)}'
+        assert 16 == len(re.findall(forseti_server_service_account, str(result.stdout)))

endtoend_tests/forseti/server/inventory/inventory_performance_test.py → endtoend_tests/forseti/inventory/inventory_performance_test.py

@@ -36,6 +36,7 @@ class TestInventoryPerformance:
 
     @pytest.mark.e2e
     @pytest.mark.inventory
+    @pytest.mark.server
     def test_inventory_performance(self,
                                    cai_dump_file_gcs_paths,
                                    cloudsql_connection,

endtoend_tests/forseti/model/model_test.py

@@ -0,0 +1,58 @@
+# Copyright 2020 The Forseti Security Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Model end-to-end tests"""
+
+import pytest
+import re
+from endtoend_tests.helpers.forseti_cli import ForsetiCli
+from sqlalchemy.sql import text
+
+
+class TestModel:
+    """Model tests
+
+    Execute the basic model functionality such as: create, get, use, etc.
+    """
+
+    @pytest.mark.client
+    @pytest.mark.e2e
+    @pytest.mark.model
+    def test_model_use(self, forseti_cli: ForsetiCli, forseti_model_readonly):
+        # Arrange
+        model_name, handle, _ = forseti_model_readonly
+
+        # Act
+        forseti_cli.model_use(model_name)
+        result = forseti_cli.config_show()
+
+        # Assert
+        assert handle
+        assert re.search(fr'{handle}', str(result.stdout))
+
+    @pytest.mark.e2e
+    @pytest.mark.model
+    @pytest.mark.server
+    def test_model_roles(self, cloudsql_connection, forseti_model_readonly):
+        # Arrange/Act
+        model_name, handle, _ = forseti_model_readonly
+
+        # Assert
+        table_name = f'forseti_security.{handle}_roles'
+        query = text('SELECT '
+                     'COUNT(*) '
+                     f'FROM {table_name}')
+        model_roles = (cloudsql_connection.execute(query).fetchone())
+        assert model_roles
+        assert model_roles[0] > 0

endtoend_tests/forseti/server/scanner/cv_cloudsql_location_test.py → endtoend_tests/forseti/scanner/cv_cloudsql_location_test.py

@@ -26,6 +26,7 @@ class TestConfigValidatorCloudSqlLocation:
 
     @pytest.mark.e2e
     @pytest.mark.scanner
+    @pytest.mark.server
     def test_cv_cloudsql_location(self,
                                   cloudsql_connection,
                                   cloudsql_instance_name,

endtoend_tests/forseti/server/scanner/cv_compute_zone_test.py → endtoend_tests/forseti/scanner/cv_compute_zone_test.py

@@ -26,6 +26,7 @@ class TestConfigValidatorComputeZone:
 
     @pytest.mark.e2e
     @pytest.mark.scanner
+    @pytest.mark.server
     def test_cv_compute_zone(self,
                              cloudsql_connection,
                              forseti_scan_readonly,

endtoend_tests/forseti/server/scanner/cv_scan_test.py → endtoend_tests/forseti/scanner/cv_scan_test.py

@@ -26,6 +26,7 @@ class TestConfigValidatorScan:
 
     @pytest.mark.e2e
     @pytest.mark.scanner
+    @pytest.mark.server
     def test_cv_scan(self, forseti_scan_readonly):
         """Config Validator Scan test