Testing framework revamp

tests/collectors.py

@@ -0,0 +1,95 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Pytest plugin to discover tests specified in YAML files.
+
+This plugin uses the pytest_collect_file hook to collect all files
+matching tftest*.yaml and runs plan_validate for each test found.
+See FabricTestFile for details on the file structure.
+
+"""
+
+import pytest
+import yaml
+
+from .fixtures import plan_summary, plan_validator
+
+
+class FabricTestFile(pytest.File):
+
+  def collect(self):
+    """Read yaml test spec and yield test items for each test definition.
+
+    The test spec should contain a `module` key with the path of the
+    terraform module to test, relative to the root of the repository
+
+    Tests are defined within the top-level `tests` key, and should
+    have the following structure:
+
+    test-name:
+      tfvars:
+        - tfvars1.tfvars
+        - tfvars2.tfvars
+      inventory:
+        - inventory1.yaml
+        - inventory2.yaml
+
+    All paths specifications are relative to the location of the test
+    spec. The inventory key is optional, if omitted, the inventory
+    will be taken from the file test-name.yaml
+
+    """
+
+    try:
+      raw = yaml.safe_load(self.path.open())
+      module = raw.pop('module')
+    except (IOError, OSError, yaml.YAMLError) as e:
+      raise Exception(f'cannot read test spec {self.path}: {e}')
+    except KeyError as e:
+      raise Exception(f'`module` key not found in {self.path}: {e}')
+    for test_name, spec in raw.get('tests', {}).items():
+      inventories = spec.get('inventory', [f'{test_name}.yaml'])
+      try:
+        tfvars = spec['tfvars']
+      except KeyError:
+        raise Exception(
+            f'test `{test_name}` in {self.path} does not contain a `tfvars` key'
+        )
+      for i in inventories:
+        name = test_name
+        if isinstance(inventories, list) and len(inventories) > 1:
+          name = f'{test_name}[{i}]'
+        yield FabricTestItem.from_parent(self, name=name, module=module,
+                                         inventory=[i], tfvars=tfvars)
+
+
+class FabricTestItem(pytest.Item):
+
+  def __init__(self, name, parent, module, inventory, tfvars):
+    super().__init__(name, parent)
+    self.module = module
+    self.inventory = inventory
+    self.tfvars = tfvars
+
+  def runtest(self):
+    s = plan_validator(self.module, self.inventory, self.parent.path.parent,
+                       self.tfvars)
+
+  def reportinfo(self):
+    return self.path, None, self.name
+
+
+def pytest_collect_file(parent, file_path):
+  'Collect tftest*.yaml files and run plan_validator from them.'
+  if file_path.suffix == '.yaml' and file_path.name.startswith('tftest'):
+    return FabricTestFile.from_parent(parent, path=file_path)

tests/conftest.py

@@ -11,144 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"Shared fixtures"
-
-import inspect
-import os
-import shutil
-import tempfile
+'Pytest configuration.'
 
 import pytest
-import tftest
-
-BASEDIR = os.path.dirname(os.path.dirname(__file__))
-
-
-@pytest.fixture(scope='session')
-def _plan_runner():
-  "Returns a function to run Terraform plan on a fixture."
-
-  def run_plan(fixture_path=None, extra_files=None, tf_var_file=None,
-               targets=None, refresh=True, tmpdir=True, **tf_vars):
-    "Runs Terraform plan and returns parsed output."
-    if fixture_path is None:
-      # find out the fixture directory from the caller's directory
-      caller = inspect.stack()[2]
-      fixture_path = os.path.join(os.path.dirname(caller.filename), "fixture")
-
-    fixture_parent = os.path.dirname(fixture_path)
-    fixture_prefix = os.path.basename(fixture_path) + "_"
-    with tempfile.TemporaryDirectory(prefix=fixture_prefix,
-                                     dir=fixture_parent) as tmp_path:
-      # copy fixture to a temporary directory so we can execute
-      # multiple tests in parallel
-      if tmpdir:
-        shutil.copytree(fixture_path, tmp_path, dirs_exist_ok=True)
-      tf = tftest.TerraformTest(tmp_path if tmpdir else fixture_path, BASEDIR,
-                                os.environ.get('TERRAFORM', 'terraform'))
-      tf.setup(extra_files=extra_files, upgrade=True)
-      plan = tf.plan(output=True, refresh=refresh, tf_var_file=tf_var_file,
-                     tf_vars=tf_vars, targets=targets)
-    return plan
-
-  return run_plan
-
-
-@pytest.fixture(scope='session')
-def plan_runner(_plan_runner):
-  "Returns a function to run Terraform plan on a module fixture."
-
-  def run_plan(fixture_path=None, extra_files=None, tf_var_file=None,
-               targets=None, **tf_vars):
-    "Runs Terraform plan and returns plan and module resources."
-    plan = _plan_runner(fixture_path, extra_files=extra_files,
-                        tf_var_file=tf_var_file, targets=targets, **tf_vars)
-    # skip the fixture
-    root_module = plan.root_module['child_modules'][0]
-    return plan, root_module['resources']
-
-  return run_plan
-
-
-@pytest.fixture(scope='session')
-def e2e_plan_runner(_plan_runner):
-  "Returns a function to run Terraform plan on an end-to-end fixture."
-
-  def run_plan(fixture_path=None, tf_var_file=None, targets=None, 
-               refresh=True, include_bare_resources=False, **tf_vars):
-    "Runs Terraform plan on an end-to-end module using defaults, returns data."
-    plan = _plan_runner(fixture_path, tf_var_file=tf_var_file, targets=targets, 
-                        refresh=refresh, **tf_vars)
-    # skip the fixture
-    root_module = plan.root_module['child_modules'][0]
-    modules = dict((mod['address'], mod['resources'])
-                   for mod in root_module['child_modules'])
-    resources = [r for m in modules.values() for r in m]
-    if include_bare_resources:
-      bare_resources = root_module['resources']
-      resources.extend(bare_resources)
-    return modules, resources
-
-  return run_plan
-
-
-@pytest.fixture(scope='session')
-def recursive_e2e_plan_runner(_plan_runner):
-  """Plan runner for end-to-end root module, returns total number of
-  (nested) modules and resources"""
-
-  def walk_plan(node, modules, resources):
-    # TODO(jccb): this would be better with node.get() but
-    # TerraformPlanOutput objects don't have it
-    new_modules = node.get('child_modules', [])
-    resources += node.get('resources', [])
-    modules += new_modules
-    for module in new_modules:
-      walk_plan(module, modules, resources)
-
-  def run_plan(fixture_path=None, tf_var_file=None, targets=None, refresh=True,
-               include_bare_resources=False, compute_sums=True, tmpdir=True,
-               **tf_vars):
-    "Runs Terraform plan on a root module using defaults, returns data."
-    plan = _plan_runner(fixture_path, tf_var_file=tf_var_file, targets=targets, 
-                        refresh=refresh, tmpdir=tmpdir, **tf_vars)
-    modules = []
-    resources = []
-    walk_plan(plan.root_module, modules, resources)
-    return len(modules), len(resources)
-
-  return run_plan
-
-
-@pytest.fixture(scope='session')
-def apply_runner():
-  "Returns a function to run Terraform apply on a fixture."
-
-  def run_apply(fixture_path=None, **tf_vars):
-    "Runs Terraform plan and returns parsed output."
-    if fixture_path is None:
-      # find out the fixture directory from the caller's directory
-      caller = inspect.stack()[1]
-      fixture_path = os.path.join(os.path.dirname(caller.filename), "fixture")
-
-    fixture_parent = os.path.dirname(fixture_path)
-    fixture_prefix = os.path.basename(fixture_path) + "_"
-
-    with tempfile.TemporaryDirectory(prefix=fixture_prefix,
-                                     dir=fixture_parent) as tmp_path:
-      # copy fixture to a temporary directory so we can execute
-      # multiple tests in parallel
-      shutil.copytree(fixture_path, tmp_path, dirs_exist_ok=True)
-      tf = tftest.TerraformTest(tmp_path, BASEDIR,
-                                os.environ.get('TERRAFORM', 'terraform'))
-      tf.setup(upgrade=True)
-      apply = tf.apply(tf_vars=tf_vars)
-      output = tf.output(json_format=True)
-      return apply, output
-
-  return run_apply
-
 
-@pytest.fixture
-def basedir():
-  return BASEDIR
+pytest_plugins = (
+    'tests.fixtures',
+    'tests.legacy_fixtures',
+    'tests.collectors',
+)

tests/fast/stages/s00_bootstrap/simple.tfvars

@@ -0,0 +1,11 @@
+organization = {
+  domain      = "fast.example.com"
+  id          = 123456789012
+  customer_id = "C00000000"
+}
+billing_account = {
+  id              = "000000-111111-222222"
+  organization_id = 123456789012
+}
+prefix           = "fast"
+outputs_location = "/fast-config"

tests/fast/stages/s00_bootstrap/simple.yaml

@@ -0,0 +1,49 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+counts:
+  google_bigquery_dataset: 2
+  google_bigquery_dataset_iam_member: 2
+  google_bigquery_default_service_account: 3
+  google_logging_organization_sink: 2
+  google_organization_iam_binding: 19
+  google_organization_iam_custom_role: 2
+  google_organization_iam_member: 16
+  google_project: 3
+  google_project_iam_binding: 9
+  google_project_iam_member: 1
+  google_project_service: 29
+  google_project_service_identity: 2
+  google_service_account: 3
+  google_service_account_iam_binding: 3
+  google_storage_bucket: 4
+  google_storage_bucket_iam_binding: 2
+  google_storage_bucket_iam_member: 3
+  google_storage_bucket_object: 5
+  google_storage_project_service_account: 3
+  local_file: 5
+
+outputs:
+  custom_roles:
+    organization_iam_admin: organizations/123456789012/roles/organizationIamAdmin
+    service_project_network_admin: organizations/123456789012/roles/serviceProjectNetworkAdmin
+  outputs_bucket: fast-prod-iac-core-outputs-0
+  project_ids:
+    automation: fast-prod-iac-core-0
+    billing-export: fast-prod-billing-exp-0
+    log-export: fast-prod-audit-logs-0
+  service_accounts:
+    bootstrap: fast-prod-bootstrap-0@fast-prod-iac-core-0.iam.gserviceaccount.com
+    cicd: fast-prod-cicd-0@fast-prod-iac-core-0.iam.gserviceaccount.com
+    resman: fast-prod-resman-0@fast-prod-iac-core-0.iam.gserviceaccount.com

tests/fast/stages/s00_bootstrap/simple_projects.yaml

@@ -0,0 +1,33 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+values:
+  module.automation-project.google_project.project[0]:
+    auto_create_network: false
+    billing_account: 000000-111111-222222
+    name: fast-prod-iac-core-0
+    org_id: '123456789012'
+    project_id: fast-prod-iac-core-0
+  module.billing-export-project[0].google_project.project[0]:
+    auto_create_network: false
+    billing_account: 000000-111111-222222
+    name: fast-prod-billing-exp-0
+    org_id: '123456789012'
+    project_id: fast-prod-billing-exp-0
+  module.log-export-project.google_project.project[0]:
+    auto_create_network: false
+    billing_account: 000000-111111-222222
+    name: fast-prod-audit-logs-0
+    org_id: '123456789012'
+    project_id: fast-prod-audit-logs-0