Terraform: Add experimental-deploy and wire-in dependency inference

src/python/pants/backend/experimental/terraform/register.py

@@ -2,26 +2,28 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.goals import lockfile as python_lockfile
-from pants.backend.terraform import dependency_inference, tool
-from pants.backend.terraform.goals import check, tailor
+from pants.backend.terraform import dependencies, dependency_inference, tool
+from pants.backend.terraform.goals import check, deploy, tailor
 from pants.backend.terraform.lint.tffmt.tffmt import rules as tffmt_rules
-from pants.backend.terraform.target_types import TerraformModuleTarget
+from pants.backend.terraform.target_types import TerraformDeploymentTarget, TerraformModuleTarget
 from pants.backend.terraform.target_types import rules as target_types_rules
 from pants.engine.rules import collect_rules
 
 
 def target_types():
-    return [TerraformModuleTarget]
+    return [TerraformModuleTarget, TerraformDeploymentTarget]
 
 
 def rules():
     return [
         *collect_rules(),
+        *dependencies.rules(),
         *check.rules(),
         *dependency_inference.rules(),
         *tailor.rules(),
         *target_types_rules(),
         *tool.rules(),
         *tffmt_rules(),
+        *deploy.rules(),
         *python_lockfile.rules(),
     ]

src/python/pants/backend/terraform/dependencies.py

@@ -0,0 +1,173 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Tuple
+
+from pants.backend.terraform.partition import partition_files_by_directory
+from pants.backend.terraform.target_types import (
+    TerraformBackendConfigField,
+    TerraformDependenciesField,
+    TerraformRootModuleField,
+)
+from pants.backend.terraform.tool import TerraformProcess
+from pants.backend.terraform.utils import terraform_arg, terraform_relpath
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.internals.native_engine import (
+    EMPTY_DIGEST,
+    Address,
+    AddressInput,
+    Digest,
+    MergeDigests,
+)
+from pants.engine.internals.selectors import Get, MultiGet
+from pants.engine.process import FallibleProcessResult
+from pants.engine.rules import collect_rules, rule
+from pants.engine.target import (
+    DependenciesRequest,
+    SourcesField,
+    Targets,
+    WrappedTarget,
+    WrappedTargetRequest,
+)
+
+
+@dataclass(frozen=True)
+class TerraformDependenciesRequest:
+    source_files: SourceFiles
+    directories: Tuple[str, ...]
+    backend_config: SourceFiles
+    dependencies_files: SourceFiles
+
+    # Not initialising the backend means we won't access remote state. Useful for `validate`
+    initialise_backend: bool = False
+
+
+@dataclass(frozen=True)
+class TerraformDependenciesResponse:
+    fetched_deps: Tuple[Tuple[str, Digest], ...]
+
+
+@rule
+async def get_terraform_providers(
+    req: TerraformDependenciesRequest,
+) -> TerraformDependenciesResponse:
+    args = ["init"]
+    if req.backend_config.files:
+        args.append(
+            terraform_arg(
+                "-backend-config",
+                terraform_relpath(req.directories[0], req.backend_config.files[0]),
+            )
+        )
+        backend_digest = req.backend_config.snapshot.digest
+    else:
+        backend_digest = EMPTY_DIGEST
+
+    args.append(terraform_arg("-backend", str(req.initialise_backend)))
+
+    with_backend_config = await Get(
+        Digest,
+        MergeDigests(
+            [
+                req.source_files.snapshot.digest,
+                backend_digest,
+                req.dependencies_files.snapshot.digest,
+            ]
+        ),
+    )
+
+    # TODO: Does this need to be a MultiGet? I think we will now always get one directory
+    fetched_deps = await MultiGet(
+        Get(
+            FallibleProcessResult,
+            TerraformProcess(
+                args=tuple(args),
+                input_digest=with_backend_config,
+                output_files=(".terraform.lock.hcl",),
+                output_directories=(".terraform",),
+                description="Run `terraform init` to fetch dependencies",
+                chdir=directory,
+            ),
+        )
+        for directory in req.directories
+    )
+
+    return TerraformDependenciesResponse(
+        tuple(zip(req.directories, (x.output_digest for x in fetched_deps)))
+    )
+
+
+@dataclass(frozen=True)
+class TerraformInitRequest:
+    root_module: TerraformRootModuleField
+    backend_config: TerraformBackendConfigField
+    dependencies: TerraformDependenciesField
+
+    # Not initialising the backend means we won't access remote state. Useful for `validate`
+    initialise_backend: bool = False
+
+
+@dataclass(frozen=True)
+class TerraformInitResponse:
+    sources_and_deps: Digest
+    terraform_files: tuple[str, ...]
+    chdir: str
+
+
+@rule
+async def init_terraform(request: TerraformInitRequest) -> TerraformInitResponse:
+    address_input = request.root_module.to_address_input()
+    module_address = await Get(Address, AddressInput, address_input)
+    module = await Get(
+        WrappedTarget,
+        WrappedTargetRequest(
+            module_address, description_of_origin=address_input.description_of_origin
+        ),
+    )
+
+    root_dependencies, module_dependencies = await MultiGet(
+        Get(Targets, DependenciesRequest(request.dependencies)),
+        Get(Targets, DependenciesRequest(module.target.get(TerraformDependenciesField))),
+    )
+
+    source_files, backend_config, dependencies_files = await MultiGet(
+        Get(SourceFiles, SourceFilesRequest([module.target.get(SourcesField)])),
+        Get(SourceFiles, SourceFilesRequest([request.backend_config])),
+        Get(
+            SourceFiles,
+            SourceFilesRequest(
+                [tgt.get(SourcesField) for tgt in (*root_dependencies, *module_dependencies)]
+            ),
+        ),
+    )
+    files_by_directory = partition_files_by_directory(source_files.files)
+
+    fetched_deps = await Get(
+        TerraformDependenciesResponse,
+        TerraformDependenciesRequest(
+            source_files,
+            tuple(files_by_directory.keys()),
+            backend_config,
+            dependencies_files,
+            initialise_backend=request.initialise_backend,
+        ),
+    )
+
+    merged_fetched_deps = await Get(Digest, MergeDigests([x[1] for x in fetched_deps.fetched_deps]))
+
+    sources_and_deps = await Get(
+        Digest,
+        MergeDigests(
+            [source_files.snapshot.digest, merged_fetched_deps, dependencies_files.snapshot.digest]
+        ),
+    )
+
+    assert len(files_by_directory) == 1, "Multiple directories found, unable to identify a root"
+    chdir, files = next(iter(files_by_directory.items()))
+    return TerraformInitResponse(sources_and_deps, tuple(files), chdir)
+
+
+def rules():
+    return collect_rules()

src/python/pants/backend/terraform/dependencies_test.py

@@ -0,0 +1,131 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import json
+import textwrap
+from pathlib import Path
+from typing import Optional
+
+from pants.backend.terraform.dependencies import TerraformInitRequest, TerraformInitResponse
+from pants.backend.terraform.goals.deploy import DeployTerraformFieldSet
+from pants.backend.terraform.goals.deploy_test import (
+    StandardDeployment,
+    rule_runner,
+    standard_deployment,
+)
+from pants.engine.fs import DigestContents, FileContent
+from pants.engine.internals.native_engine import Address
+from pants.testutil.rule_runner import RuleRunner
+
+rule_runner = rule_runner
+standard_deployment = standard_deployment
+
+
+def _do_init_terraform(
+    rule_runner: RuleRunner, standard_deployment: StandardDeployment, initialise_backend: bool
+) -> DigestContents:
+    rule_runner.write_files(standard_deployment.files)
+    target = rule_runner.get_target(standard_deployment.target)
+    field_set = DeployTerraformFieldSet.create(target)
+    result = rule_runner.request(
+        TerraformInitResponse,
+        [
+            TerraformInitRequest(
+                field_set.root_module,
+                field_set.backend_config,
+                field_set.dependencies,
+                initialise_backend=initialise_backend,
+            )
+        ],
+    )
+    initialised_files = rule_runner.request(DigestContents, [result.sources_and_deps])
+    assert isinstance(initialised_files, DigestContents)
+    return initialised_files
+
+
+def find_file(files: DigestContents, pattern: str) -> Optional[FileContent]:
+    return next((file for file in files if Path(file.path).match(pattern)), None)
+
+
+def test_init_terraform(rule_runner: RuleRunner, standard_deployment: StandardDeployment) -> None:
+    """Test for the happy path of initialising Terraform with a backend config."""
+    initialised_files = _do_init_terraform(
+        rule_runner, standard_deployment, initialise_backend=True
+    )
+
+    # Assert uses backend by checking that the overrides in the backend file are present in the local stub state file
+    stub_tfstate_raw = find_file(initialised_files, "src/tf/.terraform/terraform.tfstate")
+    assert stub_tfstate_raw
+    stub_tfstate = json.loads(stub_tfstate_raw.content)
+    assert stub_tfstate["backend"]["config"]["path"] == str(standard_deployment.state_file)
+
+    # Assert dependencies are initialised by checking for the dependency itself
+    assert find_file(
+        initialised_files,
+        ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*",
+    ), "Did not find expected provider"
+
+    # Assert lockfile is included
+    assert find_file(initialised_files, ".terraform.lock.hcl"), "Did not find expected provider"
+
+
+def test_init_terraform_without_backends(
+    rule_runner: RuleRunner, standard_deployment: StandardDeployment
+) -> None:
+    initialised_files = _do_init_terraform(
+        rule_runner, standard_deployment, initialise_backend=False
+    )
+
+    # Not initialising the backend means that ./.terraform/.terraform.tfstate will not be present
+    assert not find_file(
+        initialised_files, "**/*.tfstate"
+    ), "Terraform state file should not be present if the the request was to not initialise the backend"
+
+    # The dependencies should still be present
+    assert find_file(
+        initialised_files,
+        ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*",
+    ), "Did not find expected provider"
+
+
+def test_init_terraform_with_in_repo_module(rule_runner: RuleRunner, tmpdir) -> None:
+    deployment_files = {
+        "src/tf/deployment/BUILD": textwrap.dedent(
+            """\
+            terraform_deployment(name="root", root_module=":mod")
+            terraform_module(name="mod")
+        """
+        ),
+        "src/tf/deployment/main.tf": textwrap.dedent(
+            """\
+            module "mod0" {
+              source = "../module/"
+            }
+        """
+        ),
+    }
+    module_files = {
+        "src/tf/module/BUILD": "terraform_module()",
+        "src/tf/module/main.tf": 'resource "null_resource" "dep" {}',
+    }
+
+    deployment = StandardDeployment(
+        {**deployment_files, **module_files},
+        Path(str(tmpdir.mkdir(".terraform").join("state.json"))),
+        Address("src/tf/deployment", target_name="root"),
+    )
+    initialised_files = _do_init_terraform(rule_runner, deployment, initialise_backend=True)
+
+    # Assert that our module got included in the module.json
+    assert initialised_files
+    modules_file_raw = find_file(initialised_files, ".terraform/modules/modules.json")
+    assert modules_file_raw
+    modules_file = json.loads(modules_file_raw.content)
+    assert any(
+        module for module in modules_file["Modules"] if module["Key"] == "mod0"
+    ), "Did not find our module in modules.json"
+
+    # Assert that the module was explored as part of init
+    assert find_file(
+        initialised_files,
+        ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*",
+    ), "Did not find expected provider contained in module, did we successfully include it in the files passed to `init`?"