Terraform: Add experimental-deploy and wire-in dependency inference (…

…#19185)

This was actually some prework for lockfiles. But Terraform has some
ideas about differences between a "root" module (one you would deploy)
and other modules (ones which would be included in a deployment). To
model that, I creating the `TerraformDeployment` target type. Once you
have that, might as well wire it in to the `experimental-deploy` goal
(especially since that's most of the value of Terraform).
Also it looks like we weren't fetching dependencies (on other modules)
when building up the files to run Terraform commands. Now that happens.

I feel like I'm not being efficient in this MR with the way I bundle the
(pants-inferred) dependencies together with everything, since there's
this ever-growing ball of files. LMK if we've got a better way.

src/python/pants/backend/experimental/terraform/register.py

@@ -2,26 +2,28 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.python.goals import lockfile as python_lockfile
-from pants.backend.terraform import dependency_inference, tool
-from pants.backend.terraform.goals import check, tailor
+from pants.backend.terraform import dependencies, dependency_inference, tool
+from pants.backend.terraform.goals import check, deploy, tailor
 from pants.backend.terraform.lint.tffmt.tffmt import rules as tffmt_rules
-from pants.backend.terraform.target_types import TerraformModuleTarget
+from pants.backend.terraform.target_types import TerraformDeploymentTarget, TerraformModuleTarget
 from pants.backend.terraform.target_types import rules as target_types_rules
 from pants.engine.rules import collect_rules
 
 
 def target_types():
-    return [TerraformModuleTarget]
+    return [TerraformModuleTarget, TerraformDeploymentTarget]
 
 
 def rules():
     return [
         *collect_rules(),
+        *dependencies.rules(),
         *check.rules(),
         *dependency_inference.rules(),
         *tailor.rules(),
         *target_types_rules(),
         *tool.rules(),
         *tffmt_rules(),
+        *deploy.rules(),
         *python_lockfile.rules(),
     ]

src/python/pants/backend/terraform/dependencies.py

@@ -0,0 +1,173 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Tuple
+
+from pants.backend.terraform.partition import partition_files_by_directory
+from pants.backend.terraform.target_types import (
+    TerraformBackendConfigField,
+    TerraformDependenciesField,
+    TerraformRootModuleField,
+)
+from pants.backend.terraform.tool import TerraformProcess
+from pants.backend.terraform.utils import terraform_arg, terraform_relpath
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.internals.native_engine import (
+    EMPTY_DIGEST,
+    Address,
+    AddressInput,
+    Digest,
+    MergeDigests,
+)
+from pants.engine.internals.selectors import Get, MultiGet
+from pants.engine.process import FallibleProcessResult
+from pants.engine.rules import collect_rules, rule
+from pants.engine.target import (
+    DependenciesRequest,
+    SourcesField,
+    Targets,
+    WrappedTarget,
+    WrappedTargetRequest,
+)
+
+
+@dataclass(frozen=True)
+class TerraformDependenciesRequest:
+    source_files: SourceFiles
+    directories: Tuple[str, ...]
+    backend_config: SourceFiles
+    dependencies_files: SourceFiles
+
+    # Not initialising the backend means we won't access remote state. Useful for `validate`
+    initialise_backend: bool = False
+
+
+@dataclass(frozen=True)
+class TerraformDependenciesResponse:
+    fetched_deps: Tuple[Tuple[str, Digest], ...]
+
+
+@rule
+async def get_terraform_providers(
+    req: TerraformDependenciesRequest,
+) -> TerraformDependenciesResponse:
+    args = ["init"]
+    if req.backend_config.files:
+        args.append(
+            terraform_arg(
+                "-backend-config",
+                terraform_relpath(req.directories[0], req.backend_config.files[0]),
+            )
+        )
+        backend_digest = req.backend_config.snapshot.digest
+    else:
+        backend_digest = EMPTY_DIGEST
+
+    args.append(terraform_arg("-backend", str(req.initialise_backend)))
+
+    with_backend_config = await Get(
+        Digest,
+        MergeDigests(
+            [
+                req.source_files.snapshot.digest,
+                backend_digest,
+                req.dependencies_files.snapshot.digest,
+            ]
+        ),
+    )
+
+    # TODO: Does this need to be a MultiGet? I think we will now always get one directory
+    fetched_deps = await MultiGet(
+        Get(
+            FallibleProcessResult,
+            TerraformProcess(
+                args=tuple(args),
+                input_digest=with_backend_config,
+                output_files=(".terraform.lock.hcl",),
+                output_directories=(".terraform",),
+                description="Run `terraform init` to fetch dependencies",
+                chdir=directory,
+            ),
+        )
+        for directory in req.directories
+    )
+
+    return TerraformDependenciesResponse(
+        tuple(zip(req.directories, (x.output_digest for x in fetched_deps)))
+    )
+
+
+@dataclass(frozen=True)
+class TerraformInitRequest:
+    root_module: TerraformRootModuleField
+    backend_config: TerraformBackendConfigField
+    dependencies: TerraformDependenciesField
+
+    # Not initialising the backend means we won't access remote state. Useful for `validate`
+    initialise_backend: bool = False
+
+
+@dataclass(frozen=True)
+class TerraformInitResponse:
+    sources_and_deps: Digest
+    terraform_files: tuple[str, ...]
+    chdir: str
+
+
+@rule
+async def init_terraform(request: TerraformInitRequest) -> TerraformInitResponse:
+    address_input = request.root_module.to_address_input()
+    module_address = await Get(Address, AddressInput, address_input)
+    module = await Get(
+        WrappedTarget,
+        WrappedTargetRequest(
+            module_address, description_of_origin=address_input.description_of_origin
+        ),
+    )
+
+    root_dependencies, module_dependencies = await MultiGet(
+        Get(Targets, DependenciesRequest(request.dependencies)),
+        Get(Targets, DependenciesRequest(module.target.get(TerraformDependenciesField))),
+    )
+
+    source_files, backend_config, dependencies_files = await MultiGet(
+        Get(SourceFiles, SourceFilesRequest([module.target.get(SourcesField)])),
+        Get(SourceFiles, SourceFilesRequest([request.backend_config])),
+        Get(
+            SourceFiles,
+            SourceFilesRequest(
+                [tgt.get(SourcesField) for tgt in (*root_dependencies, *module_dependencies)]
+            ),
+        ),
+    )
+    files_by_directory = partition_files_by_directory(source_files.files)
+
+    fetched_deps = await Get(
+        TerraformDependenciesResponse,
+        TerraformDependenciesRequest(
+            source_files,
+            tuple(files_by_directory.keys()),
+            backend_config,
+            dependencies_files,
+            initialise_backend=request.initialise_backend,
+        ),
+    )
+
+    merged_fetched_deps = await Get(Digest, MergeDigests([x[1] for x in fetched_deps.fetched_deps]))
+
+    sources_and_deps = await Get(
+        Digest,
+        MergeDigests(
+            [source_files.snapshot.digest, merged_fetched_deps, dependencies_files.snapshot.digest]
+        ),
+    )
+
+    assert len(files_by_directory) == 1, "Multiple directories found, unable to identify a root"
+    chdir, files = next(iter(files_by_directory.items()))
+    return TerraformInitResponse(sources_and_deps, tuple(files), chdir)
+
+
+def rules():
+    return collect_rules()

src/python/pants/backend/terraform/dependencies_test.py

@@ -0,0 +1,131 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+import json
+import textwrap
+from pathlib import Path
+from typing import Optional
+
+from pants.backend.terraform.dependencies import TerraformInitRequest, TerraformInitResponse
+from pants.backend.terraform.goals.deploy import DeployTerraformFieldSet
+from pants.backend.terraform.goals.deploy_test import (
+    StandardDeployment,
+    rule_runner,
+    standard_deployment,
+)
+from pants.engine.fs import DigestContents, FileContent
+from pants.engine.internals.native_engine import Address
+from pants.testutil.rule_runner import RuleRunner
+
+rule_runner = rule_runner
+standard_deployment = standard_deployment
+
+
+def _do_init_terraform(
+    rule_runner: RuleRunner, standard_deployment: StandardDeployment, initialise_backend: bool
+) -> DigestContents:
+    rule_runner.write_files(standard_deployment.files)
+    target = rule_runner.get_target(standard_deployment.target)
+    field_set = DeployTerraformFieldSet.create(target)
+    result = rule_runner.request(
+        TerraformInitResponse,
+        [
+            TerraformInitRequest(
+                field_set.root_module,
+                field_set.backend_config,
+                field_set.dependencies,
+                initialise_backend=initialise_backend,
+            )
+        ],
+    )
+    initialised_files = rule_runner.request(DigestContents, [result.sources_and_deps])
+    assert isinstance(initialised_files, DigestContents)
+    return initialised_files
+
+
+def find_file(files: DigestContents, pattern: str) -> Optional[FileContent]:
+    return next((file for file in files if Path(file.path).match(pattern)), None)
+
+
+def test_init_terraform(rule_runner: RuleRunner, standard_deployment: StandardDeployment) -> None:
+    """Test for the happy path of initialising Terraform with a backend config."""
+    initialised_files = _do_init_terraform(
+        rule_runner, standard_deployment, initialise_backend=True
+    )
+
+    # Assert uses backend by checking that the overrides in the backend file are present in the local stub state file
+    stub_tfstate_raw = find_file(initialised_files, "src/tf/.terraform/terraform.tfstate")
+    assert stub_tfstate_raw
+    stub_tfstate = json.loads(stub_tfstate_raw.content)
+    assert stub_tfstate["backend"]["config"]["path"] == str(standard_deployment.state_file)
+
+    # Assert dependencies are initialised by checking for the dependency itself
+    assert find_file(
+        initialised_files,
+        ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*",
+    ), "Did not find expected provider"
+
+    # Assert lockfile is included
+    assert find_file(initialised_files, ".terraform.lock.hcl"), "Did not find expected provider"
+
+
+def test_init_terraform_without_backends(
+    rule_runner: RuleRunner, standard_deployment: StandardDeployment
+) -> None:
+    initialised_files = _do_init_terraform(
+        rule_runner, standard_deployment, initialise_backend=False
+    )
+
+    # Not initialising the backend means that ./.terraform/.terraform.tfstate will not be present
+    assert not find_file(
+        initialised_files, "**/*.tfstate"
+    ), "Terraform state file should not be present if the the request was to not initialise the backend"
+
+    # The dependencies should still be present
+    assert find_file(
+        initialised_files,
+        ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*",
+    ), "Did not find expected provider"
+
+
+def test_init_terraform_with_in_repo_module(rule_runner: RuleRunner, tmpdir) -> None:
+    deployment_files = {
+        "src/tf/deployment/BUILD": textwrap.dedent(
+            """\
+            terraform_deployment(name="root", root_module=":mod")
+            terraform_module(name="mod")
+        """
+        ),
+        "src/tf/deployment/main.tf": textwrap.dedent(
+            """\
+            module "mod0" {
+              source = "../module/"
+            }
+        """
+        ),
+    }
+    module_files = {
+        "src/tf/module/BUILD": "terraform_module()",
+        "src/tf/module/main.tf": 'resource "null_resource" "dep" {}',
+    }
+
+    deployment = StandardDeployment(
+        {**deployment_files, **module_files},
+        Path(str(tmpdir.mkdir(".terraform").join("state.json"))),
+        Address("src/tf/deployment", target_name="root"),
+    )
+    initialised_files = _do_init_terraform(rule_runner, deployment, initialise_backend=True)
+
+    # Assert that our module got included in the module.json
+    assert initialised_files
+    modules_file_raw = find_file(initialised_files, ".terraform/modules/modules.json")
+    assert modules_file_raw
+    modules_file = json.loads(modules_file_raw.content)
+    assert any(
+        module for module in modules_file["Modules"] if module["Key"] == "mod0"
+    ), "Did not find our module in modules.json"
+
+    # Assert that the module was explored as part of init
+    assert find_file(
+        initialised_files,
+        ".terraform/providers/registry.terraform.io/hashicorp/null/*/*/terraform-provider-null*",
+    ), "Did not find expected provider contained in module, did we successfully include it in the files passed to `init`?"