Add link_module function

CHANGELOG.md

@@ -9,6 +9,7 @@
     * More human-friendly way to define Terraform blocks in Python.
 * `workflow.delete_links()` function added.
 * `workflow.link_files()` function added.
+* `workflow.link_module()` function added.
 * `workflow.load_parent()` function added.
     * More composable workflows.
 

docs/api/workflow.md

@@ -287,6 +287,51 @@ def pretf_workflow():
     return workflow.execute_terraform()
 ```
 
+## link_module
+
+Creates symlinks from all files and directories in a module into the current directory. Remote modules are first downloaded into a cache directory.
+
+Signature:
+
+```python
+def link_module(
+    source: Union[Path,str],
+    version: Optional[str] = None,
+    update: bool = False,
+    cache_dir: Optional[Union[Path, str]] = None,
+    cwd: Optional[Union[Path, str]] = None,
+    verbose: bool = True,
+) -> List[Path]:
+
+source:
+    location of module to mirror into the current directory
+version:
+    the module version (if using registry)
+update:
+    whether to fetch the module every time, or use a cached copy
+cache_dir:
+    location to use for caching modules
+cwd:
+    current directory
+verbose:
+    whether to print information
+
+returns:
+    created symlinks
+```
+
+Example:
+
+```python
+from pretf import workflow
+
+
+def pretf_workflow():
+    workflow.delete_links()
+    workflow.link_module("claranet/vpc-modules/aws", version="1.1.0")
+    return workflow.execute_terraform()
+```
+
 ## mirror_files
 
 > This function will be removed in a future version. Use [delete_links](#delete_links) and [link_files](#link_files) instead.

pretf/pretf/test.py

@@ -15,14 +15,13 @@
 
 
 class TerraformProxy:
-    def __init__(self, cwd: Union[Path, str] = "", verbose: Optional[bool] = None):
+    def __init__(self, cwd: Union[Path, str] = "", verbose: Optional[bool] = False):
         if not isinstance(cwd, Path):
             cwd = Path(cwd)
         self.cwd = cwd
         self.env = os.environ.copy()
         self.env["TF_IN_AUTOMATION"] = "1"
-        if verbose is not None:
-            self.env["PRETF_VERBOSE"] = "1" if verbose else "0"
+        self.env["PRETF_VERBOSE"] = "1" if verbose else "0"
         self.verbose = verbose
 
     # Calling the object just returns another object with the specified path.
@@ -73,6 +72,18 @@ def destroy(self, *args: str) -> str:
                 destroy_args.append(arg)
         return self.execute(*destroy_args).stdout
 
+    def get(self, *args: str) -> str:
+        """
+        Runs terraform get and returns the stdout.
+
+        """
+
+        get_args = ["get"]
+        for arg in args:
+            if arg not in get_args:
+                get_args.append(arg)
+        return self.execute(*get_args).stdout
+
     def init(self, *args: str) -> str:
         """
         Runs terraform init and returns the stdout.

pretf/pretf/workflow.py

@@ -218,7 +218,7 @@ def delete_links(
 
     if delete and is_verbose(verbose):
         names = [path.name for path in delete]
-        log.ok(f"delete: {' '.join(sorted(names))}")
+        log.ok(f"unlink: {' '.join(sorted(names))}")
 
     # Delete links.
     deleted = []
@@ -271,7 +271,7 @@ def execute_terraform(
 
         # This is a valid executable, run it.
         return util.execute(
-            file=terraform_path, args=args, capture=capture, verbose=verbose
+            file=terraform_path, args=args, cwd=cwd, capture=capture, verbose=verbose
         )
 
     log.bad("terraform: command not found")
@@ -406,7 +406,7 @@ def link_files(
 
     if create and is_verbose(verbose):
         names = [path.name for path in create.keys()]
-        log.ok(f"mirror: {' '.join(sorted(names))}")
+        log.ok(f"link: {' '.join(sorted(names))}")
 
     # Create new symlinks.
     created = []
@@ -417,6 +417,79 @@ def link_files(
     return created
 
 
+def link_module(
+    source: str,
+    version: Optional[str] = None,
+    update: bool = False,
+    cache_dir: Optional[Union[Path, str]] = None,
+    cwd: Optional[Union[Path, str]] = None,
+    verbose: Optional[bool] = None,
+) -> List[Path]:
+    """
+    Creates symlinks from all files and directories in a module into
+    the current directory. Remote modules are first downloaded into a
+    cache directory.
+
+    """
+
+    if is_verbose(verbose):
+        if version:
+            log.ok(f"module: {source} {version}")
+        else:
+            log.ok(f"module: {source}")
+
+    if cwd is None:
+        cwd = Path.cwd()
+    elif isinstance(cwd, str):
+        cwd = Path(cwd)
+
+    paths: List[Path] = []
+
+    if source.startswith(".") or source.startswith("/"):
+
+        # Modules already on the filesystem can be used directly.
+        paths.extend(util.find_paths(path_patterns=["*"], cwd=source,))
+
+    else:
+
+        # Remote modules will be managed by Terraform in a
+        # cache directory and then symlinked from there.
+        module_name = "mirror-module"
+
+        # Ensure the module cache directory exists.
+        if cache_dir is None:
+            cache_dir = cwd / ".terraform" / "pretf" / module_name
+        elif isinstance(cache_dir, str):
+            cache_dir = Path(cache_dir)
+        cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Create a Terraform root module in the cache directory
+        # that just references the specified module.
+        module_config_path = cache_dir / "main.tf.json"
+        module_body = {"source": source}
+        if version:
+            module_body["version"] = version
+        module_json = json.dumps([{"module": {module_name: module_body}}], indent=2)
+        module_config_path.write_text(module_json)
+
+        # Run "terraform get" to download the module using Terraform.
+        from .test import TerraformProxy
+
+        terraform_get_args = ["-update"] if update else []
+        TerraformProxy(cwd=cache_dir).get(*terraform_get_args)
+
+        # Get the path to the module.
+        modules_manifest_path = cache_dir / ".terraform" / "modules" / "modules.json"
+        modules_manifest = json.loads(modules_manifest_path.read_text())
+        for module in modules_manifest["Modules"]:
+            if module["Key"] == module_name:
+                module_dir = cache_dir / module["Dir"]
+                # Use files from the downloaded module directory.
+                paths.extend(util.find_paths(path_patterns=["*"], cwd=module_dir))
+
+    return link_files(*paths, cwd=cwd, verbose=verbose)
+
+
 def mirror_files(
     *path_patterns: Union[Path, str],
     exclude_name_patterns: Sequence[str] = [".*", "_*", "pretf.workflow.py"],