Support compiling directories recursively

Implement function for globbing paths if we would want to use globs in source
paths in the future.

CHANGELOG.rst

@@ -14,6 +14,9 @@ Versioning <http://semver.org/spec/v2.0.0.html>`_.
 Added
 -----
 
+- You can now compile all templates recursively within a directory. Just set
+  ``source`` to a directory path. ``target`` must be a directory as well, and
+  the relative file hierarchy is preserved.
 - The run action now supports ``timeout`` option, in order to set
   ``run_timeout`` on command-by-command basis.
 - ``compile`` actions now support an optional ``permissions`` field for

astrality/actions.py

@@ -20,20 +20,21 @@
 import abc
 from collections import defaultdict
 import logging
+from os.path import relpath
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import (
     Any,
     Callable,
     DefaultDict,
+    Dict,
     List,
     Optional,
     Set,
     Tuple,
     Union,
 )
 
-from jinja2.exceptions import TemplateNotFound
 from mypy_extensions import TypedDict
 
 from astrality import compiler, utils
@@ -198,43 +199,70 @@ def __init__(self, *args, **kwargs) -> None:
         self._performed_compilations: DefaultDict[Path, Set[Path]] = \
             defaultdict(set)
 
-    def execute(self) -> Optional[Tuple[Path, Path]]:
+    def execute(self) -> Dict[Path, Path]:
         """
         Compile template to target destination.
 
-        :return: 2-Tuple containing template and compile target.
+        :return: Dictionary with template keys and compile target values.
         """
         if self.null_object:
             # Null objects do nothing
-            return None
+            return {}
         elif 'target' not in self._options:
             # If no target is specified, then we can create a temporary file
             # and insert it into the configuration options.
             template = self.option(key='source', path=True)
             target = self._create_temp_file(template.name)
             self._options['target'] = str(target)  # type: ignore
 
-        template = self.option(key='source', path=True)
+        template_source = self.option(key='source', path=True)
         target = self.option(key='target', path=True)
 
-        try:
-            compiler.compile_template(
-                template=template,
-                target=target,
-                context=self.context_store,
-                shell_command_working_directory=self.directory,
-                permissions=self.option(key='permissions'),
+        compilations: Dict[Path, Path] = {}
+        if template_source.is_file():
+            # Single template file, so straight forward compilation
+            self.compile_template(template=template_source, target=target)
+            self._performed_compilations[template_source].add(target)
+            compilations = {template_source: target}
+
+        elif template_source.is_dir():
+            # The template source is a directory, so we will recurse over
+            # all the files and compile every single file while preserving
+            # the directory hierarchy
+            templates = tuple(
+                path
+                for path
+                in template_source.glob('**/*')
+                if path.is_file()
+            )
+            targets = tuple(
+                target / relpath(template_file, start=template_source)
+                for template_file
+                in templates
             )
-        except TemplateNotFound:
+
+            for template, target in zip(templates, targets):
+                self.compile_template(template=template, target=target)
+                self._performed_compilations[template].add(target)
+                compilations[template] = target
+
+        else:
             logger = logging.getLogger(__name__)
             logger.error(
-                'Could not compile template '
-                f'"{template}" to target "{target}". '
-                'Template does not exist.',
+                f'Could not compile template "{template_source}" '
+                f'to target "{target}". No such path!',
             )
 
-        self._performed_compilations[template].add(target)
-        return template, target
+        return compilations
+
+    def compile_template(self, template: Path, target: Path) -> None:
+        compiler.compile_template(
+            template=template,
+            target=target,
+            context=self.context_store,
+            shell_command_working_directory=self.directory,
+            permissions=self.option(key='permissions'),
+        )
 
     def performed_compilations(self) -> DefaultDict[Path, Set[Path]]:
         """

astrality/config.py

@@ -14,6 +14,7 @@
     List,
     Optional,
     Pattern,
+    Set,
     Tuple,
     Type,
     Iterable,
@@ -273,6 +274,32 @@ def expand_path(path: Path, config_directory: Path) -> Path:
     return path.resolve()
 
 
+def expand_globbed_path(path: Path, config_directory: Path) -> Set[Path]:
+    """
+    Expand globs, i.e. * and **, of path object.
+
+    This function is actually not used at the moment, but I have left it here
+    in case we would want to support globbed paths in the future.
+
+    :param path: Path to be expanded.
+    :param config_directory: Anchor for relative paths.
+    :return: Set of file paths resulting from glob expansion.
+    """
+    # Make relative paths absolute with config_directory as anchor
+    path = expand_path(path=path, config_directory=config_directory)
+
+    # Remove root directory from path
+    relative_to_root = Path(*path.parts[1:])
+
+    # Expand all globs in the entirety of `path`, recursing if ** is present
+    expanded_paths = Path('/').glob(
+        pattern=str(relative_to_root),
+    )
+
+    # Cast generator to set, and remove directories
+    return set(path for path in expanded_paths if path.is_file())
+
+
 class EnablingStatementRequired(TypedDict):
     """The required fields of a config item which enables a specific module."""
 

astrality/tests/actions/test_compile_action.py

@@ -3,6 +3,8 @@
 import os
 from pathlib import Path
 
+import pytest
+
 from astrality.actions import CompileAction
 
 def test_null_object_pattern():
@@ -14,7 +16,7 @@ def test_null_object_pattern():
         context_store={},
     )
     target = compile_action.execute()
-    assert target is None
+    assert target == {}
 
 
 def test_compilation_of_template_to_temporary_file(template_directory):
@@ -28,10 +30,11 @@ def test_compilation_of_template_to_temporary_file(template_directory):
         replacer=lambda x: x,
         context_store={},
     )
-    template, target = compile_action.execute()
+    compilations = compile_action.execute()
 
-    assert template == template_directory / 'no_context.template'
-    assert target.read_text() == 'one\ntwo\nthree'
+    template = template_directory / 'no_context.template'
+    assert template in compilations
+    assert compilations[template].read_text() == 'one\ntwo\nthree'
 
 def test_compilation_to_specific_absolute_file_path(template_directory, tmpdir):
     """
@@ -50,7 +53,7 @@ def test_compilation_to_specific_absolute_file_path(template_directory, tmpdir):
         replacer=lambda x: x,
         context_store={},
     )
-    _, return_target = compile_action.execute()
+    return_target = list(compile_action.execute().values())[0]
 
     assert return_target == target
     assert target.read_text() == 'one\ntwo\nthree'
@@ -72,7 +75,7 @@ def test_compilation_to_specific_relative_file_path(template_directory, tmpdir):
         replacer=lambda x: x,
         context_store={},
     )
-    _, return_target = compile_action.execute()
+    return_target = list(compile_action.execute().values())[0]
 
     assert return_target == target
     assert target.read_text() == 'one\ntwo\nthree'
@@ -96,13 +99,13 @@ def test_compilation_with_context(template_directory):
     )
 
     context_store['fonts'] = {2: 'ComicSans'}
-    _, target = compile_action.execute()
+    target = list(compile_action.execute().values())[0]
 
     username = os.environ.get('USER')
     assert target.read_text() == f'some text\n{username}\nComicSans'
 
     context_store['fonts'] = {2: 'TimesNewRoman'}
-    _, target = compile_action.execute()
+    target = list(compile_action.execute().values())[0]
     assert target.read_text() == f'some text\n{username}\nTimesNewRoman'
 
 def test_setting_permissions_of_target_template(template_directory):
@@ -118,7 +121,7 @@ def test_setting_permissions_of_target_template(template_directory):
         context_store={},
     )
 
-    _, target = compile_action.execute()
+    target = list(compile_action.execute().values())[0]
     assert (target.stat().st_mode & 0o777) == 0o707
 
 def test_use_of_replacer(template_directory, tmpdir):
@@ -148,7 +151,7 @@ def replacer(string: str) -> str:
         context_store={},
     )
 
-    _, target = compile_action.execute()
+    target = list(compile_action.execute().values())[0]
     assert target.read_text() == 'one\ntwo\nthree'
     assert (target.stat().st_mode & 0o777) == 0o777
 
@@ -167,7 +170,7 @@ def test_that_current_directory_is_set_correctly(template_directory):
         replacer=lambda x: x,
         context_store={},
     )
-    _, target = compile_action.execute()
+    target = list(compile_action.execute().values())[0]
     assert target.read_text() == '/tmp'
 
 def test_retrieving_all_compiled_templates(template_directory, tmpdir):
@@ -238,3 +241,58 @@ def test_contains_with_uncompiled_template(template_directory, tmpdir):
 
     compile_action.execute()
     assert template_directory / 'empty.template' in compile_action
+
+def test_compiling_entire_directory(test_config_directory, tmpdir):
+    """All directory contents should be recursively compiled."""
+    temp_dir = Path(tmpdir)
+    templates = \
+        test_config_directory / 'test_modules' / 'using_all_actions'
+    compile_dict = {
+        'source': str(templates),
+        'target': str(temp_dir),
+    }
+    compile_action = CompileAction(
+        options=compile_dict,
+        directory=test_config_directory,
+        replacer=lambda x: x,
+        context_store={'geography': {'capitol': 'Berlin'}},
+    )
+    results = compile_action.execute()
+
+    # Check if return content is correct, showing performed compilations
+    assert templates / 'module.template' in results
+    assert results[templates / 'module.template'] == \
+        temp_dir / 'module.template'
+
+    # Check if the templates actually have been compiled
+    target_dir_content = list(temp_dir.iterdir())
+    assert len(target_dir_content) == 6
+    assert temp_dir / 'module.template' in target_dir_content
+    assert (temp_dir / 'recursive' / 'empty.template').is_file()
+
+@pytest.mark.skip(reason='Glob paths have not been implemented yet')
+def test_compiling_entire_directory_with_single_glob(  # pragma: no cover
+    test_config_directory,
+    tmpdir,
+):  # pragma: no cover
+    """All directory content should be compilable with a glob."""
+    temp_dir = Path(tmpdir)
+    templates = test_config_directory / 'test_modules' / 'using_all_actions'
+    compile_dict = {
+        'source': str(templates / '*'),
+        'target': str(temp_dir),
+    }
+    compile_action = CompileAction(
+        options=compile_dict,
+        directory=test_config_directory,
+        replacer=lambda x: x,
+        context_store={},
+    )
+    results = compile_action.execute()
+
+    assert templates / 'module.template' in results
+    assert results[templates / 'module.template'] == temp_dir / 'module.template'
+
+    target_dir_content = list(temp_dir.iterdir())
+    assert len(target_dir_content) == 5
+    assert templates / 'module.template' in target_dir_content

astrality/tests/config/test_config.py

@@ -11,6 +11,7 @@
     dict_from_config_file,
     user_configuration,
     expand_path,
+    expand_globbed_path,
     insert_into,
     resolve_config_directory,
 )
@@ -174,6 +175,26 @@ def test_expand_path_method(test_config_directory):
         config_directory=test_config_directory,
     ) == test_config_directory / 'test'
 
+def test_expand_globbed_path(test_config_directory):
+    """Globbed paths should allow one level of globbing."""
+    templates = Path('test_modules', 'using_all_actions')
+    paths = expand_globbed_path(
+        path=templates / '*',
+        config_directory=test_config_directory,
+    )
+    assert len(paths) == 5
+    assert test_config_directory / templates / 'module.template' in paths
+
+def test_expand_recursive_globbed_path(test_config_directory):
+    """Globbed paths should allow recursive globbing."""
+    templates = Path('test_modules', 'using_all_actions')
+    paths = expand_globbed_path(
+        path=templates / '**' / '*',
+        config_directory=test_config_directory,
+    )
+    assert len(paths) == 6
+    assert test_config_directory / templates / 'recursive' / 'empty.template' in paths
+
 @pytest.yield_fixture
 def dir_with_compilable_files(tmpdir):
     config_dir = Path(tmpdir)

astrality/tests/module/test_module.py

@@ -349,7 +349,7 @@ def test_missing_template_file(
         caplog.clear()
         module_manager.finish_tasks()
         assert 'Could not compile template "/not/existing" '\
-               'to target "' in caplog.record_tuples[1][2]
+               'to target "' in caplog.record_tuples[0][2]
 
     def test_compilation_of_template(
         self,