#66: Improve ArchivePackagingBaseTask - allow configuration and exe…

…cution stage usage, extract .gitignore support into `IOBaseTask`, cover with tests, fix gitignore support

src/core/rkd/core/standardlib/io.py

@@ -1,4 +1,5 @@
 import os
+from abc import ABC
 from argparse import ArgumentParser
 from tarfile import TarFile
 from typing import Dict, Union, Callable, List
@@ -11,7 +12,65 @@
 ARCHIVE_TYPE_TARGZ = 'tar+gzip'
 
 
-class ArchivePackagingBaseTask(ExtendableTaskInterface):
+class IOBaseTask(ExtendableTaskInterface, ABC):
+    _ignore: List[Callable]
+
+    def __init__(self):
+        self._ignore = [lambda x: False]
+
+    def consider_ignore(self, path: str = '.gitignore'):
+        """
+        Load ignore rules from .gitignore or other file in gitignore format
+
+        :api: configure
+        :param path:
+        :return:
+        """
+
+        if not os.path.isfile(path):
+            raise FileNotFoundError(f'Cannot find .gitignore at path "{path}"')
+
+        func = parse_gitignore(path)
+        func.path = path
+
+        self.io().debug(f'Loaded ignore from "{path}"')
+        self._ignore.append(func)
+
+    def consider_ignore_recursively(self, src_path: str, filename: str = '.gitignore'):
+        """
+        Recursively load rules from a gitignore-format file
+
+        :param src_path:
+        :param filename:
+        :return:
+        """
+
+        for root, d_names, f_names in os.walk(src_path):
+            if os.path.isfile(root + '/' + filename):
+                self.consider_ignore(root + '/' + filename)
+
+    def can_be_touched(self, path: str) -> bool:
+        """
+        Can file be touched? - Added to archive, copied, deleted etc.
+
+        :param path:
+        :return:
+        """
+
+        for callback in self._ignore:
+            if callback(path):
+                try:
+                    # noinspection PyUnresolvedReferences
+                    self.io().debug(f'can_be_touched({callback.path}) = blocked adding {path}')
+                except AttributeError:
+                    pass
+
+                return False
+
+        return True
+
+
+class ArchivePackagingBaseTask(IOBaseTask):
     """
     Packages files into a compressed archive.
     -----------------------------------------
@@ -20,9 +79,10 @@ class ArchivePackagingBaseTask(ExtendableTaskInterface):
       - dry-run mode (do not write anything to disk, just print messages)
       - copies directories recursively
       - .gitignore files support (manually added using API method)
+      - can work both as preconfigured and fully on runtime
 
 
-    Example:
+    Example (preconfigured):
 
         .. code:: python
 
@@ -36,10 +96,25 @@ def configure(task: ArchivePackagingBaseTask, event: ConfigurationLifecycleEvent
                 return [configure]
 
 
+    Example (on runtime):
+
+        .. code:: python
+
+            @extends(ArchivePackagingBaseTask)
+            def PackIntoZipTask():
+                def configure(task: ArchivePackagingBaseTask, event: ConfigurationLifecycleEvent):
+                    task.archive_path = '/tmp/test-archive.zip'
+
+                def execute(task: ArchivePackagingBaseTask):
+                    task.consider_gitignore('.gitignore')
+                    task.add('tests/samples/', './')
+                    task.perform()
+
+                return [configure, execute]
+
     """
 
     sources: Dict[str, str]
-    _gitignore: List[Callable]
 
     dry_run: bool                     # Skip IO operations, just print messages
     allow_archive_overwriting: bool   # Allow overwriting if destination file already exists
@@ -49,15 +124,15 @@ def configure(task: ArchivePackagingBaseTask, event: ConfigurationLifecycleEvent
     archive_type: str  # One of supported types (see io.ARCHIVE_TYPE_ZIP and io.ARCHIVE_TYPE_TARGZ), defaults to zip
 
     def __init__(self):
+        super().__init__()
         self.archive_type = ARCHIVE_TYPE_ZIP
         self.sources = {}
-        self._gitignore = [lambda x: True]
         self.dry_run = False
 
     def get_configuration_attributes(self) -> List[str]:
         return [
             'archive_path', 'archive_type', 'sources', 'dry_run',
-            'allow_archive_overwriting', 'add', 'consider_gitignore'
+            'allow_archive_overwriting', 'add', 'consider_ignore', 'consider_ignore_recursively'
         ]
 
     def get_name(self) -> str:
@@ -97,7 +172,7 @@ def _add(self, root, f, target_path, src_path, include_src_last_dir):
         current_file_path = os.path.abspath(os.path.join(root, f))
 
         try:
-            if not self._can_be_added(current_file_path):
+            if not self.can_be_touched(current_file_path):
                 self.io().info(f'Ignoring "{current_file_path}"')
                 return
 
@@ -121,20 +196,7 @@ def _add(self, root, f, target_path, src_path, include_src_last_dir):
                 current_file_target_path = os.path.basename(src_path) + '/' + current_file_target_path
 
         self.sources[current_file_target_path] = current_file_path
-
-    def consider_gitignore(self, path: str = '.gitignore'):
-        """
-        Load ignore rules from .gitignore
-
-        :api: configure
-        :param path:
-        :return:
-        """
-
-        if not os.path.isfile(path):
-            raise FileNotFoundError(f'Cannot find .gitignore at path "{path}"')
-
-        self._gitignore.append(parse_gitignore(path))
+        self.io().debug(f'Adding {current_file_path} as {current_file_target_path}')
 
     def execute(self, context: ExecutionContext) -> bool:
         self.dry_run = bool(context.get_arg('--dry-run'))
@@ -143,6 +205,11 @@ def execute(self, context: ExecutionContext) -> bool:
         if self.dry_run:
             self.io().warn('Dry run active, will not perform any disk operation')
 
+        self.perform(context)
+
+        return True
+
+    def perform(self, context: ExecutionContext):
         # prepare
         self._make_sure_destination_directory_exists(os.path.dirname(self.archive_path))
         self.archive = self._create_archive(self.archive_path)
@@ -153,15 +220,6 @@ def execute(self, context: ExecutionContext) -> bool:
         self.inner_execute(context)
         self._commit_changes(self.archive)
 
-        return True
-
-    def _can_be_added(self, path: str) -> bool:
-        for callback in self._gitignore:
-            if not callback(path):
-                return False
-
-        return True
-
     def _add_to_archive(self, archive: Union[ZipFile, TarFile], path: str, dest_path: str):
         self.io().info(f'Compressing "{path}" -> "{dest_path}"')
 

src/core/tests/test_standardlib_io_archivepackagingtask.py

@@ -1,5 +1,7 @@
 import os
+import subprocess
 import pytest
+from tempfile import TemporaryDirectory
 from rkd.core.api.inputoutput import BufferedSystemIO
 from rkd.core.api.temp import TempManager
 from rkd.core.api.testing import FunctionalTestingCase
@@ -104,3 +106,154 @@ def test_archive_overwrite_is_not_allowed_when_switch_not_used(self):
 
         finally:
             temp.finally_clean_up()
+
+    def test_slash_in_src_at_end_results_in_not_including_last_src_directory(self):
+        io = BufferedSystemIO()
+        task = ArchivePackagingBaseTask()
+        self.satisfy_task_dependencies(task, io=io)
+
+        temp = TempManager()
+        archive_path = temp.create_tmp_file_path()[0]
+
+        task.archive_path = archive_path
+        task.archive_type = 'zip'
+        task.add(os.path.dirname(TEST_PATH) + '/internal-samples/', 'examples')
+
+        task.execute(self.mock_execution_context(
+            task,
+            {
+                "--dry-run": True,
+                "--allow-overwrite": False
+            },
+            {}
+        ))
+
+        self.assertIn('internal-samples/jinja2/example.j2" -> "examples/jinja2/example.j2"', io.get_value())
+
+    def test_not_having_slash_at_end_of_src_path_results_in_including_last_folder_name(self):
+        io = BufferedSystemIO()
+        task = ArchivePackagingBaseTask()
+        self.satisfy_task_dependencies(task, io=io)
+
+        temp = TempManager()
+        archive_path = temp.create_tmp_file_path()[0]
+
+        task.archive_path = archive_path
+        task.archive_type = 'zip'
+        task.add(os.path.dirname(TEST_PATH) + '/internal-samples', 'examples')
+
+        task.execute(self.mock_execution_context(
+            task,
+            {
+                "--dry-run": True,
+                "--allow-overwrite": False
+            },
+            {}
+        ))
+
+        self.assertIn('internal-samples/jinja2/example.j2" -> "internal-samples/examples/jinja2/example.j2"',
+                      io.get_value())
+
+    def test_selected_gitignore_is_considered(self):
+        """
+        Create 3 files - .gitignore, to-be-ignored.txt, example.py
+        Add to-be-ignored.txt to be ignored
+        Schedule a copy of whole directory containing those 3 files
+
+        Expected:
+            To copy: .gitignore, example.py
+            To ignore: to-be-ignored.txt
+
+        :return:
+        """
+
+        io = BufferedSystemIO()
+        io.set_log_level('debug')
+
+        # prepare task
+        task = ArchivePackagingBaseTask()
+        self.satisfy_task_dependencies(task, io=io)
+
+        # prepare workspace
+        with TemporaryDirectory() as sources_path:
+            with open(sources_path + '/.gitignore', 'w') as gitignore:
+                gitignore.write("to-be-ignored.txt\n")
+
+            with open(sources_path + '/to-be-ignored.txt', 'w') as f:
+                f.write("test\n")
+
+            with open(sources_path + '/example.py', 'w') as f:
+                f.write("#!/usr/bin/env python3\nprint('Hello anarchism!')\n")
+
+            # consider our gitignore that will ignore "to-be-ignored.txt"
+            task.consider_ignore(sources_path + '/.gitignore')
+
+            # add a batch of files
+            task.add(sources_path)
+
+        self.assertNotRegex(io.get_value(), f'Ignoring "(.*)/example.py"')
+        self.assertRegex(io.get_value(), f'Ignoring "(.*)/to-be-ignored.txt"')
+
+    def test_consider_ignore_recursively(self):
+        """
+        Test that consider_ignore_recursively() method loads .gitignore files recursively
+
+        Directory structure:
+            - /.gitignore
+            - /subdir1/second-to-be-ignored.c
+            - /subdir1/subdir2/to-be-ignored.txt
+            - /subdir1/subdir2/.gitignore
+            - /example.py
+
+        :return:
+        """
+
+        io = BufferedSystemIO()
+        io.set_log_level('debug')
+
+        # prepare task
+        task = ArchivePackagingBaseTask()
+        self.satisfy_task_dependencies(task, io=io)
+
+        # prepare workspace
+        with TemporaryDirectory() as sources_path:
+            # create a subdirectory, for the depth
+            subprocess.check_call(['mkdir', '-p', sources_path + '/subdir1/subdir2'])
+
+            with open(sources_path + '/.gitignore', 'w') as gitignore:
+                gitignore.write("subdir1/second-to-be-ignored.c\n")
+
+            with open(sources_path + '/subdir1/subdir2/.gitignore', 'w') as gitignore:
+                gitignore.write("to-be-ignored.txt\n")
+
+            with open(sources_path + '/subdir1/subdir2/to-be-ignored.txt', 'w') as f:
+                f.write("test\n")
+
+            with open(sources_path + '/subdir1/second-to-be-ignored.c', 'w') as f:
+                f.write("test\n")
+
+            with open(sources_path + '/example.py', 'w') as f:
+                f.write("#!/usr/bin/env python3\nprint('Hello anarchism!')\n")
+
+            task.consider_ignore_recursively(sources_path, filename='.gitignore')
+            task.add(sources_path)
+
+        self.assertRegex(io.get_value(), 'Ignoring "(.*)/subdir1/second-to-be-ignored.c"')
+        self.assertRegex(io.get_value(), 'Ignoring "(.*)/subdir1/subdir2/to-be-ignored.txt"')
+
+        what_should_be = [
+            sources_path + '/.gitignore',
+            sources_path + '/subdir1/subdir2/.gitignore',
+            sources_path + '/example.py'
+        ]
+
+        what_should_NOT_be = [
+            sources_path + '/subdir1/subdir2/to-be-ignored.txt',
+            sources_path + '/subdir1/second-to-be-ignored.c'
+        ]
+
+        for item_should_be in what_should_be:
+            self.assertIn(item_should_be, task.sources)
+
+        for item_should_not_be in what_should_NOT_be:
+            self.assertNotIn(item_should_not_be, task.sources)

src/php/tests/samples/.rkd/makefile.py

@@ -43,7 +43,7 @@ def PackIntoZipTask():
 
     def configure(task: ArchivePackagingBaseTask, event: ConfigurationLifecycleEvent):
         task.archive_path = '/tmp/test-archive.zip'
-        task.consider_gitignore('.gitignore')
+        task.consider_ignore('.gitignore')
         task.add('tests/samples/', './')
 
     return [configure]