AppManifest: add filter_files(), drop_ds_autogen()

- Add AppManifest methods to handle general file filtering and a specific
  deployment server (autogen) use case.
- Create unit tests, docs, ...

docs/source/api/ksconf.rst

@@ -116,6 +116,14 @@ ksconf.setup\_entrypoints module
    :undoc-members:
    :show-inheritance:
 
+ksconf.types module
+-------------------
+
+.. automodule:: ksconf.types
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 ksconf.version module
 ---------------------
 

docs/source/changelog.rst

@@ -23,12 +23,13 @@ Renames:
 - ``ksconf/commands/__init__.py`` -> ``ksconf/command.py``
 
 
-Ksconf v0.13.5 (DRAFT)
+Ksconf v0.13.5 (2023-10-19)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 *  Fix bug in `ksconf diff`` when using the ``--format=json`` option.
    This has been broken for a number of releases (added a unit test to keep that from repeating).
    Fixed the newly caused import issue caused by a missed file during the v0.13.4 release, as well as earlier Enum-related bug:  ``TypeError: Object of type DiffVerb is not JSON serializable``.
+*  :py:class:`~ksconf.app.manifest.AppManifest`: Add ``filter_files()`` method for generic filtering capabilities, and ``drop_ds_autogen()`` to handle the specific use case of removing that "autogenerated" file created by Deployment Server.
 
 
 Ksconf v0.13.4 (2023-10-13)

ksconf/app/manifest.py

@@ -143,6 +143,33 @@ def _calculate_hash(self) -> Optional[str]:
         h.update(payload.encode("utf-8"))
         return h.hexdigest()
 
+    def filter_files(self, filter: Callable[[AppManifestFile], bool]):
+        """ Apply a filter function to :py:attr:`files` safely.
+
+        Note that unlike the `filter_file` argument on :py:meth:`from_filesystem` and
+        :py:meth:`from_tarball`, the :py:obj:`filter` function is given an entire
+        :py:class:`AppManifestFile` object not just the file path.
+        """
+        if self._hash is UNSET:
+            self.files = [f for f in self.files if filter(f)]
+        else:
+            raise TypeError("Inappropriate use of filter_files().  "
+                            "This must be called before hash is calculated.")
+
+    def drop_ds_autogen(self):
+        """
+        Remove place-holder files created by the deployment server from the manifest
+        for the purpose of consistent hash creation.
+
+        These files always live in ``local/app.conf`` and contain the literal
+        text ``# Autogenerated file``.  Any other forms of this file are preserved.
+        """
+        local_app = PurePosixPath('local/app.conf')
+        unwanted_hashs = {
+            "a0c13b7008d9ef75d56be47cdb5ea3157f087bb7e773bd3d426a1998049e89b3"  # Nix - sha256
+        }
+        self.filter_files(lambda f: f.path != local_app or f.hash not in unwanted_hashs)
+
     @classmethod
     def from_dict(cls, data: dict) -> AppManifest:
         files = [AppManifestFile.from_dict(f) for f in data["files"]]

tests/test_app.py

@@ -6,8 +6,9 @@
 import os
 import sys
 import unittest
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from tarfile import TarFile
+from typing import Union
 
 # Allow interactive execution from CLI,  cd tests; ./test_app.py
 if __package__ is None:
@@ -16,7 +17,7 @@
 
 from ksconf.app import get_facts_manifest_from_archive
 from ksconf.app.facts import AppFacts
-from ksconf.app.manifest import AppManifest, StoredArchiveManifest
+from ksconf.app.manifest import AppManifest, AppManifestFile, StoredArchiveManifest
 from tests.cli_helper import TestWorkDir, static_data
 
 """
@@ -26,6 +27,15 @@
 """
 
 
+def hash_string(content: Union[bytes, str], algorithm, encoding="utf-8"):
+    import hashlib
+    h = hashlib.new(algorithm)
+    if isinstance(content, str):
+        content = content.encode(encoding)
+    h.update(content)
+    return h.hexdigest()
+
+
 class AppTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -89,10 +99,28 @@ def test_filesystem_manifest_with_filter(self):
     def test_the_do_it_all_function(self):
         tarball_path = static_data("apps/modsecurity-add-on-for-splunk_12.tgz")
         info, manifest = get_facts_manifest_from_archive(tarball_path)
-
         self.assertEqual(info.name, "Splunk_TA_modsecurity")
         self.assertEqual(manifest.hash, "7f9e7b63ed13befe24b12715b1e1e9202dc1186266497aad0b723fe27ca1de12")
 
+        # Reset/clear hash
+        del manifest.hash
+
+        # Add a bogus "Deployment Server" autogen file and then remove it using drop_ds_autogen()
+        content = "# Autogenerated file "
+        manifest.files.append(AppManifestFile(
+            PurePosixPath('local/app.conf'), mode=0o644,
+            hash=hash_string(content, manifest.hash_algorithm), size=len(content)))
+
+        # Size should increase
+        self.assertEqual(len(manifest.files), 16)
+        # Should find 1 local file
+        self.assertEqual(len(list(manifest.find_local())), 1)
+
+        manifest.drop_ds_autogen()
+        self.assertEqual(len(manifest.files), 15)
+        # Confirm that hash is back to what it was...
+        self.assertEqual(manifest.hash, "7f9e7b63ed13befe24b12715b1e1e9202dc1186266497aad0b723fe27ca1de12")
+
         # No local files
         self.assertEqual(len(list(manifest.find_local())), 0)
 

tests/test_cli_combine.py

@@ -230,7 +230,7 @@ def test_keep_existing_ds_local_app(self):
             self.assertFalse(os.path.isdir(twd.get_path("etc/deployment-apps/Splunk_TA_nix/local")))
 
         # Simulate a 'splunk reload deploy-server'
-        twd.write_file("etc/deployment-apps/Splunk_TA_nix/local/app.conf", "# Autogenerated file")
+        twd.write_file("etc/deployment-apps/Splunk_TA_nix/local/app.conf", "# Autogenerated file ")
 
         with ksconf_cli:
             ko = ksconf_cli("combine", "--keep-existing", "local/app.conf",
@@ -241,7 +241,7 @@ def test_keep_existing_ds_local_app(self):
             self.assertEqual(cfg["launcher"]["version"], "7.0.0")
 
             self.assertEqual(twd.read_file("etc/deployment-apps/Splunk_TA_nix/local/app.conf"),
-                             "# Autogenerated file")
+                             "# Autogenerated file ")
 
             # This time the file will be removed
             ko = ksconf_cli("combine", "--target", target, src)