More unittest tweaks

- Up to 80 test cases and over 95% code coverage. Enabled a bunch of little test tweaks and fresh tests. - Enabled 'argparse' on one of the travis build systems to vary some behavior based on the present/absence of that library. - Created a new test_util.py script for testing internal bits and utils.
Kintyre · Jan 25, 2019 · 813df19 · 813df19
1 parent 5a2aca3
commit 813df19
Show file tree

Hide file tree

Showing 16 changed files with 131 additions and 39 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -14,6 +14,7 @@ matrix:
     python: '3.4'
   - os: linux
     python: '3.5'
+    env: EXTRA_PKGS="argcomplete"
   - os: linux
     python: '3.6'
     env: BUILD_SDIST=true
@@ -56,7 +57,7 @@ install:
         # Since pyenv can be cached across OSX builds, uninstall first for a fresh build
         if pip show kintyre-splunk-conf; then pip uninstall -y kintyre-splunk-conf; fi
     fi
-  - pip install coverage
+  - pip install coverage $EXTRA_PKGS
   - pip install coveralls
   - pip install codecov
   - pip install .

diff --git a/ksconf/commands/combine.py b/ksconf/commands/combine.py
@@ -162,7 +162,7 @@ def run(self, args):
             self.stderr.write("Must provide the '--target' directory.\n")
             return EXIT_CODE_MISSING_ARG
 
-            self.stderr.write("Combining conf files into directory {}\n".format(args.target))
+        self.stderr.write("Combining conf files into directory {}\n".format(args.target))
         args.source = list(_expand_glob_list(args.source))
         for src in args.source:
             self.stderr.write("Reading conf files from directory {}\n".format(src))

diff --git a/ksconf/commands/filter.py b/ksconf/commands/filter.py
@@ -41,9 +41,8 @@ def _feed_from_file(path):
             for line in f:
                 line = line.rstrip()
                 # Skip empty or "comment" lines
-                if not line or line[0] == "#":
-                    continue
-                items.append(line)
+                if line and line[0] != "#":
+                    items.append(line)
         sys.stderr.write("Loaded patterns from {}.  Found {} entries.\n".format(path, len(items)))
         return items
 
@@ -57,7 +56,7 @@ def feed(self, item):
         # New items added.  Mark prep-work as incomplete
         self._prep = False
 
-    def _pre_match(self):
+    def _pre_match(self):  # pragma: no cover
         pass
 
     def match(self, item):
@@ -75,12 +74,13 @@ def match(self, item):
     def has_rules(self):
         return len(self.data) > 0
 
-    def _match(self, item):
+    def _match(self, item):  # pragma: no cover
         raise NotImplementedError
 
 
 class FilteredListString(FilteredList):
     """ Handle simple string comparisons """
+
     def _pre_match(self):
         if self.flags & self.IGNORECASE:
             # Lower-case all strings in self.data.  (Only need to do this once)
@@ -100,7 +100,7 @@ def _pre_match(self):
             re_flags |= re.IGNORECASE
         # Compile all regular expressions
         # XXX: Add better error handling here for friendlier user feedback
-        self.data = [ re.compile(pattern, re_flags) for pattern in self.data ]
+        self.data = [re.compile(pattern, re_flags) for pattern in self.data]
 
     def _match(self, item):
         for pattern_re in self.data:
@@ -116,7 +116,7 @@ class FilterListWildcard(FilteredListRegex):
 
     def _pre_match(self):
         # Use fnmatch to translate wildcard expression to a regex
-        self.data = [ fnmatch.translate(pat) for pat in self.data ]
+        self.data = [fnmatch.translate(pat) for pat in self.data]
         # Now call regex (parent version)
         super(FilterListWildcard, self)._pre_match()
 
@@ -128,11 +128,10 @@ def create_filtered_list(match_mode, flags):
         return FilterListWildcard(flags)
     elif match_mode == "regex":
         return FilteredListRegex(flags)
-    else:
+    else:   # pragma: no cover
         raise ValueError("Unknown matching mode {!r}".format(match_mode))
 
 
-
 class FilterCmd(KsconfCmd):
     help = "A stanza-aware GREP tool for conf files"
     description = dedent("""
@@ -145,7 +144,6 @@ class FilterCmd(KsconfCmd):
     # format = "manual"
     maturity = "alpha"
 
-
     def __init__(self, *args, **kwargs):
         super(FilterCmd, self).__init__(*args, **kwargs)
         self.stanza_filters = None
@@ -181,7 +179,6 @@ def register_args(self, parser):
             This can be used to show what content does NOT match,
             or make a backup copy of excluded content.""")
 
-
         pg_out = parser.add_argument_group("Output mode", """
             Select an alternate output mode.
             If any of the following options are used, the stanza output is not shown.
@@ -191,7 +188,6 @@ def register_args(self, parser):
         pg_out.add_argument("--count", "-c", action="store_true",
                             help="Count matching stanzas")
 
-
         pg_sel = parser.add_argument_group("Stanza selection", """
             Include or exclude entire stanzas using these filter options.
 

diff --git a/ksconf/util/__init__.py b/ksconf/util/__init__.py
@@ -38,7 +38,8 @@ def wrapper(*args, **kwargs):
                 return rv
         return wrapper
 
-def debug_traceback():
+
+def debug_traceback():  # pragma: no cover
     """ If the 'KSCONF_DEBUG' environmental variable is set, then show a stack trace. """
     from os import environ
     if b"KSCONF_DEBUG" in environ:

diff --git a/ksconf/util/completers.py b/ksconf/util/completers.py
@@ -4,7 +4,7 @@
 try:
     from argcomplete import autocomplete
     from argcomplete.completers import FilesCompleter, DirectoriesCompleter
-except ImportError:  # pragma: no cover
+except ImportError:
 
     def _argcomplete_noop(*args, **kwargs):
         del args, kwargs

diff --git a/ksconf/vc/git.py b/ksconf/vc/git.py
@@ -14,10 +14,9 @@ def git_cmd(args, shell=False, cwd=None, capture_std=True, encoding="utf-8"):
     if isinstance(args, tuple):
         args = list(args)
     cmdline_args = [GIT_BIN] + args
+    out = None
     if capture_std:
         out = PIPE
-    else:
-        out = None
     proc = Popen(cmdline_args, stdout=out, stderr=out, shell=shell, cwd=cwd)
     (stdout, stderr) = proc.communicate()
     if hasattr(stdout, "decode"):

diff --git a/run_tests.py b/run_tests.py
@@ -1,10 +1,8 @@
 #!/usr/bin/env python
 # May eventually switch to some other mechanism (possibly nose) but for now this works
 
-# Note:  TestLoader.discover() is new in Python 2.7 (won't work in 2.6, if we care)
+from __future__ import absolute_import, unicode_literals
 
-from __future__ import absolute_import
-from __future__ import unicode_literals
 import os
 import sys
 import unittest
@@ -22,9 +20,9 @@ def run_all():
     suite = loader.discover("tests")
     runner = unittest.TextTestRunner()
     results = runner.run(suite)
-    if results.errors:
+    if results.errors:  # pragma: no cover
         return 2
-    elif results.failures:
+    elif results.failures:  # pragma: no cover
         return 1
 
 

diff --git a/tests/cli_helper.py b/tests/cli_helper.py
@@ -5,7 +5,6 @@
 import stat
 import sys
 import tempfile
-from collections import namedtuple
 from io import open, StringIO
 from subprocess import list2cmdline
 from textwrap import dedent
@@ -14,6 +13,7 @@
 
 # Some unittest fixup for various python versions
 import tests.compat as _
+
 del _
 
 from ksconf.__main__ import cli
@@ -54,6 +54,7 @@ def static_data(path):
     parts = path.split("/")
     return os.path.abspath(os.path.join(os.path.dirname(__file__), "data", *parts))
 
+
 def parse_string(text, profile=None, **kwargs):
     text = dedent(text)
     f = StringIO(text)
@@ -77,7 +78,6 @@ def ksconf_exec(args):
 '''
 
 
-
 class _KsconfCli():
     """
     CLI Wrapper context management class for unit testing;
@@ -120,7 +120,7 @@ def __call__(self, *args):
             temp_stderr = sys.stderr = StringIO()
             try:
                 rc = cli(args, _unittest=True)
-            except SystemExit as e:
+            except SystemExit as e: # pragma: no cover
                 if hasattr(e, "code"):  # PY3
                     rc = e.code
                 else:
@@ -198,16 +198,17 @@ def clean(self, force=False):
         if not hasattr(self, "_path"):
             return
 
-        if "KSCONF_KEEP_TEST_FILES" in os.environ and not force:
+        if "KSCONF_KEEP_TEST_FILES" in os.environ and not force:  # pragma: no cover
             return
 
         # Remove read-only file handler (e.g. clean .git/objects/xx/* files on Windows)
-        def del_rw(action, name, exc):
+        def del_rw(action, name, exc):  # pragma: no cover (infrequently used)
             # https://stackoverflow.com/a/21263493/315892
             # Not checking for file vs dir, ...
             os.chmod(name, stat.S_IWRITE)
             os.remove(name)
             del action, exc
+
         shutil.rmtree(self._path, onerror=del_rw)
         # Prevent the class from being used further
         del self._path

diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -44,13 +44,5 @@ def test_conffileproxy_invalid_arg(self):
                 self.assertRegex(ko.stderr, ".*(failed to parse|invalid ConfFileType).*")
 
 
-    def test_entrypoints(self):
-        from ksconf.commands import get_entrypoints, _get_fallback
-        get_entrypoints("ksconf_cmd", "sort")
-
-        # Just to exercise this (coverage and prevent regressions)
-        _get_fallback("ksconf_cmd")
-
-
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()
diff --git a/tests/test_cli_combine.py b/tests/test_cli_combine.py
@@ -74,6 +74,7 @@ def test_combine_3dir(self):
         """)
         default = twd.get_path("etc/apps/Splunk_TA_aws/default")
         with ksconf_cli:
+            ko = ksconf_cli("combine", "--dry-run", "--target", default, default + ".d/*")
             ko = ksconf_cli("combine", "--target", default, default + ".d/*")
             self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
             cfg = parse_conf(twd.get_path("etc/apps/Splunk_TA_aws/default/props.conf"))
@@ -83,6 +84,7 @@ def test_combine_3dir(self):
             self.assertEqual(cfg["aws:config"]["TRUNCATE"], '9999999')
             nav_content = twd.read_file("etc/apps/Splunk_TA_aws/default/data/ui/nav/default.xml")
             self.assertIn("My custom view", nav_content)
+
         twd.write_conf("etc/apps/Splunk_TA_aws/default.d/99-theforce/props.conf", {
             "aws:config": {"TIME_FORMAT": "%Y-%m-%dT%H:%M:%S.%6NZ"}
         })
@@ -94,6 +96,13 @@ def test_combine_3dir(self):
         </nav>
         """)
         twd.write_file("etc/apps/Splunk_TA_aws/default/data/dead.conf", "# File to remove")
+        twd.write_file("etc/apps/Splunk_TA_aws/default/data/tags.conf", "# Locally created file")
+
+        twd.write_file("etc/apps/Splunk_TA_aws/default.d/99-blah/same.txt", "SAME TEXT")
+        twd.write_file("etc/apps/Splunk_TA_aws/default/same.txt", "SAME TEXT")
+
+        twd.write_file("etc/apps/Splunk_TA_aws/default.d/99-blah/binary.bin", b"#BINARY \xff \x00")
+        twd.write_file("etc/apps/Splunk_TA_aws/default/binary.bin", b"#BINARY NEW \x00 \xff \xFB")
         with ksconf_cli:
             ko = ksconf_cli("combine", "--dry-run", "--target", default, default + ".d/*")
             self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
@@ -107,6 +116,14 @@ def test_require_arg(self):
             ko = ksconf_cli("combine", "source-dir")
             self.assertRegex(ko.stderr, "Must provide [^\r\n]+--target")
 
+    def test_missing_combine_dir(self):
+        twd = TestWorkDir()
+        twd.write_file("source-dir/someapp/default/blah.conf", "[entry]\nboring=yes\n")
+        twd.write_file("dest-dir/someapp/default//blah.conf", "[entry]\nboring=yes\n")
+
+        ko = ksconf_cli("combine", twd.get_path("source-dir"), "--target", twd.get_path("dest-dir"))
+        self.assertEqual(ko.returncode, EXIT_CODE_COMBINE_MARKER_MISSING)
+        self.assertRegex(ko.stderr, r".*Marker file missing\b.*")
 
 
 if __name__ == '__main__':  # pragma: no cover

diff --git a/tests/test_cli_filter.py b/tests/test_cli_filter.py
@@ -12,6 +12,7 @@
 
 from ksconf.consts import *
 from tests.cli_helper import *
+from ksconf.conf.parser import PARSECONF_STRICT
 
 class CliKsconfFilter(unittest.TestCase):
 
@@ -237,7 +238,6 @@ def test_invert_full_circle(self):
         conf = self.sample01
         matched = self.twd.get_path("output-match.conf")
         rejected = self.twd.get_path("output-rejected.conf")
-
         with ksconf_cli:
             # Simple direct stanza match (saved to an output file)
             ko = ksconf_cli("filter", conf, "--stanza", "Errors in the last hour",
@@ -251,6 +251,7 @@ def test_invert_full_circle(self):
                             "--invert-match", "--output", rejected)
             self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
             out = self.twd.read_conf("output-rejected.conf")
+            self.twd.read_file("output-match.conf", as_bytes=True)  # Boost code coverage
             self.assertEqual(len(out), 3)
 
             # Combine 2 output files, then compare to original
@@ -268,7 +269,7 @@ def test_conf_via_stdin(self):
             with ksconf_cli:
                 ko = ksconf_cli("filter", "-", "--stanza", "Errors in the last hour")
                 self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
-                out = ko.get_conf()
+                out = ko.get_conf(PARSECONF_STRICT)     # explict profile to boost coverage
                 self.assertIn("Errors in the last hour", out)
 
     def test_has_attr(self):

diff --git a/tests/test_cli_merge.py b/tests/test_cli_merge.py
@@ -53,5 +53,19 @@ def test_merge_dry_run(self):
             self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
             self.assertRegex(ko.stdout, r"^$")
 
+    def test_magic_stanza_drop(self):
+        twd = TestWorkDir()
+        conf1 = twd.copy_static("inputs-ta-nix-local.conf", "inputs.conf")
+        conf2 = twd.write_file("inputs2.conf", """
+        [script://./bin/ps.sh]
+        _stanza = <<DROP>>
+        """)
+        with ksconf_cli:
+            ko = ksconf_cli("merge", conf1, conf2)
+            self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
+            conf = ko.get_conf()
+            self.assertNotIn("script://./bin/ps.sh", conf)
+
+
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()
diff --git a/tests/test_cli_snapshot.py b/tests/test_cli_snapshot.py
@@ -41,7 +41,12 @@ def test_app_simple(self):
         [ui]
         is_visible = false
         """)
-        twd.write_file("apps/MyApp/metadata/default.conf", """\
+        twd.write_file("apps/MyApp/local/props.conf", """\
+        # very bad idea...
+        SHOULD_LINEMERGE = true
+        """)
+
+        twd.write_file("apps/MyApp/metadata/local.meta", """\
         []
         export = system
         """)
@@ -90,6 +95,14 @@ def test_bad_conf_file(self):
             self.assertRegex(ko.stdout, r"\"failure\"\s*:\s*\"")
             json.loads(ko.stdout)
 
+    def test_missing_conf_file(self):
+        twd = TestWorkDir()
+        with ksconf_cli:
+            ko = ksconf_cli("snapshot", twd.get_path("not/a/file.conf"))
+            self.assertEqual(ko.returncode, EXIT_CODE_NO_SUCH_FILE)
+            self.assertRegex(ko.stderr, r"No such file")
+
+
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()
diff --git a/tests/test_cli_sort.py b/tests/test_cli_sort.py
@@ -82,6 +82,11 @@ def test_sort_glob(self):
             self.assertRegex(ko.stderr, r"badfile\.conf")
     '''
 
+    def test_bad_file(self):
+        with ksconf_cli:
+            ko = ksconf_cli("sort", self.conf_bad)
+            self.assertEqual(ko.returncode, EXIT_CODE_BAD_CONF_FILE)
+
     def test_sort_mixed(self):
         # Not yet implemented.  Currently relying on the shell to do this.
         with ksconf_cli: