Add '--ignore-missing' to merge

- Allow 'ksconf merge' to silently ignore missing input files, which comes in
  quite handy when using bashisms like MyApp/{default,local}/props.conf
  This closes #42.
- Added unit tests to confirm that (1) missing files prevent execution by
  default, (2) that --ignore-missing allows silent skipping, and (3) that any
  conf parsing error is reported as an error (not an unhandled exception).
- Update the core cli test to use a different command (other than merge)
  that use ConfFileType() with mode of 'load' (picked rest-export, for now)

NOTE:  We need to find a better replacement for ConfFileType() since *most* of
the time we can't use its built-in helper features because we need to modify
its behavior from another CLI argument.  It was an interesting integration
technique but time to look for a better way.

docs/source/dyn/cli.rst

@@ -308,7 +308,8 @@ ksconf merge
 
  .. code-block:: none
 
-    usage: ksconf merge [-h] [--target FILE] [--dry-run] [--banner BANNER]
+    usage: ksconf merge [-h] [--target FILE] [--ignore-missing] [--dry-run]
+                        [--banner BANNER]
                         FILE [FILE ...]
     
     Merge two or more .conf files into a single combined .conf file. This is
@@ -325,6 +326,7 @@ ksconf merge
                             Save the merged configuration files to this target
                             file. If not provided, the merged conf is written to
                             standard output.
+      --ignore-missing, -s  Silently ignore any missing CONF files.
       --dry-run, -D         Enable dry-run mode. Instead of writing to TARGET,
                             preview changes in 'diff' format. If TARGET doesn't
                             exist, then show the merged file.

docs/source/dyn/ksconf_subcommands.csv

@@ -2,11 +2,11 @@
 :ref:`ksconf combine <ksconf_cmd_combine>`,beta,Combine configuration files across multiple source directories into a single destination directory.  This allows for an arbitrary number of splunk configuration layers to coexist within a single app.  Useful in both ongoing merge and one-time ad-hoc use. 
 :ref:`ksconf diff <ksconf_cmd_diff>`,stable,Compare settings differences between two .conf files ignoring spacing and sort order
 :ref:`ksconf filter <ksconf_cmd_filter>`,alpha,A stanza-aware GREP tool for conf files
+:ref:`ksconf promote <ksconf_cmd_promote>`,beta,"Promote .conf settings between layers using either either in batch mode (all changes) or interactive mode.  Frequently this is used to promote conf changes made via the UI (stored in the ``local`` folder) to a version-controlled directory, often ``default``. "
 :ref:`ksconf merge <ksconf_cmd_merge>`,stable,Merge two or more .conf files
 :ref:`ksconf minimize <ksconf_cmd_minimize>`,beta,Minimize the target file by removing entries duplicated in the default conf(s)
-:ref:`ksconf promote <ksconf_cmd_promote>`,beta,"Promote .conf settings between layers using either either in batch mode (all changes) or interactive mode.  Frequently this is used to promote conf changes made via the UI (stored in the ``local`` folder) to a version-controlled directory, often ``default``. "
-:ref:`ksconf rest-export <ksconf_cmd_rest-export>`,beta,Export .conf settings as a curl script to apply to a Splunk instance later (via REST)
-:ref:`ksconf rest-publish <ksconf_cmd_rest-publish>`,alpha,Publish .conf settings to a live Splunk instance via REST
 :ref:`ksconf snapshot <ksconf_cmd_snapshot>`,alpha,Snapshot .conf file directories into a JSON dump format
 :ref:`ksconf sort <ksconf_cmd_sort>`,stable,Sort a Splunk .conf file creating a normalized format appropriate for version control
+:ref:`ksconf rest-export <ksconf_cmd_rest-export>`,beta,Export .conf settings as a curl script to apply to a Splunk instance later (via REST)
+:ref:`ksconf rest-publish <ksconf_cmd_rest-publish>`,alpha,Publish .conf settings to a live Splunk instance via REST
 :ref:`ksconf unarchive <ksconf_cmd_unarchive>`,beta,Install or upgrade an existing app in a git-friendly and safe way

ksconf/commands/__init__.py

@@ -59,6 +59,9 @@ def __init__(self, name, mode, stream=None, parse_profile=None, is_file=None):
         self._data = None
         self._parse_profile = parse_profile or {}
 
+    def exists(self):
+        return os.path.isfile(self.name)
+
     def readable(self):
         return "r" in self._mode
 

ksconf/commands/merge.py

@@ -10,8 +10,8 @@
 
 from ksconf.commands import KsconfCmd, dedent, ConfFileProxy, ConfFileType
 from ksconf.conf.merge import merge_conf_files
-from ksconf.conf.parser import PARSECONF_STRICT, PARSECONF_MID
-from ksconf.consts import EXIT_CODE_SUCCESS
+from ksconf.conf.parser import PARSECONF_STRICT, PARSECONF_MID, ConfParserException
+from ksconf.consts import EXIT_CODE_SUCCESS, EXIT_CODE_NO_SUCH_FILE, EXIT_CODE_BAD_CONF_FILE
 from ksconf.util.completers import conf_files_completer
 
 
@@ -26,7 +26,7 @@ class MergeCmd(KsconfCmd):
 
     def register_args(self, parser):
         parser.add_argument("conf", metavar="FILE", nargs="+",
-                            type=ConfFileType("r", "load", parse_profile=PARSECONF_MID),
+                            type=ConfFileType("r", "none", parse_profile=PARSECONF_MID),
                             help="The source configuration file(s) to collect settings from."
                             ).completer = conf_files_completer
         parser.add_argument("--target", "-t", metavar="FILE",
@@ -35,6 +35,12 @@ def register_args(self, parser):
             Save the merged configuration files to this target file.
             If not provided, the merged conf is written to standard output.""")
                             ).completer = conf_files_completer
+
+        # This is helpful when writing bash expressions like MyApp/{default,local}/props.conf;
+        # when either default or local may not be present.
+        parser.add_argument("--ignore-missing", "-s", default=False, action="store_true",
+                            help="Silently ignore any missing CONF files.")
+
         parser.add_argument("--dry-run", "-D", default=False, action="store_true", help=dedent("""\
             Enable dry-run mode.
             Instead of writing to TARGET, preview changes in 'diff' format.
@@ -45,5 +51,22 @@ def register_args(self, parser):
 
     def run(self, args):
         ''' Merge multiple configuration files into one '''
-        merge_conf_files(args.target, args.conf, dry_run=args.dry_run, banner_comment=args.banner)
+        try:
+            merge_conf_files(args.target, args.conf, dry_run=args.dry_run,
+                             banner_comment=args.banner, skip_missing=args.ignore_missing)
+        except ConfParserException as e:
+            # TODO:  We should really show *which* file had the parse error.
+            # self.stderr.write("Error: failed to parse '{}': {}\n".format(e)
+            self.stderr.write("Error: failed to parse: {}\n".format(e))
+            return EXIT_CODE_BAD_CONF_FILE
+        except IOError as e:
+            if e.errno == 2:
+                self.stderr.write("Error: can't open '{}'\n"
+                                  "If you'd like to silently ignore missing input files, "
+                                  "use '--ignore-missing'.\n".format(e.filename))
+                return EXIT_CODE_NO_SUCH_FILE
+            else:
+                self.stderr.write(e)
+                # Is there a better exit code for this?
+                return EXIT_CODE_NO_SUCH_FILE
         return EXIT_CODE_SUCCESS

ksconf/conf/merge.py

@@ -10,6 +10,7 @@
 from ksconf.conf.delta import compare_cfgs, show_diff
 from ksconf.conf.parser import GLOBAL_STANZA, _extract_comments, inject_section_comments
 from ksconf.consts import SMART_UPDATE
+from ksconf.commands import ConfFileProxy
 
 ####################################################################################################
 ## Merging logic
@@ -56,9 +57,13 @@ def merge_conf_dicts(*dicts):
     return result
 
 
-def merge_conf_files(dest, configs, dry_run=False, banner_comment=None):
+def merge_conf_files(dest, configs, dry_run=False, banner_comment=None, skip_missing=False):
+    # type: (str, ConfFileProxy, bool, str, bool) -> dict
     # Parse all config files
-    cfgs = [conf.data for conf in configs]
+    if skip_missing:
+        cfgs = [conf.data for conf in configs if conf.exists()]
+    else:
+        cfgs = [conf.data for conf in configs]
     # Merge all config files:
     merged_cfg = merge_conf_dicts(*cfgs)
     if banner_comment:

tests/test_cli.py

@@ -30,16 +30,20 @@ def test_conffileproxy_invalid_arg(self):
         twd = TestWorkDir()
         badfile = twd.write_file("bad_conf.conf", bad_conf)
         with ksconf_cli:
-            ko = ksconf_cli("merge", twd.get_path("a_non_existant_file.conf"))
+
+            # A command that uses ConfFileType() with mode="load"
+            base_cmd = [ "rest-export" ]
+
+            ko = ksconf_cli(*base_cmd + [twd.get_path("a_non_existent_file.conf")])
             self.assertIn(ko.returncode, (EXIT_CODE_USER_QUIT, EXIT_CODE_NO_SUCH_FILE))
             self.assertRegex(ko.stderr, r".*\b(can't open '[^']+\.conf'|invalid ConfFileType).*")
 
-            ko = ksconf_cli("merge", badfile)
+            ko = ksconf_cli(*base_cmd + [badfile])
             self.assertIn(ko.returncode, (EXIT_CODE_USER_QUIT, EXIT_CODE_NO_SUCH_FILE))
             self.assertRegex(ko.stderr, ".*(failed to parse|invalid ConfFileType).*")
 
             with FakeStdin(bad_conf):
-                ko = ksconf_cli("merge", "-")
+                ko = ksconf_cli(*base_cmd + ["-"])
                 self.assertIn(ko.returncode, (EXIT_CODE_USER_QUIT, EXIT_CODE_NO_SUCH_FILE))
                 self.assertRegex(ko.stderr, ".*(failed to parse|invalid ConfFileType).*")
 

tests/test_cli_merge.py

@@ -66,6 +66,36 @@ def test_magic_stanza_drop(self):
             conf = ko.get_conf()
             self.assertNotIn("script://./bin/ps.sh", conf)
 
+    def test_missing_file(self):
+        twd = TestWorkDir()
+        with ksconf_cli:
+            # Missing files should be reported as an error, by default
+            ko = ksconf_cli("merge", twd.get_path("a_non_existent_file.conf"))
+            self.assertIn(ko.returncode, (EXIT_CODE_USER_QUIT, EXIT_CODE_NO_SUCH_FILE))
+            self.assertRegex(ko.stderr, r".*\b(can't open '[^']+\.conf'|invalid ConfFileType).*")
+
+            # Make sure that with --ignore-missing missing files are silently ignored
+            ko = ksconf_cli("merge", "--ignore-missing", twd.get_path("a_non_existent_file.conf"))
+            self.assertEqual(ko.returncode, EXIT_CODE_SUCCESS)
+            self.assertEqual(ko.stdout.strip(), "")
+
+    def test_invalid_conf(self):
+        bad_conf = """
+        [dangling stanza
+        attr = 1
+        bad file =  very true"""
+        twd = TestWorkDir()
+        badfile = twd.write_file("bad_conf.conf", bad_conf)
+        with ksconf_cli:
+            ko = ksconf_cli("merge", badfile)
+            self.assertIn(ko.returncode, (EXIT_CODE_USER_QUIT, EXIT_CODE_BAD_CONF_FILE))
+            self.assertRegex(ko.stderr, ".*(failed to parse|invalid ConfFileType).*")
+
+            with FakeStdin(bad_conf):
+                ko = ksconf_cli("merge", "-")
+                self.assertIn(ko.returncode, (EXIT_CODE_USER_QUIT, EXIT_CODE_BAD_CONF_FILE))
+                self.assertRegex(ko.stderr, ".*(failed to parse|invalid ConfFileType).*")
+
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()