Implemented composite configurations.

doc/configuring.rst

@@ -119,3 +119,33 @@ of strings::
 or::
 
     include = ['first-file-to-include', 'second-file-to-include']
+
+
+Composite Configurations
+-------------------------
+
+It is possible to define composite configurations that allow you to run several 
+configurations at once.  This might be useful if you have files that benefit, 
+for example, from different prune schedules.
+
+As an example, consider having three configurations that you would like to run 
+all at once. You can specify these configurations as follows::
+
+    configurations = 'root1 root2 root3 root=root1,root2,root3'
+
+In this case *root1*, *root2* and *root3* are simple configurations and *root* 
+is a composite configuration.  *root1*, *root2*, and *root3* would have 
+configuration files whereas *root* would not.
+
+You can run a specific configuration with:
+
+    emborg -c root1 extract ~/bin
+
+You can run all three configurations with:
+
+    emborg -c root create
+
+Only certain commands support composite configurations. Specifically, *create*, 
+*check*, *configs*, *due*, *help*, *info*, *prune*, and *version* support 
+composite configures.  Specifying a composite configuration to a command that 
+does not support them results in an error.

emborg/command.py

@@ -158,6 +158,7 @@ class BorgCommand(Command):
         repository.  The passphrase is set before the command is run.
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -182,6 +183,7 @@ class BreakLockCommand(Command):
         running before using this command.
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -216,6 +218,7 @@ class CreateCommand(Command):
         This can help you debug slow create operations.
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = True
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -314,6 +317,7 @@ class CheckCommand(Command):
             -v, --verify-data    perform a full integrity verification (slow)
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = True
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -341,6 +345,7 @@ class ConfigsCommand(Command):
             emborg configs
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = None
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -363,6 +368,7 @@ class DeleteCommand(Command):
             emborg delete <archive>
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -390,6 +396,7 @@ class DiffCommand(Command):
             emborg diff <archive1> <archive2>
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -445,6 +452,7 @@ class DueCommand(Command):
             It has been 4 months since the last backup.
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = True
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -542,6 +550,7 @@ class ExtractCommand(Command):
             ./home/ken/src/verif/av/manpages/settings.py
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -596,6 +605,7 @@ class HelpCommand(Command):
             emborg help [<topic>]
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = None
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -618,6 +628,7 @@ class InfoCommand(Command):
             -f, --fast            only report local information
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = True
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -661,6 +672,7 @@ class InitializeCommand(Command):
             emborg init
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -689,6 +701,7 @@ class ListCommand(Command):
             emborg lr
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -715,6 +728,7 @@ class LogCommand(Command):
             emborg log
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -761,6 +775,7 @@ class ManifestCommand(Command):
             emborg manifest --date 2018-12-05T12:39
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -827,6 +842,7 @@ class MountCommand(Command):
         You should use `emborg umount` when you are done.
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -870,6 +886,7 @@ class PruneCommand(Command):
             emborg prune
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = True
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -909,6 +926,7 @@ class SettingsCommand(Command):
             -a, --available   list available settings
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -947,6 +965,7 @@ class UmountCommand(Command):
             emborg [options] unmount <mount_point>
     """).strip()
     REQUIRES_EXCLUSIVITY = True
+    COMPOSITE_CONFIGS = False
 
     @classmethod
     def run(cls, command, args, settings, options):
@@ -977,6 +996,7 @@ class VersionCommand(Command):
             emborg version
     """).strip()
     REQUIRES_EXCLUSIVITY = False
+    COMPOSITE_CONFIGS = None
 
     @classmethod
     def run(cls, command, args, settings, options):

emborg/main.py

@@ -44,8 +44,8 @@
 # Imports {{{1
 from . import __version__, __released__
 from .command import Command
-from .settings import Settings
-from inform import Inform, Error, cull, fatal, display, terminate, os_error
+from .settings import Settings, NoMoreConfigs
+from inform import Inform, Error, cull, display, fatal, terminate, os_error
 from docopt import docopt
 
 # Globals {{{1
@@ -74,14 +74,15 @@ def main():
             inform.narrate = True
 
         try:
-            cmd, cmd_name = Command.find(command)
+            while True:
+                cmd, cmd_name = Command.find(command)
 
-            with Settings(config, cmd.REQUIRES_EXCLUSIVITY, options) as settings:
-                try:
-                    exit_status = cmd.execute(cmd_name, args, settings, options)
-                except Error as e:
-                    settings.fail(e)
-                    e.terminate(True)
+                with Settings(config, cmd, options) as settings:
+                    try:
+                        exit_status = cmd.execute(cmd_name, args, settings, options)
+                    except Error as e:
+                        settings.fail(e)
+                        e.terminate(True)
 
         except KeyboardInterrupt:
             display('Terminated by user.')
@@ -90,4 +91,6 @@ def main():
             e.terminate()
         except OSError as e:
             fatal(os_error(e))
+        except NoMoreConfigs:
+            pass
         terminate(exit_status)

emborg/preferences.py

@@ -54,6 +54,7 @@
     default_configuration = 'default emborg configuration',
     encryption = 'encryption method (see borg documentation)',
     excludes = 'list of glob strings of files or directories to skip',
+    exclude_from = 'file that contains exclude patterns',
     must_exist = 'if set, each of these files or directorys must exist or create will quit with an error',
     needs_ssh_agent = 'if set, emborg will complain if ssh_agent is not available',
     notifier = 'notification program',
@@ -87,11 +88,6 @@
         cmds = 'create',
         desc = 'exclude directories that contain a CACHEDIR.TAG file'
     ),
-    exclude_from = dict(
-        cmds = 'create',
-        arg = 'FILE',
-        desc = 'file that contains exclude patterns',
-    ),
     exclude_if_present = dict(
         cmds = 'create',
         arg = 'NAME',

emborg/settings.py

@@ -41,8 +41,8 @@
 from .utilities import gethostname, getusername, render_paths
 from shlib import getmod, mv, rm, Run, to_path, render_command, to_path
 from inform import (
-    Error, codicil, conjoin, done, full_stop, get_informer, indent, is_str,
-    narrate, output, render, warn,
+    Error, codicil, conjoin, display, done, full_stop, get_informer, indent,
+    is_str, narrate, output, render, warn,
 )
 from textwrap import dedent
 import arrow
@@ -53,24 +53,84 @@
 hostname = gethostname()
 username = getusername()
 
+borg_commands_with_dryrun = 'create extract delete prune upgrade recreate'
+
 # borg_options_arg_count {{{2
 borg_options_arg_count = {
     'borg': 1,
     '--exclude': 1,
+    '--exclude-from': 1,
     '--encryption': 1,
 }
 for name, attrs in BORG_SETTINGS.items():
     if 'arg' in attrs and attrs['arg']:
         borg_options_arg_count[convert_name_to_option(name)] = 1
 
-commands_with_dryrun = 'create extract delete prune upgrade recreate'.split()
+# get_config {{{2
+config_queue = None
+
+class NoMoreConfigs(Exception):
+    pass
+
+def get_config(name, settings, composite_config_allowed):
+    global config_queue
+
+    if config_queue is not None:
+        try:
+            active_config = config_queue.pop(0)
+            display('\n===', active_config, '===')
+            return active_config
+        except IndexError:
+            raise NoMoreConfigs()
+
+    configs = Collection(settings.get(CONFIGS_SETTING, ''))
+    default = settings.get(DEFAULT_CONFIG_SETTING)
+
+    config_groups = {}
+    for config in configs:
+        if '=' in config:
+            group, _, sub_configs = config.partition('=')
+            sub_configs = sub_configs.split(',')
+        else:
+            group = config
+            sub_configs = [config]
+        config_groups[group] = sub_configs
+
+    if not name:
+        name = default
+    if name:
+        if name not in config_groups:
+            raise Error('unknown configuration.', culprit=name)
+    else:
+        if len(configs) > 1:
+            name = configs[0]
+        else:
+            raise Error(
+                'no known configurations.',
+                culprit=(settings_filename, CONFIGS_SETTING)
+            )
+    configs = list(config_groups[name])
+    num_configs = len(configs)
+    if num_configs > 1 and composite_config_allowed is False:
+        raise Error('command does not support composite configs.', culprit=name)
+    elif num_configs < 1:
+        raise Error('empty composite config.', culprit=name)
+    active_config = configs.pop(0)
+    if composite_config_allowed is None:
+        config_queue = []
+    else:
+        config_queue = configs
+    if config_queue:
+        display('===', active_config, '===')
+    return active_config
 
 
 # Settings class {{{1
 class Settings:
     # Constructor {{{2
-    def __init__(self, name=None, requires_exclusivity=True, options=()):
-        self.requires_exclusivity = requires_exclusivity
+    def __init__(self, name, command, options=()):
+        self.requires_exclusivity = command.REQUIRES_EXCLUSIVITY
+        self.composite_config_allowed = command.COMPOSITE_CONFIGS
         self.settings = {}
         self.options = options
         self.read(name)
@@ -96,7 +156,7 @@ def read(self, name=None, path=None):
             parent = path.parent
             includes = Collection(settings.get(INCLUDE_SETTING))
         else:
-            # this is generic settings file
+            # this is the generic settings file
             parent = to_path(CONFIG_DIR)
             if not parent.exists():
                 # config dir does not exist, create and populate it
@@ -123,25 +183,7 @@ def read(self, name=None, path=None):
             settings_filename = path.path
             settings = path.run()
 
-            configs = Collection(settings.get(CONFIGS_SETTING, ''))
-            default = settings.get(DEFAULT_CONFIG_SETTING)
-            if not name:
-                name = default
-            if name:
-                if name not in configs:
-                    raise Error(
-                        'unknown configuration.',
-                        culprit=(settings_filename, CONFIGS_SETTING, name)
-                    )
-                config = name
-            else:
-                if len(configs) > 1:
-                    config = configs[0]
-                else:
-                    raise Error(
-                        'no known configurations.',
-                        culprit=(settings_filename, CONFIGS_SETTING)
-                    )
+            config = get_config(name, settings, self.composite_config_allowed)
             settings['config_name'] = config
             self.config_name = config
             includes = Collection(settings.get(INCLUDE_SETTING))
@@ -258,7 +300,7 @@ def borg_options(self, cmd, options):
             args.append('--verbose')
         elif self.value('verbose'):
             args.append('--verbose')
-        if 'trial-run' in options and cmd in commands_with_dryrun:
+        if 'trial-run' in options and cmd in borg_commands_with_dryrun.split():
             args.append('--dry-run')
         if cmd == 'create':
             if 'verbose' in options: