Add tertiary authentication for metadata (TAM)

Signed-off-by: Thomas Waldmann <tw@waldmann-edv.de>

borg/archive.py

@@ -231,7 +231,7 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False,
 
     def _load_meta(self, id):
         data = self.key.decrypt(id, self.repository.get(id))
-        metadata = msgpack.unpackb(data)
+        metadata = msgpack.unpackb(data, unicode_errors='surrogateescape')
         if metadata[b'version'] != 1:
             raise Exception('Unknown archive metadata version')
         return metadata
@@ -325,7 +325,7 @@ def save(self, name=None, timestamp=None):
             'time': start.isoformat(),
             'time_end': end.isoformat(),
         })
-        data = msgpack.packb(metadata, unicode_errors='surrogateescape')
+        data = self.key.pack_and_authenticate_metadata(metadata, context=b'archive')
         self.id = self.key.id_hash(data)
         self.cache.add_chunk(self.id, data, self.stats)
         self.manifest.archives[name] = {'id': self.id, 'time': metadata['time']}

borg/archiver.py

@@ -30,7 +30,7 @@
 from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
 from .repository import Repository
 from .cache import Cache
-from .key import key_creator, RepoKey, PassphraseKey
+from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
 from .keymanager import KeyManager
 from .archive import backup_io, BackupOSError, Archive, ArchiveChecker, CHUNKER_PARAMS, is_special
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
@@ -51,7 +51,7 @@ def argument(args, str_or_bool):
     return str_or_bool
 
 
-def with_repository(fake=False, create=False, lock=True, exclusive=False, manifest=True, cache=False):
+def with_repository(fake=False, invert_fake=False, create=False, lock=True, exclusive=False, manifest=True, cache=False):
     """
     Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …)
 
@@ -68,7 +68,7 @@ def decorator(method):
         def wrapper(self, args, **kwargs):
             location = args.location  # note: 'location' must be always present in args
             append_only = getattr(args, 'append_only', False)
-            if argument(args, fake):
+            if argument(args, fake) ^ invert_fake:
                 return method(self, args, repository=None, **kwargs)
             elif location.proto == 'ssh':
                 repository = RemoteRepository(location, create=create, exclusive=argument(args, exclusive),
@@ -135,6 +135,8 @@ def do_init(self, args, repository):
         repository.commit()
         with Cache(repository, key, manifest, warn_if_unencrypted=False):
             pass
+        tam_file = tam_required_file(repository)
+        open(tam_file, 'w').close()
         return self.exit_code
 
     @with_repository(exclusive=True, manifest=False)
@@ -161,6 +163,7 @@ def do_check(self, args, repository):
     def do_change_passphrase(self, args, repository, manifest, key):
         """Change repository key file passphrase"""
         key.change_passphrase()
+        logger.info('Key updated')
         return EXIT_SUCCESS
 
     @with_repository(lock=False, exclusive=False, manifest=False, cache=False)
@@ -209,6 +212,7 @@ def do_migrate_to_repokey(self, args, repository):
         key_new.id_key = key_old.id_key
         key_new.chunk_seed = key_old.chunk_seed
         key_new.change_passphrase()  # option to change key protection passphrase, save
+        logger.info('Key updated')
         return EXIT_SUCCESS
 
     @with_repository(fake='dry_run', exclusive=True)
@@ -705,21 +709,43 @@ def do_prune(self, args, repository, manifest, key):
                           DASHES)
         return self.exit_code
 
-    def do_upgrade(self, args):
+    @with_repository(fake='tam', invert_fake=True, manifest=False, exclusive=True)
+    def do_upgrade(self, args, repository, manifest=None, key=None):
         """upgrade a repository from a previous version"""
-        # mainly for upgrades from Attic repositories,
-        # but also supports borg 0.xx -> 1.0 upgrade.
+        if args.tam:
+            manifest, key = Manifest.load(repository, force_tam_not_required=args.force)
+
+            if not manifest.tam_verified:
+                # The standard archive listing doesn't include the archive ID like in borg 1.1.x
+                print('Manifest contents:')
+                for archive_info in manifest.list_archive_infos(sort_by='ts'):
+                    print(format_archive(archive_info), '[%s]' % bin_to_hex(archive_info.id))
+                manifest.write()
+                repository.commit()
+            if not key.tam_required:
+                key.tam_required = True
+                key.change_passphrase(key._passphrase)
+                print('Updated key')
+                if hasattr(key, 'find_key'):
+                    print('Key location:', key.find_key())
+            if not tam_required(repository):
+                tam_file = tam_required_file(repository)
+                open(tam_file, 'w').close()
+                print('Updated security database')
+        else:
+            # mainly for upgrades from Attic repositories,
+            # but also supports borg 0.xx -> 1.0 upgrade.
 
-        repo = AtticRepositoryUpgrader(args.location.path, create=False)
-        try:
-            repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
-        except NotImplementedError as e:
-            print("warning: %s" % e)
-        repo = BorgRepositoryUpgrader(args.location.path, create=False)
-        try:
-            repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
-        except NotImplementedError as e:
-            print("warning: %s" % e)
+            repo = AtticRepositoryUpgrader(args.location.path, create=False)
+            try:
+                repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
+            except NotImplementedError as e:
+                print("warning: %s" % e)
+            repo = BorgRepositoryUpgrader(args.location.path, create=False)
+            try:
+                repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
+            except NotImplementedError as e:
+                print("warning: %s" % e)
         return self.exit_code
 
     def do_debug_info(self, args):
@@ -1613,6 +1639,28 @@ def build_parser(self, args=None, prog=None):
 
         upgrade_epilog = textwrap.dedent("""
         Upgrade an existing Borg repository.
+
+        Borg 1.x.y upgrades
+        -------------------
+
+        Use ``borg upgrade --tam REPO`` to require manifest authentication
+        introduced with Borg 1.0.9 to address security issues. This means
+        that modifying the repository after doing this with a version prior
+        to 1.0.9 will raise a validation error, so only perform this upgrade
+        after updating all clients using the repository to 1.0.9 or newer.
+
+        This upgrade should be done on each client for safety reasons.
+
+        If a repository is accidentally modified with a pre-1.0.9 client after
+        this upgrade, use ``borg upgrade --tam --force REPO`` to remedy it.
+
+        See
+        https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability
+        for details.
+
+        Attic and Borg 0.xx to Borg 1.x
+        -------------------------------
+
         This currently supports converting an Attic repository to Borg and also
         helps with converting Borg 0.xx to 1.0.
 
@@ -1665,6 +1713,10 @@ def build_parser(self, args=None, prog=None):
                                default=False, action='store_true',
                                help="""rewrite repository in place, with no chance of going back to older
                                versions of the repository.""")
+        subparser.add_argument('--force', dest='force', action='store_true',
+                               help="""Force upgrade""")
+        subparser.add_argument('--tam', dest='tam', action='store_true',
+                               help="""Enable manifest authentication (in key and cache) (Borg 1.0.9 and later)""")
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False),
                                help='path to the repository to be upgraded')

borg/crypto.pyx

@@ -2,9 +2,13 @@
 
 This could be replaced by PyCrypto maybe?
 """
+import hashlib
+import hmac
+from math import ceil
+
 from libc.stdlib cimport malloc, free
 
-API_VERSION = 2
+API_VERSION = 3
 
 cdef extern from "openssl/rand.h":
     int  RAND_bytes(unsigned char *buf, int num)
@@ -171,3 +175,30 @@ cdef class AES:
             return out[:ptl]
         finally:
             free(out)
+
+
+def hkdf_hmac_sha512(ikm, salt, info, output_length):
+    """
+    Compute HKDF-HMAC-SHA512 with input key material *ikm*, *salt* and *info* to produce *output_length* bytes.
+
+    This is the "HMAC-based Extract-and-Expand Key Derivation Function (HKDF)" (RFC 5869)
+    instantiated with HMAC-SHA512.
+
+    *output_length* must not be greater than 64 * 255 bytes.
+    """
+    digest_length = 64
+    assert output_length <= (255 * digest_length), 'output_length must be <= 255 * 64 bytes'
+    # Step 1. HKDF-Extract (ikm, salt) -> prk
+    if salt is None:
+        salt = bytes(64)
+    prk = hmac.HMAC(salt, ikm, hashlib.sha512).digest()
+
+    # Step 2. HKDF-Expand (prk, info, output_length) -> output key
+    n = ceil(output_length / digest_length)
+    t_n = b''
+    output = b''
+    for i in range(n):
+        msg = t_n + info + (i + 1).to_bytes(1, 'little')
+        t_n = hmac.HMAC(prk, msg, hashlib.sha512).digest()
+        output += t_n
+    return output[:output_length]

borg/helpers.py

@@ -86,7 +86,7 @@ def check_extension_modules():
         raise ExtensionModuleError
     if chunker.API_VERSION != 2:
         raise ExtensionModuleError
-    if crypto.API_VERSION != 2:
+    if crypto.API_VERSION != 3:
         raise ExtensionModuleError
     if platform.API_VERSION != 3:
         raise ExtensionModuleError
@@ -103,10 +103,11 @@ def __init__(self, key, repository, item_keys=None):
         self.key = key
         self.repository = repository
         self.item_keys = frozenset(item_keys) if item_keys is not None else ITEM_KEYS
+        self.tam_verified = False
 
     @classmethod
-    def load(cls, repository, key=None):
-        from .key import key_factory
+    def load(cls, repository, key=None, force_tam_not_required=False):
+        from .key import key_factory, tam_required_file, tam_required
         from .repository import Repository
         from .archive import ITEM_KEYS
         try:
@@ -117,8 +118,8 @@ def load(cls, repository, key=None):
             key = key_factory(repository, cdata)
         manifest = cls(key, repository)
         data = key.decrypt(None, cdata)
+        m, manifest.tam_verified = key.unpack_and_verify_manifest(data, force_tam_not_required=force_tam_not_required)
         manifest.id = key.id_hash(data)
-        m = msgpack.unpackb(data)
         if not m.get(b'version') == 1:
             raise ValueError('Invalid manifest version')
         manifest.archives = dict((k.decode('utf-8'), v) for k, v in m[b'archives'].items())
@@ -128,19 +129,27 @@ def load(cls, repository, key=None):
         manifest.config = m[b'config']
         # valid item keys are whatever is known in the repo or every key we know
         manifest.item_keys = frozenset(m.get(b'item_keys', [])) | ITEM_KEYS
+        if manifest.config.get(b'tam_required', False) and manifest.tam_verified and not tam_required(repository):
+            logger.debug('Manifest is TAM verified and says TAM is required, updating security database...')
+            file = tam_required_file(repository)
+            open(file, 'w').close()
         return manifest, key
 
     def write(self):
+        if self.key.tam_required:
+            self.config[b'tam_required'] = True
         self.timestamp = datetime.utcnow().isoformat()
-        data = msgpack.packb(StableDict({
+        m = {
             'version': 1,
-            'archives': self.archives,
+            'archives': StableDict((name, StableDict(archive)) for name, archive in self.archives.items()),
             'timestamp': self.timestamp,
-            'config': self.config,
-            'item_keys': tuple(self.item_keys),
-        }))
+            'config': StableDict(self.config),
+            'item_keys': tuple(sorted(self.item_keys)),
+        }
+        self.tam_verified = True
+        data = self.key.pack_and_authenticate_metadata(m)
         self.id = self.key.id_hash(data)
-        self.repository.put(self.MANIFEST_ID, self.key.encrypt(data))
+        self.repository.put(self.MANIFEST_ID, self.key.encrypt(data, none_compression=True))
 
     def list_archive_infos(self, sort_by=None, reverse=False):
         # inexpensive Archive.list_archives replacement if we just need .name, .id, .ts
@@ -249,6 +258,18 @@ def get_keys_dir():
     return keys_dir
 
 
+def get_security_dir(repository_id=None):
+    """Determine where to store local security information."""
+    xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config'))
+    security_dir = os.environ.get('BORG_SECURITY_DIR', os.path.join(xdg_config, 'borg', 'security'))
+    if repository_id:
+        security_dir = os.path.join(security_dir, repository_id)
+    if not os.path.exists(security_dir):
+        os.makedirs(security_dir)
+        os.chmod(security_dir, stat.S_IRWXU)
+    return security_dir
+
+
 def get_cache_dir():
     """Determine where to repository keys and cache"""
     xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache'))