Merge pull request #8104 from ThomasWaldmann/error-msg-bad-nonce-file…

…2-master

refactor: use less binascii

src/borg/archiver/config_cmd.py

@@ -1,13 +1,12 @@
 import argparse
 import configparser
-from binascii import unhexlify
 
 from ._common import with_repository
 from ..cache import Cache, assert_secure
 from ..constants import *  # NOQA
 from ..helpers import Error, CommandError
 from ..helpers import Location
-from ..helpers import parse_file_size
+from ..helpers import parse_file_size, hex_to_bin
 from ..manifest import Manifest
 
 from ..logger import create_logger
@@ -46,12 +45,7 @@ def repo_validate(section, name, value=None, check_value=True):
                     raise ValueError("Invalid value")
             elif name in ["id"]:
                 if check_value:
-                    try:
-                        bin_id = unhexlify(value)
-                    except:  # noqa
-                        raise ValueError("Invalid value, must be 64 hex digits") from None
-                    if len(bin_id) != 32:
-                        raise ValueError("Invalid value, must be 64 hex digits")
+                    hex_to_bin(value, length=32)
             else:
                 raise ValueError("Invalid name")
 

src/borg/archiver/debug_cmd.py

@@ -1,5 +1,4 @@
 import argparse
-from binascii import unhexlify, hexlify
 import functools
 import json
 import textwrap
@@ -9,7 +8,7 @@
 from ..constants import *  # NOQA
 from ..helpers import msgpack
 from ..helpers import sysinfo
-from ..helpers import bin_to_hex, prepare_dump_dict
+from ..helpers import bin_to_hex, hex_to_bin, prepare_dump_dict
 from ..helpers import dash_open
 from ..helpers import StableDict
 from ..helpers import positive_int_validator, archivename_validator
@@ -179,7 +178,7 @@ def print_finding(info, wanted, data, offset):
         wanted = args.wanted
         try:
             if wanted.startswith("hex:"):
-                wanted = unhexlify(wanted[4:])
+                wanted = hex_to_bin(wanted[4:])
             elif wanted.startswith("str:"):
                 wanted = wanted[4:].encode()
             else:
@@ -235,9 +234,7 @@ def do_debug_get_obj(self, args, repository):
         """get object contents from the repository and write it into file"""
         hex_id = args.id
         try:
-            id = unhexlify(hex_id)
-            if len(id) != 32:  # 256bit
-                raise ValueError("id must be 256bits or 64 hex digits")
+            id = hex_to_bin(hex_id, length=32)
         except ValueError as err:
             raise CommandError(f"object id {hex_id} is invalid [{str(err)}].")
         try:
@@ -264,9 +261,7 @@ def do_debug_parse_obj(self, args, repository, manifest):
         # get the object from id
         hex_id = args.id
         try:
-            id = unhexlify(hex_id)
-            if len(id) != 32:  # 256bit
-                raise ValueError("id must be 256bits or 64 hex digits")
+            id = hex_to_bin(hex_id, length=32)
         except ValueError as err:
             raise CommandError(f"object id {hex_id} is invalid [{str(err)}].")
 
@@ -289,9 +284,7 @@ def do_debug_format_obj(self, args, repository, manifest):
         # get the object from id
         hex_id = args.id
         try:
-            id = unhexlify(hex_id)
-            if len(id) != 32:  # 256bit
-                raise ValueError("id must be 256bits or 64 hex digits")
+            id = hex_to_bin(hex_id, length=32)
         except ValueError as err:
             raise CommandError(f"object id {hex_id} is invalid [{str(err)}].")
 
@@ -315,9 +308,7 @@ def do_debug_put_obj(self, args, repository):
             data = f.read()
         hex_id = args.id
         try:
-            id = unhexlify(hex_id)
-            if len(id) != 32:  # 256bit
-                raise ValueError("id must be 256bits or 64 hex digits")
+            id = hex_to_bin(hex_id, length=32)
         except ValueError as err:
             raise CommandError(f"object id {hex_id} is invalid [{str(err)}].")
 
@@ -331,7 +322,7 @@ def do_debug_delete_obj(self, args, repository):
         modified = False
         for hex_id in args.ids:
             try:
-                id = unhexlify(hex_id)
+                id = hex_to_bin(hex_id, length=32)
             except ValueError:
                 print("object id %s is invalid." % hex_id)
             else:
@@ -350,7 +341,7 @@ def do_debug_refcount_obj(self, args, repository, manifest, cache):
         """display refcounts for the objects with the given IDs"""
         for hex_id in args.ids:
             try:
-                id = unhexlify(hex_id)
+                id = hex_to_bin(hex_id, length=32)
             except ValueError:
                 print("object id %s is invalid." % hex_id)
             else:
@@ -370,7 +361,7 @@ def do_debug_dump_hints(self, args, repository):
                 segments=repository.segments,
                 compact=repository.compact,
                 storage_quota_use=repository.storage_quota_use,
-                shadow_index={hexlify(k).decode(): v for k, v in repository.shadow_index.items()},
+                shadow_index={bin_to_hex(k): v for k, v in repository.shadow_index.items()},
             )
             with dash_open(args.path, "w") as fd:
                 json.dump(hints, fd, indent=4)

src/borg/cache.py

@@ -2,7 +2,6 @@
 import os
 import shutil
 import stat
-from binascii import unhexlify
 from collections import namedtuple
 from time import perf_counter
 
@@ -17,7 +16,7 @@
 from .helpers import Location
 from .helpers import Error
 from .helpers import get_cache_dir, get_security_dir
-from .helpers import bin_to_hex, parse_stringified_list
+from .helpers import bin_to_hex, hex_to_bin, parse_stringified_list
 from .helpers import format_file_size
 from .helpers import safe_ns
 from .helpers import yes
@@ -299,7 +298,7 @@ def load(self):
             self._config.read_file(fd)
         self._check_upgrade(self.config_path)
         self.id = self._config.get("cache", "repository")
-        self.manifest_id = unhexlify(self._config.get("cache", "manifest"))
+        self.manifest_id = hex_to_bin(self._config.get("cache", "manifest"))
         self.timestamp = self._config.get("cache", "timestamp", fallback=None)
         self.key_type = self._config.get("cache", "key_type", fallback=None)
         self.ignored_features = set(parse_stringified_list(self._config.get("cache", "ignored_features", fallback="")))
@@ -733,8 +732,8 @@ def cached_archives():
                 fns = os.listdir(archive_path)
                 # filenames with 64 hex digits == 256bit,
                 # or compact indices which are 64 hex digits + ".compact"
-                return {unhexlify(fn) for fn in fns if len(fn) == 64} | {
-                    unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith(".compact")
+                return {hex_to_bin(fn) for fn in fns if len(fn) == 64} | {
+                    hex_to_bin(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith(".compact")
                 }
             else:
                 return set()

src/borg/crypto/key.py

@@ -2,7 +2,6 @@
 import hmac
 import os
 import textwrap
-from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, pbkdf2_hmac
 from typing import Literal, Callable, ClassVar
 
@@ -382,7 +381,7 @@ def detect(cls, repository, manifest_data):
         return key
 
     def _load(self, key_data, passphrase):
-        cdata = a2b_base64(key_data)
+        cdata = binascii.a2b_base64(key_data)
         data = self.decrypt_key_file(cdata, passphrase)
         if data:
             data = msgpack.unpackb(data)
@@ -507,7 +506,7 @@ def _save(self, passphrase, algorithm):
             chunk_seed=self.chunk_seed,
         )
         data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase, algorithm)
-        key_data = "\n".join(textwrap.wrap(b2a_base64(data).decode("ascii")))
+        key_data = "\n".join(textwrap.wrap(binascii.b2a_base64(data).decode("ascii")))
         return key_data
 
     def change_passphrase(self, passphrase=None):
@@ -547,7 +546,7 @@ def create(cls, repository, args, *, other_key=None):
 
     def sanity_check(self, filename, id):
         file_id = self.FILE_ID.encode() + b" "
-        repo_id = hexlify(id)
+        repo_id = bin_to_hex(id).encode("ascii")
         with open(filename, "rb") as fd:
             # we do the magic / id check in binary mode to avoid stumbling over
             # decoding errors if somebody has binary files in the keys dir for some reason.
@@ -567,7 +566,7 @@ def sanity_check(self, filename, id):
                 raise KeyfileInvalidError(self.repository._location.canonical_path(), filename)
             key_b64 = "".join(lines[1:])
             try:
-                key = a2b_base64(key_b64)
+                key = binascii.a2b_base64(key_b64)
             except binascii.Error:
                 logger.warning(f"borg key sanity check: key line 2+ does not look like base64. [{filename}]")
                 raise KeyfileInvalidError(self.repository._location.canonical_path(), filename)

src/borg/crypto/keymanager.py

@@ -1,10 +1,9 @@
 import binascii
 import pkgutil
 import textwrap
-from binascii import unhexlify, a2b_base64, b2a_base64
 from hashlib import sha256
 
-from ..helpers import Error, yes, bin_to_hex, dash_open
+from ..helpers import Error, yes, bin_to_hex, hex_to_bin, dash_open
 from ..manifest import Manifest, NoManifestError
 from ..repository import Repository
 from ..repoobj import RepoObj
@@ -127,7 +126,7 @@ def grouped(s):
 
         export = "To restore key use borg key import --paper /path/to/repo\n\n"
 
-        binary = a2b_base64(self.keyblob)
+        binary = binascii.a2b_base64(self.keyblob)
         export += "BORG PAPER KEY v1\n"
         lines = (len(binary) + 17) // 18
         repoid = bin_to_hex(self.repository.id)[:18]
@@ -218,9 +217,9 @@ def import_paperkey(self, args):
                         print("each line must contain exactly one '-', try again")
                         continue
                     try:
-                        part = unhexlify(data)
-                    except binascii.Error:
-                        print("only characters 0-9 and a-f and '-' are valid, try again")
+                        part = hex_to_bin(data)
+                    except ValueError as e:
+                        print(f"only characters 0-9 and a-f and '-' are valid, try again [{e}]")
                         continue
                     if sha256_truncated(idx.to_bytes(2, byteorder="big") + part, 2) != checksum:
                         print(f"line checksum did not match, try line {idx} again")
@@ -234,7 +233,7 @@ def import_paperkey(self, args):
                     print("The overall checksum did not match, retry or enter a blank line to abort.")
                     continue
 
-                self.keyblob = "\n".join(textwrap.wrap(b2a_base64(result).decode("ascii"))) + "\n"
+                self.keyblob = "\n".join(textwrap.wrap(binascii.b2a_base64(result).decode("ascii"))) + "\n"
                 self.store_keyblob(args)
                 break
 

src/borg/repository.py

@@ -5,7 +5,6 @@
 import stat
 import struct
 import time
-from binascii import unhexlify
 from collections import defaultdict
 from configparser import ConfigParser
 from datetime import datetime, timezone
@@ -18,7 +17,7 @@
 from .helpers import Error, ErrorWithTraceback, IntegrityError, format_file_size, parse_file_size
 from .helpers import Location
 from .helpers import ProgressIndicatorPercent
-from .helpers import bin_to_hex
+from .helpers import bin_to_hex, hex_to_bin
 from .helpers import secure_erase, safe_unlink
 from .helpers import msgpack
 from .helpers.lrucache import LRUCache
@@ -490,7 +489,7 @@ def open(self, path, exclusive, lock_wait=None, lock=True):
         if self.storage_quota is None:
             # self.storage_quota is None => no explicit storage_quota was specified, use repository setting.
             self.storage_quota = parse_file_size(self.config.get("repository", "storage_quota", fallback=0))
-        self.id = unhexlify(self.config.get("repository", "id").strip())
+        self.id = hex_to_bin(self.config.get("repository", "id").strip(), length=32)
         self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir)
 
     def _load_hints(self):

src/borg/testsuite/archiver/key_cmds.py

@@ -1,13 +1,13 @@
+import binascii
 import os
-from binascii import unhexlify, b2a_base64, a2b_base64
 
 import pytest
 
 from ...constants import *  # NOQA
 from ...crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPOKeyfileKey, Passphrase
 from ...crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
 from ...helpers import EXIT_ERROR, CommandError
-from ...helpers import bin_to_hex
+from ...helpers import bin_to_hex, hex_to_bin
 from ...helpers import msgpack
 from ...repository import Repository
 from .. import key
@@ -223,12 +223,12 @@ def test_key_export_paperkey(archivers, request):
     repo_id = "e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239"
     export_file = archiver.output_path + "/exported"
     cmd(archiver, "rcreate", KF_ENCRYPTION)
-    _set_repository_id(archiver.repository_path, unhexlify(repo_id))
+    _set_repository_id(archiver.repository_path, hex_to_bin(repo_id))
     key_file = archiver.keys_path + "/" + os.listdir(archiver.keys_path)[0]
 
     with open(key_file, "w") as fd:
         fd.write(CHPOKeyfileKey.FILE_ID + " " + repo_id + "\n")
-        fd.write(b2a_base64(b"abcdefghijklmnopqrstu").decode())
+        fd.write(binascii.b2a_base64(b"abcdefghijklmnopqrstu").decode())
 
     cmd(archiver, "key", "export", "--paper", export_file)
 
@@ -251,12 +251,12 @@ def test_key_import_paperkey(archivers, request):
     archiver = request.getfixturevalue(archivers)
     repo_id = "e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239"
     cmd(archiver, "rcreate", KF_ENCRYPTION)
-    _set_repository_id(archiver.repository_path, unhexlify(repo_id))
+    _set_repository_id(archiver.repository_path, hex_to_bin(repo_id))
 
     key_file = archiver.keys_path + "/" + os.listdir(archiver.keys_path)[0]
     with open(key_file, "w") as fd:
         fd.write(AESOCBKeyfileKey.FILE_ID + " " + repo_id + "\n")
-        fd.write(b2a_base64(b"abcdefghijklmnopqrstu").decode())
+        fd.write(binascii.b2a_base64(b"abcdefghijklmnopqrstu").decode())
 
     typed_input = (
         b"2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41  02\n"  # Forgot to type "-"
@@ -298,7 +298,7 @@ def test_init_defaults_to_argon2(archivers, request):
     archiver = request.getfixturevalue(archivers)
     cmd(archiver, "rcreate", RK_ENCRYPTION)
     with Repository(archiver.repository_path) as repository:
-        key = msgpack.unpackb(a2b_base64(repository.load_key()))
+        key = msgpack.unpackb(binascii.a2b_base64(repository.load_key()))
         assert key["algorithm"] == "argon2 chacha20-poly1305"
 
 
@@ -309,7 +309,7 @@ def test_change_passphrase_does_not_change_algorithm_argon2(archivers, request):
     cmd(archiver, "key", "change-passphrase")
 
     with Repository(archiver.repository_path) as repository:
-        key = msgpack.unpackb(a2b_base64(repository.load_key()))
+        key = msgpack.unpackb(binascii.a2b_base64(repository.load_key()))
         assert key["algorithm"] == "argon2 chacha20-poly1305"
 
 
@@ -319,5 +319,5 @@ def test_change_location_does_not_change_algorithm_argon2(archivers, request):
     cmd(archiver, "key", "change-location", "repokey")
 
     with Repository(archiver.repository_path) as repository:
-        key = msgpack.unpackb(a2b_base64(repository.load_key()))
+        key = msgpack.unpackb(binascii.a2b_base64(repository.load_key()))
         assert key["algorithm"] == "argon2 chacha20-poly1305"

src/borg/testsuite/archiver/rcompress_cmd.py

@@ -1,10 +1,10 @@
 import os
-from binascii import hexlify
 
 from ...constants import *  # NOQA
 from ...repository import Repository
 from ...manifest import Manifest
 from ...compress import ZSTD, ZLIB, LZ4, CNONE
+from ...helpers import bin_to_hex
 
 from . import create_regular_file, cmd, RK_ENCRYPTION
 
@@ -27,15 +27,7 @@ def check_compression(ctype, clevel, olevel):
                     )  # will also decompress according to metadata
                     m_olevel = meta.get("olevel", -1)
                     m_psize = meta.get("psize", -1)
-                    print(
-                        hexlify(id).decode(),
-                        meta["ctype"],
-                        meta["clevel"],
-                        meta["csize"],
-                        meta["size"],
-                        m_olevel,
-                        m_psize,
-                    )
+                    print(bin_to_hex(id), meta["ctype"], meta["clevel"], meta["csize"], meta["size"], m_olevel, m_psize)
                     # this is not as easy as one thinks due to the DecidingCompressor choosing the smallest of
                     # (desired compressed, lz4 compressed, not compressed).
                     assert meta["ctype"] in (ctype, LZ4.ID, CNONE.ID)

src/borg/testsuite/checksums.py

@@ -1,7 +1,5 @@
-from binascii import unhexlify
-
 from .. import checksums
-from ..helpers import bin_to_hex
+from ..helpers import bin_to_hex, hex_to_bin
 
 
 def test_xxh64():
@@ -10,7 +8,7 @@ def test_xxh64():
     assert (
         bin_to_hex(
             checksums.xxh64(
-                unhexlify(
+                hex_to_bin(
                     "6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724"
                     "fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d"
                     "cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331"