Merge pull request #7740 from ThomasWaldmann/black-and-flake8

simplify flake8 config, fix some complaints

setup.cfg

@@ -76,112 +76,33 @@ markers =
 
 [flake8]
 # for reference ...
-#   E121 continuation line under-indented for hanging indent
-#   E122 continuation line missing indentation or outdented
-#   E123 closing bracket does not match indentation of opening bracket's line
-#   E125 continuation line with same indent as next logical line
-#   E126 continuation line over-indented for hanging indent
-#   E127 continuation line over-indented for visual indent
-#   E128 continuation line under-indented for visual indent
-#   E221 multiple spaces before operator
-#   E226 missing whitespace around arithmetic operator
-#   E261 at least two spaces before inline comment
-#   E265 block comment should start with '# '
-#   E301 expected 1 blank line
-#   E305 expected 2 blank lines after class or function definition
-#   E401 multiple imports on one line
 #   E402 module level import not at top
 #   E501 line too long
-#   E722 do not use bare except
-#   E731 do not assign a lambda expression, use def
-#   E741 ambiguous variable name
 #   F401 import unused
-#   F403 from ... import * used, unable to detect undefined names
 #   F405 undefined or defined from star imports
 #   F811 redef of unused var
-#   F821 undefined name
-#   W391 blank line at end of file
 #   #### Pick either W503, or W504 - latest recommendation from pep8 is to ignore W503
 #   W503 line break before binary operator
 #   W504 line break after binary operator
 
 # borg code style guidelines:
 # Ignoring E203 due to https://github.com/PyCQA/pycodestyle/issues/373
-ignore = E226, W503, E203
+ignore = W503, E203, F405, E402
 
 # Code style violation exceptions:
 # please note that the values are adjusted so that they do not cause failures
 # with existing code. if you want to change them, you should first fix all
 # flake8 failures that appear with your change.
 per_file_ignores =
-    docs/conf.py:E121,E126,E265,E305,E401,E402
-    src/borg/archive.py:E122,E125,E127,E402,E501,F401,F405,W504
-    src/borg/archiver/__init__.py:E402,E501,E722,E741,F405
-    src/borg/archiver/_common.py:E501,F405
-    src/borg/archiver/benchmark_cmd.py:F405
-    src/borg/archiver/config_cmd.py:F405,E722
-    src/borg/archiver/create_cmd.py:E501,F405
-    src/borg/archiver/debug_cmd.py:F405
-    src/borg/archiver/delete_cmd.py:F405
-    src/borg/archiver/diff_cmd.py:F405
-    src/borg/archiver/help_cmd.py:E501,F405
-    src/borg/archiver/key_cmds.py:F405
-    src/borg/archiver/prune_cmd.py:F405
-    src/borg/archiver/rcompress_cmd.py:F405
-    src/borg/archiver/recreate_cmd.py:F405
-    src/borg/archiver/rdelete_cmd.py:F405
-    src/borg/archiver/rlist_cmd.py:E501
-    src/borg/archiver/tar_cmds.py:F405
-    src/borg/cache.py:E127,E128,E402,E501,E722,W504
-    src/borg/fuse.py:E402,E501,E722,W504
-    src/borg/fuse_impl.py:F811
-    src/borg/locking.py:E128,E501,E722
-    src/borg/manifest.py:E128,E402,E501,F405
-    src/borg/remote.py:E128,E501,F405
-    src/borg/repository.py:E126,E128,E501,F401,F405,W504
-    src/borg/upgrader.py:E501
-    src/borg/xattr.py:E402
-    src/borg/crypto/key.py:E125,E128,E402,E501,F401,F405,W504
-    src/borg/crypto/keymanager.py:E126,E128,E501,F401
-    src/borg/crypto/nonces.py:E128,E501
-    src/borg/helpers/__init__.py:F401,F405
-    src/borg/helpers/checks.py:F401
-    src/borg/helpers/errors.py:F405
-    src/borg/helpers/fs.py:F405
-    src/borg/helpers/misc.py:E402,E722,F401,F405
-    src/borg/helpers/msgpack.py:E127,F405
-    src/borg/helpers/parseformat.py:E402,E501,E741,F401,F405
-    src/borg/helpers/process.py:E402,F401,W504
-    src/borg/helpers/progress.py:E402
-    src/borg/helpers/shellpattern.py:E501
-    src/borg/platform/__init__.py:F401,F811
-    src/borg/platform/base.py:E402
-    src/borg/testsuite/__init__.py:E501,F401
-    src/borg/testsuite/archive.py:E128,W504
-    src/borg/testsuite/archiver/__init__.py:E128,E501,E722,F401,F405,F811
-    src/borg/testsuite/archiver/debug_cmds.py:E501,F405
-    src/borg/testsuite/archiver/disk_full.py:F401,F405,F811
-    src/borg/testsuite/archiver/extract_cmd.py:F405
-    src/borg/testsuite/archiver/mount_cmds.py:E501,E722
-    src/borg/testsuite/archiver/prune_cmd.py:F405
-    src/borg/testsuite/archiver/rcompress_cmd.py:F405
-    src/borg/testsuite/archiver/recreate_cmd.py:F405
-    src/borg/testsuite/archiver/return_codes.py:F401,F405,F811
-    src/borg/testsuite/benchmark.py:F401,F811
-    src/borg/testsuite/chunker.py:E501,F405
-    src/borg/testsuite/chunker_pytest.py:F401,F405
-    src/borg/testsuite/chunker_slow.py:F405
-    src/borg/testsuite/crypto.py:E126,E501,E741
-    src/borg/testsuite/file_integrity.py:F401
-    src/borg/testsuite/hashindex.py:F401
-    src/borg/testsuite/helpers.py:E126,E127,E128,E501,F401
-    src/borg/testsuite/key.py:E501,F401
-    src/borg/testsuite/locking.py:E126,E128,E501,E722,F401
-    src/borg/testsuite/patterns.py:E123
-    src/borg/testsuite/platform.py:E128,E501,F401,F811
-    src/borg/testsuite/repository.py:E128,E501,F401
-    src/borg/testsuite/shellpattern.py:E123
-    src/borg/testsuite/upgrader.py:F405
+    src/borg/archive.py:E501
+    src/borg/archiver/help_cmd.py:E501
+    src/borg/cache.py:E501
+    src/borg/helpers/__init__.py:F401
+    src/borg/platform/__init__.py:F401
+    src/borg/testsuite/archiver/disk_full.py:F811
+    src/borg/testsuite/archiver/return_codes.py:F811
+    src/borg/testsuite/benchmark.py:F811
+    src/borg/testsuite/platform.py:F811
 
 max_line_length = 120
 exclude = build,dist,.git,.idea,.cache,.tox

src/borg/archive.py

@@ -6,7 +6,7 @@
 import time
 from collections import OrderedDict, defaultdict
 from contextlib import contextmanager
-from datetime import datetime, timedelta
+from datetime import timedelta
 from functools import partial
 from getpass import getuser
 from io import BytesIO
@@ -22,7 +22,7 @@
 from . import xattr
 from .chunker import get_chunker, Chunk
 from .cache import ChunkListEntry
-from .crypto.key import key_factory, UnsupportedPayloadError, AEADKeyBase
+from .crypto.key import key_factory, UnsupportedPayloadError
 from .compress import Compressor, CompressionSpec
 from .constants import *  # NOQA
 from .crypto.low_level import IntegrityError as IntegrityErrorBase
@@ -974,8 +974,8 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
             if not self.noacls:
                 acl_set(path, item, self.numeric_ids, fd=fd)
             if not self.noxattrs and "xattrs" in item:
-                # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include
-                # the Linux capabilities in the "security.capability" attribute.
+                # chown removes Linux capabilities, so set the extended attributes at the end, after chown,
+                # since they include the Linux capabilities in the "security.capability" attribute.
                 warning = xattr.set_all(fd or path, item.xattrs, follow_symlinks=False)
                 if warning:
                     set_ec(EXIT_WARNING)

src/borg/archiver/__init__.py

@@ -570,8 +570,8 @@ def format_tb(exc):
     remote = isinstance(exc, RemoteRepository.RPCError)
     if remote:
         prefix = "Borg server: "
-        trace_back = "\n".join(prefix + l for l in exc.exception_full.splitlines())
-        sys_info = "\n".join(prefix + l for l in exc.sysinfo.splitlines())
+        trace_back = "\n".join(prefix + line for line in exc.exception_full.splitlines())
+        sys_info = "\n".join(prefix + line for line in exc.sysinfo.splitlines())
     else:
         trace_back = traceback.format_exc()
         sys_info = sysinfo()

src/borg/archiver/_common.py

@@ -85,7 +85,8 @@ def with_repository(
     :param manifest: load manifest and repo_objs (key), pass them as keyword arguments
     :param cache: open cache, pass it as keyword argument (implies manifest)
     :param secure: do assert_secure after loading manifest
-    :param compatibility: mandatory if not create and (manifest or cache), specifies mandatory feature categories to check
+    :param compatibility: mandatory if not create and (manifest or cache), specifies mandatory
+           feature categories to check
     """
     # Note: with_repository decorator does not have a "key" argument (yet?)
     compatibility = compat_check(

src/borg/archiver/config_cmd.py

@@ -50,7 +50,7 @@ def repo_validate(section, name, value=None, check_value=True):
                 if check_value:
                     try:
                         bin_id = unhexlify(value)
-                    except:
+                    except:  # noqa
                         raise ValueError("Invalid value, must be 64 hex digits") from None
                     if len(bin_id) != 32:
                         raise ValueError("Invalid value, must be 64 hex digits")

src/borg/archiver/rlist_cmd.py

@@ -66,7 +66,7 @@ def build_parser_rlist(self, subparsers, common_parser, mid_common_parser):
             # Strings are left-aligned, numbers are right-aligned.
             # Note: time columns except ``isomtime``, ``isoctime`` and ``isoatime`` cannot be padded.
             $ borg rlist --format '{archive:36} {time} [{id}]{NL}' /path/to/repo
-            ArchiveFoo                           Thu, 2021-12-09 10:22:28 [0b8e9a312bef3f2f6e2d0fc110c196827786c15eba0188738e81697a7fa3b274]
+            ArchiveFoo                           Thu, 2021-12-09 10:22:28 [0b8e9...3b274]
             ...
 
         The following keys are always available:

src/borg/cache.py

@@ -534,7 +534,7 @@ def __init__(
             if sync and self.manifest.id != self.cache_config.manifest_id:
                 self.sync()
                 self.commit()
-        except:
+        except:  # noqa
             self.close()
             raise
 
@@ -895,8 +895,9 @@ def create_master_idx(chunk_idx):
 
         self.begin_txn()
         with cache_if_remote(self.repository, decrypted_cache=self.repo_objs) as decrypted_repository:
-            # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d -
-            # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk)
+            # TEMPORARY HACK:
+            # to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d -
+            # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local).
             self.do_cache = os.path.isdir(archive_path)
             self.chunks = create_master_idx(self.chunks)
 

src/borg/crypto/keymanager.py

@@ -158,7 +158,7 @@ def import_keyfile(self, args):
     def import_paperkey(self, args):
         try:
             # imported here because it has global side effects
-            import readline
+            import readline  # noqa
         except ImportError:
             print("Note: No line editing available due to missing readline support")
 

src/borg/fuse.py

@@ -52,7 +52,7 @@ def fuse_main():
     if has_pyfuse3:
         try:
             trio.run(llfuse.main)
-        except:
+        except:  # noqa
             return 1  # TODO return signal number if it was killed by signal
         else:
             return None

src/borg/fuse_impl.py

@@ -31,6 +31,6 @@
     else:
         raise RuntimeError("unknown fuse implementation in BORG_FUSE_IMPL: '%s'" % BORG_FUSE_IMPL)
 else:
-    llfuse = None
+    llfuse = None  # noqa
     has_llfuse = False
     has_pyfuse3 = False

src/borg/helpers/checks.py

@@ -1,7 +1,7 @@
 import os
 
 from .errors import Error
-from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
+from ..platformflags import is_win32
 
 
 class PythonLibcTooOld(Error):

src/borg/helpers/misc.py

@@ -13,7 +13,6 @@
 
 from . import msgpack
 from .. import __version__ as borg_version
-from .. import chunker
 
 
 def sysinfo():
@@ -35,7 +34,7 @@ def sysinfo():
         linux_distribution = None
     try:
         msgpack_version = ".".join(str(v) for v in msgpack.version)
-    except:
+    except:  # noqa
         msgpack_version = "unknown"
     from ..fuse_impl import llfuse, BORG_FUSE_IMPL
 

src/borg/helpers/parseformat.py

@@ -9,7 +9,7 @@
 import shlex
 import stat
 import uuid
-from typing import List, Dict, Set, Tuple, ClassVar, Any, TYPE_CHECKING, Literal
+from typing import Dict, Set, Tuple, ClassVar, Any, TYPE_CHECKING, Literal
 from binascii import hexlify
 from collections import Counter, OrderedDict
 from datetime import datetime, timezone
@@ -27,7 +27,6 @@
 from .. import __version__ as borg_version
 from .. import __version_tuple__ as borg_version_tuple
 from ..constants import *  # NOQA
-from ..platformflags import is_win32
 
 if TYPE_CHECKING:
     from ..item import ItemDiff
@@ -374,8 +373,8 @@ def format_archive(archive):
 
 
 def parse_stringified_list(s):
-    l = re.split(" *, *", s)
-    return [item for item in l if item != ""]
+    items = re.split(" *, *", s)
+    return [item for item in items if item != ""]
 
 
 class Location:
@@ -457,7 +456,7 @@ class Location:
         (?:file://)?                                        # optional file protocol
         (?P<path>
             (?:[a-zA-Z]:)?                                  # Drive letter followed by a colon (optional)
-            (?:[^:]+)                                       # Anything which does not contain a :, at least one character
+            (?:[^:]+)                                       # Anything which does not contain a :, at least one char
         )
         """,
         re.VERBOSE,

src/borg/helpers/process.py

@@ -1,7 +1,6 @@
 import contextlib
 import os
 import os.path
-import re
 import shlex
 import signal
 import subprocess
@@ -11,7 +10,7 @@
 
 from .. import __version__
 
-from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
+from ..platformflags import is_win32
 from ..logger import create_logger
 
 logger = create_logger()

src/borg/helpers/shellpattern.py

@@ -6,10 +6,9 @@
 def translate(pat, match_end=r"\Z"):
     """Translate a shell-style pattern to a regular expression.
 
-    The pattern may include ``**<sep>`` (<sep> stands for the platform-specific path separator; "/" on POSIX systems) for
-    matching zero or more directory levels and "*" for matching zero or more arbitrary characters with the exception of
-    any path separator. Wrap meta-characters in brackets for a literal match (i.e. "[?]" to match the literal character
-    "?").
+    The pattern may include ``**<sep>`` (<sep> stands for the platform-specific path separator; "/" on POSIX systems)
+    for matching zero or more directory levels and "*" for matching zero or more arbitrary characters except any path
+    separator. Wrap meta-characters in brackets for a literal match (i.e. "[?]" to match the literal character "?").
 
     Using match_end=regex one can give a regular expression that is used to match after the regex that is generated from
     the pattern. The default is to match the end of the string.

src/borg/locking.py

@@ -157,11 +157,11 @@ def acquire(self, timeout=None, sleep=None):
                 # should be cleaned up anyway. Try to clean up, but don't crash.
                 try:
                     os.unlink(temp_unique_name)
-                except:
+                except:  # noqa
                     pass
                 try:
                     os.rmdir(temp_path)
-                except:
+                except:  # noqa
                     pass
 
     def release(self):
@@ -417,7 +417,7 @@ def _wait_for_readers_finishing(self, remove, sleep):
                 # restore the roster state as before (undo the roster change):
                 if remove is not None:
                     self._roster.modify(remove, ADD)
-            except:
+            except:  # noqa
                 # avoid orphan lock when an exception happens here, e.g. Ctrl-C!
                 self._lock.release()
                 raise

src/borg/manifest.py

@@ -164,7 +164,8 @@ def list_considering(self, args):
         consider_checkpoints = getattr(args, "consider_checkpoints", None)
         if name is not None:
             raise Error(
-                "Giving a specific name is incompatible with options --first, --last, -a / --match-archives, and --consider-checkpoints."
+                "Giving a specific name is incompatible with options --first, --last, "
+                "-a / --match-archives, and --consider-checkpoints."
             )
         return self.list(
             sort_by=args.sort_by.split(","),

src/borg/remote.py

@@ -108,7 +108,7 @@ def __init__(self, data):
 # All method calls on the remote repository object must be allowlisted in RepositoryServer.rpc_methods and have api
 # stubs in RemoteRepository. The @api decorator on these stubs is used to set server version requirements.
 #
-# Method parameters are identified only by name and never by position. Unknown parameters are ignored by the server side.
+# Method parameters are identified only by name and never by position. Unknown parameters are ignored by the server.
 # If a new parameter is important and may not be ignored, on the client a parameter specific version requirement needs
 # to be added.
 # When parameters are removed, they need to be preserved as defaulted parameters on the client stubs so that older

src/borg/repository.py

@@ -881,13 +881,15 @@ def complete_xfer(intermediate=True):
                         #
                         # Now we crash. But only segment 2 gets deleted, while segment 1 is still around. Now key 1
                         # is suddenly undeleted (because the delete in segment 2 is now missing).
-                        # Again, note the requirement here. We delete these in the correct order that this doesn't happen,
-                        # and only if the FS materialization of these deletes is reordered or parts dropped this can happen.
-                        # In this case it doesn't cause outright corruption, 'just' an index count mismatch, which will be
-                        # fixed by borg-check --repair.
+                        # Again, note the requirement here. We delete these in the correct order that this doesn't
+                        # happen, and only if the FS materialization of these deletes is reordered or parts dropped
+                        # this can happen.
+                        # In this case it doesn't cause outright corruption, 'just' an index count mismatch, which
+                        # will be fixed by borg-check --repair.
                         #
-                        # Note that in this check the index state is the proxy for a "most definitely settled" repository state,
-                        # i.e. the assumption is that *all* operations on segments <= index state are completed and stable.
+                        # Note that in this check the index state is the proxy for a "most definitely settled"
+                        # repository state, i.e. the assumption is that *all* operations on segments <= index state
+                        # are completed and stable.
                         try:
                             new_segment, size = self.io.write_delete(key, raise_full=True)
                         except LoggedIO.SegmentFull: