Chore: Unlock dependencies & update them all

.dockerignore

@@ -1,21 +1,28 @@
+# Tool caches
 **/__pycache__
-/src-ui/.vscode
-/src-ui/node_modules
-/src-ui/dist
+**/.ruff_cache/
+**/.mypy_cache/
+# Virtual environment & similar
+.venv/
+./src-ui/node_modules
+./src-ui/dist
+# IDE folders
+.idea/
+.vscode/
+./src-ui/.vscode
+# VCS
 .git
-/export
-/consume
-/media
-/data
-/docs
-.pytest_cache
-/dist
-/scripts
-/resources
+# Test related
+**/.pytest_cache
 **/tests
 **/*.spec.ts
 **/htmlcov
-/src/.pytest_cache
-.idea
-.venv/
-.vscode/
+# Local folders
+./export
+./consume
+./media
+./data
+./docs
+./dist
+./scripts
+./resources

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ on:
 env:
   # This is the version of pipenv all the steps will use
   # If changing this, change Dockerfile
-  DEFAULT_PIP_ENV_VERSION: "2023.7.23"
+  DEFAULT_PIP_ENV_VERSION: "2023.9.8"
   # This is the default version of Python to use in most steps
   # If changing this, change Dockerfile
   DEFAULT_PYTHON_VERSION: "3.9"

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: 'v3.0.0'
+    rev: 'v3.0.3'
     hooks:
       - id: prettier
         types_or:
@@ -36,11 +36,11 @@ repos:
           - markdown
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.280'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.0.287'
     hooks:
       - id: ruff
-  - repo: https://github.com/psf/black
+  - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.7.0
     hooks:
       - id: black

.python-version

@@ -1 +1 @@
-3.8.16
+3.8.17

Dockerfile

@@ -29,7 +29,7 @@ COPY Pipfile* ./
 
 RUN set -eux \
   && echo "Installing pipenv" \
-    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.7.23 \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.9.8 \
   && echo "Generating requirement.txt" \
     && pipenv requirements > requirements.txt
 
@@ -174,9 +174,6 @@ ARG TARGETVARIANT
 
 # Can be workflow provided, defaults set for manual building
 ARG JBIG2ENC_VERSION=0.29
-ARG QPDF_VERSION=11.3.0
-ARG PIKEPDF_VERSION=7.2.0
-ARG PSYCOPG2_VERSION=2.9.6
 
 # Install the built packages from the installer library images
 # These change sometimes

Pipfile

@@ -3,10 +3,6 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-[[source]]
-url = "https://www.piwheels.org/simple"
-verify_ssl = true
-name = "piwheels"
 
 [packages]
 dateparser = "~=1.1"
@@ -55,17 +51,6 @@ pdf2image = "*"
 flower = "*"
 bleach = "*"
 zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
-#
-# Packages locked due to issues (try to check if these are fixed in a release every so often)
-#
-# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
-scipy = "==1.8.1"
-# v4 brings in extra dependencies for features not used here
-reportlab = "==3.6.12"
-# Pin these until piwheels is building a newer version (see https://www.piwheels.org/project/{package}/)
-cryptography = "==40.0.1"
-pikepdf = "==7.2.0"
-pillow = "==9.5.0"
 
 [dev-packages]
 # Linting
@@ -98,7 +83,6 @@ celery-types = "*"
 django-stubs = {extras= ["compatible-mypy"], version="*"}
 types-dateparser = "*"
 types-bleach = "*"
-types-humanfriendly = "*"
 types-redis = "*"
 types-tqdm = "*"
 types-Markdown = "*"

src/documents/management/commands/document_importer.py

@@ -182,8 +182,8 @@ def _check_manifest_valid(self):
             doc_path = self.source / doc_file
             if not doc_path.exists():
                 raise CommandError(
-                    'The manifest file refers to "{}" which does not '
-                    "appear to be in the source directory.".format(doc_file),
+                    f'The manifest file refers to "{doc_file}" which does not '
+                    "appear to be in the source directory.",
                 )
             try:
                 with doc_path.open(mode="rb") as infile:

src/documents/serialisers.py

@@ -530,9 +530,9 @@ class DocumentListSerializer(serializers.Serializer):
     )
 
     def _validate_document_id_list(self, documents, name="documents"):
-        if not type(documents) == list:
+        if not isinstance(documents, list):
             raise serializers.ValidationError(f"{name} must be a list")
-        if not all(type(i) == int for i in documents):
+        if not all(isinstance(i, int) for i in documents):
             raise serializers.ValidationError(f"{name} must be a list of integers")
         count = Document.objects.filter(id__in=documents).count()
         if not count == len(documents):
@@ -565,9 +565,9 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin):
     parameters = serializers.DictField(allow_empty=True)
 
     def _validate_tag_id_list(self, tags, name="tags"):
-        if not type(tags) == list:
+        if not isinstance(tags, list):
             raise serializers.ValidationError(f"{name} must be a list")
-        if not all(type(i) == int for i in tags):
+        if not all(isinstance(i, int) for i in tags):
             raise serializers.ValidationError(f"{name} must be a list of integers")
         count = Tag.objects.filter(id__in=tags).count()
         if not count == len(tags):
@@ -932,9 +932,9 @@ class AcknowledgeTasksViewSerializer(serializers.Serializer):
 
     def _validate_task_id_list(self, tasks, name="tasks"):
         pass
-        if not type(tasks) == list:
+        if not isinstance(tasks, list):
             raise serializers.ValidationError(f"{name} must be a list")
-        if not all(type(i) == int for i in tasks):
+        if not all(isinstance(i, int) for i in tasks):
             raise serializers.ValidationError(f"{name} must be a list of integers")
         count = PaperlessTask.objects.filter(id__in=tasks).count()
         if not count == len(tasks):

src/paperless/views.py

@@ -43,7 +43,7 @@ def get_all_result_ids(self):
         if hasattr(self.page.paginator.object_list, "saved_results"):
             results_page = self.page.paginator.object_list.saved_results[0]
             if results_page is not None:
-                for i in range(0, len(results_page.results.docs())):
+                for i in range(len(results_page.results.docs())):
                     try:
                         fields = results_page.results.fields(i)
                         if "id" in fields:

src/paperless_mail/mail.py

@@ -151,7 +151,7 @@ def __init__(self, parameter: str, supports_gmail_labels: bool):
             _, self.color = parameter.split(":")
             self.color = self.color.strip()
 
-            if self.color.lower() not in APPLE_MAIL_TAG_COLORS.keys():
+            if self.color.lower() not in APPLE_MAIL_TAG_COLORS:
                 raise MailError("Not a valid AppleMail tag color.")
 
             self.keyword = None

src/paperless_mail/parsers.py

@@ -9,7 +9,7 @@
 from django.conf import settings
 from django.utils.timezone import is_naive
 from django.utils.timezone import make_aware
-from humanfriendly import format_size
+from humanize import naturalsize
 from imap_tools import MailAttachment
 from imap_tools import MailMessage
 from tika_client import TikaClient
@@ -72,7 +72,7 @@ def extract_metadata(self, document_path: Path, mime_type: str):
                 "key": "attachments",
                 "value": ", ".join(
                     f"{attachment.filename}"
-                    f"({format_size(attachment.size, binary=True)})"
+                    f"({naturalsize(attachment.size, binary=True, format='%.2f')})"
                     for attachment in mail.attachments
                 ),
             },
@@ -124,7 +124,10 @@ def build_formatted_text(mail_message: MailMessage) -> str:
             if mail_message.attachments:
                 att = []
                 for a in mail.attachments:
-                    att.append(f"{a.filename} ({format_size(a.size, binary=True)})")
+                    attachment_size = naturalsize(a.size, binary=True, format="%.2f")
+                    att.append(
+                        f"{a.filename} ({attachment_size})",
+                    )
                 fmt_text += f"Attachments: {', '.join(att)}\n\n"
 
             if mail.html:
@@ -247,7 +250,7 @@ def clean_html(text: str) -> str:
             """
             if isinstance(text, list):
                 text = "\n".join([str(e) for e in text])
-            if type(text) != str:
+            if not isinstance(text, str):
                 text = str(text)
             text = escape(text)
             text = clean(text)
@@ -275,7 +278,9 @@ def clean_html(text: str) -> str:
 
         att = []
         for a in mail.attachments:
-            att.append(f"{a.filename} ({format_size(a.size, binary=True)})")
+            att.append(
+                f"{a.filename} ({naturalsize(a.size, binary=True, format='%.2f')})",
+            )
         data["attachments"] = clean_html(", ".join(att))
         if data["attachments"]:
             data["attachments_label"] = "Attachments"