address deprecation of werkzeug.urls.url_parse (#1144)

* chore: delete __unicode__ methods

* fix: disuse deprecated functions from werkzeug.urls

* fix: disuse deprecated werkzeug.urls functions

* fix: do not override user-specified warnings filter

* fix: pin werkzeug<2.4, ignore deprecation warnings in tests

lektor/cli.py

@@ -41,7 +41,8 @@ def cli(ctx, project=None, language=None):
     This command can invoke lektor locally and serve up the website.  It's
     intended for local development of websites.
     """
-    warnings.simplefilter("default")
+    if not sys.warnoptions:
+        warnings.simplefilter("default")
     if language is not None:
         ctx.ui_lang = language
     if project is not None:

lektor/db.py

@@ -11,12 +11,12 @@
 from datetime import timedelta
 from itertools import islice
 from operator import methodcaller
+from urllib.parse import urljoin
 
 from jinja2 import is_undefined
 from jinja2 import Undefined
 from jinja2.exceptions import UndefinedError
 from jinja2.utils import LRUCache
-from werkzeug.urls import url_join
 from werkzeug.utils import cached_property
 
 from lektor import metaformat
@@ -957,8 +957,6 @@ def __str__(self):
             "frames directly, use .thumbnail()."
         )
 
-    __unicode__ = __str__
-
     @require_ffmpeg
     def thumbnail(self, width=None, height=None, mode=None, upscale=None, quality=None):
         """Utility to create thumbnails."""
@@ -1628,7 +1626,7 @@ def make_absolute_url(self, url):
                 "To use absolute URLs you need to configure "
                 "the URL in the project config."
             )
-        return url_join(base_url.rstrip("/") + "/", url.lstrip("/"))
+        return urljoin(base_url.rstrip("/") + "/", url.lstrip("/"))
 
     def make_url(self, url, base_url=None, absolute=None, external=None):
         """Helper method that creates a finalized URL based on the parameters
@@ -1646,9 +1644,9 @@ def make_url(self, url, base_url=None, absolute=None, external=None):
                     "To use absolute URLs you need to "
                     "configure the URL in the project config."
                 )
-            return url_join(external_base_url, url.lstrip("/"))
+            return urljoin(external_base_url, url.lstrip("/"))
         if absolute:
-            return url_join(self.db.config.base_path, url.lstrip("/"))
+            return urljoin(self.db.config.base_path, url.lstrip("/"))
         if base_url is None:
             raise RuntimeError(
                 "Cannot calculate a relative URL if no base " "URL has been provided."

lektor/environment/config.py

@@ -2,9 +2,9 @@
 import os
 import re
 from collections import OrderedDict
+from urllib.parse import urlsplit
 
 from inifile import IniFile
-from werkzeug.urls import url_parse
 from werkzeug.utils import cached_property
 
 from lektor.constants import PRIMARY_ALT
@@ -273,7 +273,7 @@ def primary_alternative(self):
     def base_url(self):
         """The external base URL."""
         url = self.values["PROJECT"].get("url")
-        if url and url_parse(url).scheme:
+        if url and urlsplit(url).scheme:
             return url.rstrip("/") + "/"
         return None
 
@@ -282,7 +282,7 @@ def base_path(self):
         """The base path of the URL."""
         url = self.values["PROJECT"].get("url")
         if url:
-            return url_parse(url).path.rstrip("/") + "/"
+            return urlsplit(url).path.rstrip("/") + "/"
         return "/"
 
     @cached_property

lektor/markdown.py

@@ -1,9 +1,9 @@
 import threading
+from urllib.parse import urlsplit
 from weakref import ref as weakref
 
 import mistune
 from markupsafe import Markup
-from werkzeug.urls import url_parse
 
 from lektor.context import get_ctx
 
@@ -18,7 +18,7 @@ def escape(text: str) -> str:
 class ImprovedRenderer(mistune.Renderer):
     def link(self, link, title, text):
         if self.record is not None:
-            url = url_parse(link)
+            url = urlsplit(link)
             if not url.scheme:
                 link = self.record.url_to("!" + link, base_url=get_ctx().base_url)
         link = escape(link)
@@ -29,7 +29,7 @@ def link(self, link, title, text):
 
     def image(self, src, title, text):
         if self.record is not None:
-            url = url_parse(src)
+            url = urlsplit(src)
             if not url.scheme:
                 src = self.record.url_to("!" + src, base_url=get_ctx().base_url)
         src = escape(src)

lektor/utils.py

@@ -18,14 +18,15 @@
 from pathlib import PurePosixPath
 from queue import Queue
 from threading import Thread
+from urllib.parse import urlsplit
 
 import click
 from jinja2 import is_undefined
 from markupsafe import Markup
 from slugify import slugify as _slugify
-from werkzeug import urls
 from werkzeug.http import http_date
-from werkzeug.urls import url_parse
+from werkzeug.urls import iri_to_uri
+from werkzeug.urls import uri_to_iri
 
 
 is_windows = os.name == "nt"
@@ -388,24 +389,21 @@ def wait_for_completion(self):
 
 
 class Url:
-    def __init__(self, value):
+    def __init__(self, value: str):
         self.url = value
-        u = url_parse(value)
-        i = u.to_iri_tuple()
-        self.ascii_url = str(u)
-        self.host = i.host
-        self.ascii_host = u.ascii_host
+        u = urlsplit(value)
+        i = urlsplit(uri_to_iri(u.geturl()))
+        self.ascii_url = iri_to_uri(u.geturl())
+        self.host = i.hostname
+        self.ascii_host = urlsplit(self.ascii_url).hostname
         self.port = u.port
         self.path = i.path
         self.query = u.query
         self.anchor = i.fragment
         self.scheme = u.scheme
 
-    def __unicode__(self):
-        return self.url
-
     def __str__(self):
-        return self.ascii_url
+        return self.url
 
 
 def is_unsafe_to_delete(path, base):
@@ -511,17 +509,12 @@ def is_valid_id(value):
     )
 
 
-def secure_url(url):
-    url = urls.url_parse(url)
-    if url.password is not None:
-        url = url.replace(
-            netloc="%s@%s"
-            % (
-                url.username,
-                url.netloc.split("@")[-1],
-            )
-        )
-    return url.to_url()
+def secure_url(url: str) -> str:
+    parts = urlsplit(url)
+    if parts.password is not None:
+        _, _, host_port = parts.netloc.rpartition("@")
+        parts = parts._replace(netloc=f"{parts.username}@{host_port}")
+    return parts.geturl()
 
 
 def bool_from_string(val, default=None):

setup.cfg

@@ -51,7 +51,7 @@ install_requires =
     requests
     setuptools>=45.2
     watchdog
-    Werkzeug<3
+    Werkzeug<2.4
 
 [options.extras_require]
 ipython =

tests/test_config.py

@@ -1,3 +1,50 @@
+import inspect
+
+import pytest
+
+from lektor.environment.config import Config
+
+
 def test_custom_attachment_types(env):
     attachment_types = env.load_config().values["ATTACHMENT_TYPES"]
     assert attachment_types[".foo"] == "text"
+
+
+@pytest.fixture(scope="function")
+def config(tmp_path, project_url):
+    projectfile = tmp_path / "scratch.lektorproject"
+    projectfile.write_text(
+        inspect.cleandoc(
+            f"""
+            [project]
+            url = {project_url}
+            """
+        )
+    )
+    return Config(projectfile)
+
+
+@pytest.mark.parametrize(
+    "project_url, expected",
+    [
+        ("", None),
+        ("/path/", None),
+        ("https://example.org", "https://example.org/"),
+    ],
+)
+def test_base_url(config, expected):
+    assert config.base_url == expected
+
+
+@pytest.mark.parametrize(
+    "project_url, expected",
+    [
+        ("", "/"),
+        ("/path", "/path/"),
+        ("/path/", "/path/"),
+        ("https://example.org", "/"),
+        ("https://example.org/pth", "/pth/"),
+    ],
+)
+def test_base_path(config, expected):
+    assert config.base_path == expected

tests/test_db.py

@@ -487,3 +487,30 @@ def test_Page_url_path_raise_error_if_paginated_and_dotted(scratch_pad):
 def test_Attachment_url_path_is_for_primary_alt(scratch_pad, alt):
     attachment = scratch_pad.get("/test.txt")
     assert attachment.url_path == "/en/test.txt"
+
+
+@pytest.mark.parametrize(
+    "url, base_url, absolute, external, project_url, expected",
+    [
+        ("/a/b.html", "/a/", None, None, None, "b.html"),
+        ("/a/b/", "/a/", None, None, None, "b/"),
+        ("/a/b/", "/a", None, None, None, "a/b/"),
+        ("/a/b/", "/a", True, None, None, "/a/b/"),
+        ("/a/b/", "/a", True, None, "https://example.net/pfx/", "/pfx/a/b/"),
+        ("/a/b/", "/a", None, True, "https://example.org", "https://example.org/a/b/"),
+    ],
+)
+def test_Pad_make_url(url, base_url, absolute, external, project_url, expected, pad):
+    if project_url is not None:
+        pad.db.config.values["PROJECT"]["url"] = project_url
+    assert pad.make_url(url, base_url, absolute, external) == expected
+
+
+def test_Pad_make_url_raises_runtime_error_if_no_project_url(pad):
+    with pytest.raises(RuntimeError, match="(?i)configure the url in the project"):
+        pad.make_url("/a/b", external=True)
+
+
+def test_Pad_make_url_raises_runtime_error_if_no_base_url(pad):
+    with pytest.raises(RuntimeError, match="(?i)no base url"):
+        pad.make_url("/a/b")

tests/test_deploy.py

@@ -9,6 +9,9 @@
 from lektor.publisher import RsyncPublisher
 
 
+pytestmark = pytest.mark.filterwarnings(r"ignore:'werkzeug\.urls:DeprecationWarning")
+
+
 def test_get_server(env):
     server = env.load_config().get_server("production")
     assert server.name == "Production"

tests/test_publisher.py

@@ -42,6 +42,7 @@ def test_Command_triggers_no_warnings():
     which("rsync") is None, reason="rsync is not available on this system"
 )
 @pytest.mark.parametrize("delete", ["yes", "no"])
+@pytest.mark.filterwarnings(r"ignore:'werkzeug\.urls:DeprecationWarning")
 def test_RsyncPublisher_integration(env, tmp_path, delete):
     # Integration test of local rsync deployment
     # Ensures that RsyncPublisher can successfully invoke rsync