From ffcebb39c31846169451319e41ec4e79adacb06f Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Tue, 2 Jan 2024 13:24:47 +0100 Subject: [PATCH 1/5] Allow http(s) as constraint file --- piptools/scripts/options.py | 4 +- piptools/scripts/types.py | 99 +++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 piptools/scripts/types.py diff --git a/piptools/scripts/options.py b/piptools/scripts/options.py index d3e82533..8f97fd8f 100644 --- a/piptools/scripts/options.py +++ b/piptools/scripts/options.py @@ -9,6 +9,8 @@ from piptools.locations import CACHE_DIR, DEFAULT_CONFIG_FILE_NAMES from piptools.utils import UNSAFE_PACKAGES, override_defaults_from_config_file +from .types import EnhancedPath + BuildTargetT = Literal["sdist", "wheel", "editable"] ALL_BUILD_TARGETS: tuple[BuildTargetT, ...] = ( "editable", @@ -337,7 +339,7 @@ def _get_default_option(option_name: str) -> Any: constraint = click.option( "-c", "--constraint", - type=click.Path( + type=EnhancedPath( exists=True, file_okay=True, dir_okay=False, diff --git a/piptools/scripts/types.py b/piptools/scripts/types.py new file mode 100644 index 00000000..a03c43c3 --- /dev/null +++ b/piptools/scripts/types.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import typing +from gettext import gettext + +import click +import os +from typing import Union, Optional, Any + + +class EnhancedPath(click.Path): + """The ``EnhancedPath`` type extends the built-in ``click.Path`` type, to + also support URLs, for example HTTP(S). Note that ``file://`` support + """ + + def __init__(self, **kwargs: Any): + super().__init__(**kwargs) + + def convert( + self, + value: Union[str, os.PathLike[str]], + param: Optional[click.Parameter], + ctx: Optional[click.Context], + ) -> Union[str, bytes, os.PathLike[str]]: + if isinstance(value, os.PathLike) or not EnhancedPath.is_url(value): + return typing.cast( + Union[str, bytes, os.PathLike[str]], super().convert(value, param, ctx) + ) + + if EnhancedPath.is_file_scheme(value): + super().convert(EnhancedPath.file_url_to_path(value), param, ctx) + return value + + from urllib.request import urlopen + from urllib.error import HTTPError, URLError + + def handle_http_error( + http_error: HTTPError, + ) -> None: + if http_error.code == 404 and self.exists: + self.fail( + gettext("{name} {filename!r} does not exist.").format( + name=self.name.title(), filename=value + ), + param, + ctx, + ) + + if http_error.code == 403 and self.readable: + self.fail( + gettext("{name} {filename!r} is not readable.").format( + name=self.name.title(), filename=value + ), + param, + ctx, + ) + self.fail( + gettext( + "failed checking {name} {filename!r} with error code {code}." + ).format( + name=self.name.title(), + filename=value, + code=http_error.code, + ), + param, + ctx, + ) + + try: + urlopen(value) + return value + except URLError as e: + if isinstance(e, HTTPError): + handle_http_error(e) + + self.fail( + gettext("failed checking {name} {filename!r}.").format( + name=self.name.title(), + filename=value, + ), + param, + ctx, + ) + + @staticmethod + def is_url(value: str) -> bool: + return True if "://" in value else False + + @staticmethod + def is_file_scheme(url: str) -> bool: + from urllib.parse import urlparse + + return urlparse(url).scheme == "file" + + @staticmethod + def file_url_to_path(url: str) -> str: + from urllib.parse import unquote_plus, urlparse + + return unquote_plus(urlparse(url).path) From b6a5a1faa8b9dd876570084f0ee9c4dbd1961ab9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 12:26:13 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/scripts/types.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/piptools/scripts/types.py b/piptools/scripts/types.py index a03c43c3..8724b551 100644 --- a/piptools/scripts/types.py +++ b/piptools/scripts/types.py @@ -1,11 +1,11 @@ from __future__ import annotations +import os import typing from gettext import gettext +from typing import Any, Optional, Union import click -import os -from typing import Union, Optional, Any class EnhancedPath(click.Path): @@ -18,10 +18,10 @@ def __init__(self, **kwargs: Any): def convert( self, - value: Union[str, os.PathLike[str]], - param: Optional[click.Parameter], - ctx: Optional[click.Context], - ) -> Union[str, bytes, os.PathLike[str]]: + value: str | os.PathLike[str], + param: click.Parameter | None, + ctx: click.Context | None, + ) -> str | bytes | os.PathLike[str]: if isinstance(value, os.PathLike) or not EnhancedPath.is_url(value): return typing.cast( Union[str, bytes, os.PathLike[str]], super().convert(value, param, ctx) @@ -31,8 +31,8 @@ def convert( super().convert(EnhancedPath.file_url_to_path(value), param, ctx) return value - from urllib.request import urlopen from urllib.error import HTTPError, URLError + from urllib.request import urlopen def handle_http_error( http_error: HTTPError, From 4bfa442eb2d85d665886744d7c439c1f27730ec0 Mon Sep 17 00:00:00 2001 From: Honnix Date: Tue, 2 Jan 2024 13:34:04 +0100 Subject: [PATCH 3/5] Update types.py --- piptools/scripts/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/scripts/types.py b/piptools/scripts/types.py index 8724b551..ee516348 100644 --- a/piptools/scripts/types.py +++ b/piptools/scripts/types.py @@ -3,7 +3,7 @@ import os import typing from gettext import gettext -from typing import Any, Optional, Union +from typing import Any import click From 2b7ba22382b93287eef3f6d5bd5602faf1b7a2c1 Mon Sep 17 00:00:00 2001 From: Honnix Date: Tue, 2 Jan 2024 13:35:13 +0100 Subject: [PATCH 4/5] Update types.py --- piptools/scripts/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/scripts/types.py b/piptools/scripts/types.py index ee516348..12d95376 100644 --- a/piptools/scripts/types.py +++ b/piptools/scripts/types.py @@ -3,7 +3,7 @@ import os import typing from gettext import gettext -from typing import Any +from typing import Any, Union import click From 34f10ab557b077163f42ac0d06604b64fa34ac76 Mon Sep 17 00:00:00 2001 From: Hongxin Liang Date: Tue, 2 Jan 2024 15:14:13 +0100 Subject: [PATCH 5/5] Fix --- piptools/scripts/types.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piptools/scripts/types.py b/piptools/scripts/types.py index 12d95376..4a8e041e 100644 --- a/piptools/scripts/types.py +++ b/piptools/scripts/types.py @@ -24,7 +24,8 @@ def convert( ) -> str | bytes | os.PathLike[str]: if isinstance(value, os.PathLike) or not EnhancedPath.is_url(value): return typing.cast( - Union[str, bytes, os.PathLike[str]], super().convert(value, param, ctx) + Union[str, bytes, "os.PathLike[str]"], + super().convert(value, param, ctx), ) if EnhancedPath.is_file_scheme(value): @@ -67,7 +68,7 @@ def handle_http_error( ) try: - urlopen(value) + urlopen(value) # nosec return value except URLError as e: if isinstance(e, HTTPError):