From 7ec1d31a93461924636c003e61fed34d13071608 Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Wed, 3 Apr 2024 22:27:30 +0200 Subject: [PATCH 1/3] feat(parse_header): provide our own implementation of `parse_header()` (#2217) * feat(parse_header): provide our own implementation of `parse_header()` * docs(newsfragments): add a newsfragment + address 1 review comment * test(test_mediatypes.py): add tests for multiple parameters --- README.rst | 2 +- docs/_newsfragments/2066.newandimproved.rst | 4 + docs/api/util.rst | 5 ++ docs/user/recipes/pretty-json.rst | 3 +- falcon/__init__.py | 1 + falcon/asgi/multipart.py | 5 +- falcon/media/multipart.py | 10 +-- falcon/testing/helpers.py | 4 +- falcon/util/__init__.py | 1 + falcon/util/mediatypes.py | 89 +++++++++++++++++++++ falcon/vendor/mimeparse/mimeparse.py | 4 +- pyproject.toml | 1 - tests/test_mediatypes.py | 41 ++++++++++ 13 files changed, 154 insertions(+), 16 deletions(-) create mode 100644 docs/_newsfragments/2066.newandimproved.rst create mode 100644 falcon/util/mediatypes.py create mode 100644 tests/test_mediatypes.py diff --git a/README.rst b/README.rst index 26105b0dd..738b2f2b8 100644 --- a/README.rst +++ b/README.rst @@ -1027,7 +1027,7 @@ See also: `CONTRIBUTING.md `: .. code:: python - import cgi import json import falcon @@ -66,7 +65,7 @@ implemented with a :ref:`custom media handler `: return json.loads(data.decode()) def serialize(self, media, content_type): - _, params = cgi.parse_header(content_type) + _, params = falcon.parse_header(content_type) indent = params.get('indent') if indent is not None: try: diff --git a/falcon/__init__.py b/falcon/__init__.py index 1d33539b3..745856058 100644 --- a/falcon/__init__.py +++ b/falcon/__init__.py @@ -77,6 +77,7 @@ from falcon.util import IS_64_BITS from falcon.util import is_python_func from falcon.util import misc +from falcon.util import parse_header from falcon.util import reader from falcon.util import runs_sync from falcon.util import secure_filename diff --git a/falcon/asgi/multipart.py b/falcon/asgi/multipart.py index b268c2b5a..52cb0505e 100644 --- a/falcon/asgi/multipart.py +++ b/falcon/asgi/multipart.py @@ -14,11 +14,10 @@ """ASGI multipart form media handler components.""" -import cgi - from falcon.asgi.reader import BufferedReader from falcon.errors import DelimiterError from falcon.media import multipart +from falcon.util.mediatypes import parse_header _ALLOWED_CONTENT_HEADERS = multipart._ALLOWED_CONTENT_HEADERS _CRLF = multipart._CRLF @@ -54,7 +53,7 @@ async def get_media(self): return self._media async def get_text(self): - content_type, options = cgi.parse_header(self.content_type) + content_type, options = parse_header(self.content_type) if content_type != 'text/plain': return None diff --git a/falcon/media/multipart.py b/falcon/media/multipart.py index c3fc37d56..5b55d4b4f 100644 --- a/falcon/media/multipart.py +++ b/falcon/media/multipart.py @@ -14,7 +14,6 @@ """Multipart form media handler.""" -import cgi import re from urllib.parse import unquote_to_bytes @@ -24,6 +23,7 @@ from falcon.stream import BoundedStream from falcon.util import BufferedReader from falcon.util import misc +from falcon.util.mediatypes import parse_header # TODO(vytas): @@ -249,7 +249,7 @@ def get_text(self): str: The part decoded as a text string provided the part is encoded as ``text/plain``, ``None`` otherwise. """ - content_type, options = cgi.parse_header(self.content_type) + content_type, options = parse_header(self.content_type) if content_type != 'text/plain': return None @@ -275,7 +275,7 @@ def filename(self): if self._content_disposition is None: value = self._headers.get(b'content-disposition', b'') - self._content_disposition = cgi.parse_header(value.decode()) + self._content_disposition = parse_header(value.decode()) _, params = self._content_disposition @@ -311,7 +311,7 @@ def name(self): if self._content_disposition is None: value = self._headers.get(b'content-disposition', b'') - self._content_disposition = cgi.parse_header(value.decode()) + self._content_disposition = parse_header(value.decode()) _, params = self._content_disposition self._name = params.get('name') @@ -493,7 +493,7 @@ def __init__(self, parse_options=None): def _deserialize_form( self, stream, content_type, content_length, form_cls=MultipartForm ): - _, options = cgi.parse_header(content_type) + _, options = parse_header(content_type) try: boundary = options['boundary'] except KeyError: diff --git a/falcon/testing/helpers.py b/falcon/testing/helpers.py index 00959495a..39e8c12f8 100644 --- a/falcon/testing/helpers.py +++ b/falcon/testing/helpers.py @@ -23,7 +23,6 @@ """ import asyncio -import cgi from collections import defaultdict from collections import deque import contextlib @@ -51,6 +50,7 @@ from falcon.constants import SINGLETON_HEADERS import falcon.request from falcon.util import uri +from falcon.util.mediatypes import parse_header # NOTE(kgriffs): Changed in 3.0 from 'curl/7.24.0 (x86_64-apple-darwin12.0)' DEFAULT_UA = 'falcon-client/' + falcon.__version__ @@ -802,7 +802,7 @@ def get_encoding_from_headers(headers): if not content_type: return None - content_type, params = cgi.parse_header(content_type) + content_type, params = parse_header(content_type) if 'charset' in params: return params['charset'].strip('\'"') diff --git a/falcon/util/__init__.py b/falcon/util/__init__.py index 3fec8b06e..1cead07e8 100644 --- a/falcon/util/__init__.py +++ b/falcon/util/__init__.py @@ -29,6 +29,7 @@ from falcon.util.deprecation import deprecated from falcon.util.deprecation import deprecated_args from falcon.util.deprecation import DeprecatedWarning +from falcon.util.mediatypes import parse_header from falcon.util.misc import code_to_http_status from falcon.util.misc import dt_to_http from falcon.util.misc import get_argnames diff --git a/falcon/util/mediatypes.py b/falcon/util/mediatypes.py new file mode 100644 index 000000000..c0dca5121 --- /dev/null +++ b/falcon/util/mediatypes.py @@ -0,0 +1,89 @@ +# Copyright 2023-2024 by Vytautas Liuolia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Media (aka MIME) type parsing and matching utilities.""" + +import typing + + +def _parse_param_old_stdlib(s): # type: ignore + while s[:1] == ';': + s = s[1:] + end = s.find(';') + while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: + end = s.find(';', end + 1) + if end < 0: + end = len(s) + f = s[:end] + yield f.strip() + s = s[end:] + + +def _parse_header_old_stdlib(line): # type: ignore + """Parse a Content-type like header. + + Return the main content-type and a dictionary of options. + + Note: + This method has been copied (almost) verbatim from CPython 3.8 stdlib. + It is slated for removal from the stdlib in 3.13. + """ + parts = _parse_param_old_stdlib(';' + line) + key = parts.__next__() + pdict = {} + for p in parts: + i = p.find('=') + if i >= 0: + name = p[:i].strip().lower() + value = p[i + 1 :].strip() + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + value = value.replace('\\\\', '\\').replace('\\"', '"') + pdict[name] = value + return key, pdict + + +def parse_header(line: str) -> typing.Tuple[str, dict]: + """Parse a Content-type like header. + + Return the main content-type and a dictionary of options. + + Args: + line: A header value to parse. + + Returns: + tuple: (the main content-type, dictionary of options). + + Note: + This function replaces an equivalent method previously available in the + stdlib as ``cgi.parse_header()``. + It was removed from the stdlib in Python 3.13. + """ + if '"' not in line and '\\' not in line: + key, semicolon, parts = line.partition(';') + if not semicolon: + return (key.strip(), {}) + + pdict = {} + for part in parts.split(';'): + name, equals, value = part.partition('=') + if equals: + pdict[name.strip().lower()] = value.strip() + + return (key.strip(), pdict) + + return _parse_header_old_stdlib(line) + + +__all__ = ['parse_header'] diff --git a/falcon/vendor/mimeparse/mimeparse.py b/falcon/vendor/mimeparse/mimeparse.py index 0218553cf..f96e63384 100755 --- a/falcon/vendor/mimeparse/mimeparse.py +++ b/falcon/vendor/mimeparse/mimeparse.py @@ -1,4 +1,4 @@ -import cgi +from falcon.util.mediatypes import parse_header __version__ = '1.6.0' __author__ = 'Joe Gregorio' @@ -23,7 +23,7 @@ def parse_mime_type(mime_type): :rtype: (str,str,dict) """ - full_type, params = cgi.parse_header(mime_type) + full_type, params = parse_header(mime_type) # Java URLConnection class sends an Accept header that includes a # single '*'. Turn it into a legal wildcard. if full_type == '*': diff --git a/pyproject.toml b/pyproject.toml index 5ed0c5fab..44a829feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,7 +94,6 @@ filterwarnings = [ "ignore:Using or importing the ABCs:DeprecationWarning", "ignore:cannot collect test class 'TestClient':pytest.PytestCollectionWarning", "ignore:inspect.getargspec\\(\\) is deprecated:DeprecationWarning", - "ignore:.cgi. is deprecated and slated for removal:DeprecationWarning", "ignore:path is deprecated\\. Use files\\(\\) instead:DeprecationWarning", "ignore:This process \\(.+\\) is multi-threaded", ] diff --git a/tests/test_mediatypes.py b/tests/test_mediatypes.py new file mode 100644 index 000000000..0fae79b43 --- /dev/null +++ b/tests/test_mediatypes.py @@ -0,0 +1,41 @@ +import pytest + +from falcon.util import mediatypes + + +@pytest.mark.parametrize( + 'value,expected', + [ + ('', ('', {})), + ('strange', ('strange', {})), + ('text/plain', ('text/plain', {})), + ('text/plain ', ('text/plain', {})), + (' text/plain', ('text/plain', {})), + (' text/plain ', ('text/plain', {})), + (' text/plain ', ('text/plain', {})), + ( + 'falcon/peregrine; key1; key2=value; key3', + ('falcon/peregrine', {'key2': 'value'}), + ), + ( + 'audio/pcm;rate=48000;encoding=float;bits=32', + ('audio/pcm', {'bits': '32', 'encoding': 'float', 'rate': '48000'}), + ), + ( + 'falcon/*; genus=falco; family=falconidae; class=aves; ', + ('falcon/*', {'class': 'aves', 'family': 'falconidae', 'genus': 'falco'}), + ), + ('"falcon/peregrine" ; key="value"', ('"falcon/peregrine"', {'key': 'value'})), + ('falcon/peregrine; empty=""', ('falcon/peregrine', {'empty': ''})), + ('falcon/peregrine; quote="', ('falcon/peregrine', {'quote': '"'})), + ('text/plain; charset=utf-8', ('text/plain', {'charset': 'utf-8'})), + ('stuff/strange; missing-value; missing-another', ('stuff/strange', {})), + ('stuff/strange; missing-value\\missing-another', ('stuff/strange', {})), + ( + 'application/falcon; P1 = "key; value"; P2="\\""', + ('application/falcon', {'p1': 'key; value', 'p2': '"'}), + ), + ], +) +def test_parse_header(value, expected): + assert mediatypes.parse_header(value) == expected From 64af3cfd0aa9c74c160560e8089a5a03699dfd58 Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Fri, 5 Apr 2024 22:33:42 +0200 Subject: [PATCH 2/3] chore(typing): add more descriptive typing to `Context` (#2226) * chore(typing): add more descriptive typing to `Context` * chore(typing): define only stubs when under `TYPE_CHECKING` --- falcon/util/structures.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/falcon/util/structures.py b/falcon/util/structures.py index fc7ba2a88..5a9e51176 100644 --- a/falcon/util/structures.py +++ b/falcon/util/structures.py @@ -37,6 +37,7 @@ from typing import KeysView from typing import Optional from typing import Tuple +from typing import TYPE_CHECKING from typing import ValuesView @@ -141,6 +142,19 @@ class Context: True """ + # NOTE(vytas): Define synthetic attr access methods (under TYPE_CHECKING) + # merely to let mypy know this is a namespace object. + if TYPE_CHECKING: + + def __getattr__(self, name: str) -> Any: + ... + + def __setattr__(self, name: str, value: Any) -> None: + ... + + def __delattr__(self, name: str) -> None: + ... + def __contains__(self, key: str) -> bool: return self.__dict__.__contains__(key) From 52422170cb72b3776f80bdc18ee04488178fb854 Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Wed, 10 Apr 2024 13:22:38 +0200 Subject: [PATCH 3/3] chore: make CPython 3.12 default CI version (#2227) * chore: make CPython 3.12 default CI version * chore(CI): update forgotten Windows, macOS & cov gates * chore: try codecov v4 --- .github/workflows/tests.yaml | 22 +++++++++++----------- CONTRIBUTING.md | 2 +- docs/changes/4.0.0.rst | 2 +- tox.ini | 12 ++++++------ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6a0ddbe40..cdb9d9cfd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: python-version: - - "3.10" + - "3.12" os: - "ubuntu-latest" toxenv: @@ -28,9 +28,9 @@ jobs: - "pep8-docstrings" - "mypy" - "mypy_tests" - - "py310" - - "py310_sans_msgpack" - - "py310_cython" + - "py312" + - "py312_sans_msgpack" + - "py312_cython" - "docs" - "towncrier" - "look" @@ -82,12 +82,12 @@ jobs: - python-version: "3.12" os: ubuntu-latest toxenv: py312_cython - - python-version: "3.10" + - python-version: "3.12" os: macos-latest - toxenv: py310_nocover - - python-version: "3.10" + toxenv: py312_nocover + - python-version: "3.12" os: windows-latest - toxenv: py310_nocover + toxenv: py312_nocover # These env require 3.8 and 20.04, see tox.ini - python-version: "3.8" os: ubuntu-20.04 @@ -133,14 +133,14 @@ jobs: run: tox -e ${{ matrix.toxenv }} - name: Combine coverage - if: ${{ matrix.toxenv == 'py310' || matrix.toxenv == 'py310_sans_msgpack' }} + if: ${{ matrix.toxenv == 'py312' || matrix.toxenv == 'py312_sans_msgpack' }} run: | coverage --version coverage combine - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - if: ${{ matrix.toxenv == 'py310' || matrix.toxenv == 'py310_sans_msgpack' }} + uses: codecov/codecov-action@v4 + if: ${{ matrix.toxenv == 'py312' || matrix.toxenv == 'py312_sans_msgpack' }} with: env_vars: PYTHON fail_ci_if_error: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e3d8082f..b28054167 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ $ pip install -U blue $ blue . ``` -You can check all this by running ``tox`` from within the Falcon project directory. Your environment must be based on CPython 3.8, 3.10 or 3.11: +You can check all this by running ``tox`` from within the Falcon project directory. Your environment must be based on CPython 3.8, 3.10, 3.11 or 3.12: ```bash $ pip install -U tox diff --git a/docs/changes/4.0.0.rst b/docs/changes/4.0.0.rst index 93359c678..361c17619 100644 --- a/docs/changes/4.0.0.rst +++ b/docs/changes/4.0.0.rst @@ -13,7 +13,7 @@ Changes to Supported Platforms ------------------------------ - CPython 3.11 is now fully supported. (`#2072 `__) -- CPython 3.12 will be fully supported. (`#2196 `__) +- CPython 3.12 is now fully supported. (`#2196 `__) - End-of-life Python 3.5 & 3.6 are no longer supported. (`#2074 `__) - Python 3.7 is no longer actively supported, but the framework should still continue to install from source. We may remove the support for 3.7 altogether diff --git a/tox.ini b/tox.ini index 1715258c5..67d9d7c7d 100644 --- a/tox.ini +++ b/tox.ini @@ -99,21 +99,21 @@ deps = {[testenv]deps} commands = pip uninstall --yes msgpack coverage run -m pytest tests -k "test_ws and test_msgpack_missing" -[testenv:py310] -basepython = python3.10 +[testenv:py312] +basepython = python3.12 deps = {[testenv]deps} pytest-randomly jsonschema commands = {[with-coverage]commands} -[testenv:py310_sans_msgpack] -basepython = python3.10 +[testenv:py312_sans_msgpack] +basepython = python3.12 deps = {[testenv]deps} commands = pip uninstall --yes msgpack coverage run -m pytest tests -k "test_ws and test_msgpack_missing" -[testenv:py310_nocover] -basepython = python3.10 +[testenv:py312_nocover] +basepython = python3.12 deps = {[testenv]deps} pytest-randomly jsonschema