Email HTML Body
\n\n" + ), + }, + { + "tag": "account:email:sent", + "user_id": stub_user.id, + "additional": { + "from_": "noreply@example.com", + "to": stub_user.email, + "subject": "Email Subject", + "redact_ip": False, + }, + }, + ) + ] + class TestUserTermsOfServiceUpdateEmail: def test_user_terms_of_service_updated( diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 496023bab443..06d4e0176df2 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -10,6 +10,7 @@ import zipfile from cgi import FieldStorage +from textwrap import dedent from unittest import mock import pretend @@ -91,6 +92,17 @@ def _get_whl_testdata(name="fake_package", version="1.0"): zfp.writestr( f"{name}-{version}.dist-info/licenses/LICENSE.APACHE", "Fake License" ) + zfp.writestr( + f"{name}-{version}.dist-info/RECORD", + dedent( + f"""\ + {name}-{version}.dist-info/METADATA, + {name}-{version}.dist-info/licenses/LICENSE.MIT, + {name}-{version}.dist-info/licenses/LICENSE.APACHE, + {name}-{version}.dist-info/RECORD, + """, + ), + ) return temp_f.getvalue() @@ -3367,7 +3379,7 @@ def test_upload_fails_with_unsupported_wheel_plat( "400 Binary wheel .* has an unsupported platform tag .*", resp.status ) - def test_upload_fails_with_missing_metadata_wheel( + def test_upload_fails_with_missing_record_wheel( self, monkeypatch, pyramid_config, db_request ): user = UserFactory.create() @@ -3413,6 +3425,278 @@ def test_upload_fails_with_missing_metadata_wheel( resp = excinfo.value + assert resp.status_code == 400 + assert re.match( + "400 Wheel .* does not contain the required RECORD file: .*", resp.status + ) + + def test_upload_warns_with_mismatched_wheel_and_zip_contents( + self, monkeypatch, pyramid_config, db_request + ): + user = UserFactory.create() + pyramid_config.testing_securitypolicy(identity=user) + db_request.user = user + db_request.user_agent = "warehouse-tests/6.6.6" + EmailFactory.create(user=user) + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + RoleFactory.create(user=user, project=project) + + temp_f = io.BytesIO() + project_name = project.normalized_name.replace("-", "_") + with zipfile.ZipFile(file=temp_f, mode="w") as zfp: + zfp.writestr("some_file", "some_data") + zfp.writestr(f"{project_name}-{release.version}.dist-info/METADATA", "") + zfp.writestr( + f"{project_name}-{release.version}.dist-info/RECORD", + f"{project_name}-{release.version}.dist-info/RECORD,", + ) + + filename = "{}-{}-cp34-none-any.whl".format( + project.normalized_name.replace("-", "_"), + release.version, + ) + filebody = temp_f.getvalue() + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": release.version, + "filetype": "bdist_wheel", + "pyversion": "cp34", + "md5_digest": hashlib.md5(filebody).hexdigest(), + "content": pretend.stub( + filename=filename, + file=io.BytesIO(filebody), + type="application/zip", + ), + } + ) + + monkeypatch.setattr( + legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None) + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + }.get(svc) + + send_email = pretend.call_recorder(lambda *a, **kw: None) + monkeypatch.setattr(legacy, "send_wheel_record_mismatch_email", send_email) + + resp = legacy.file_upload(db_request) + assert send_email.calls == [ + pretend.call( + db_request, + {user}, + project_name=project.name, + filename=filename, + ), + ] + assert resp.status_code == 200 + + def test_upload_record_does_not_warn_with_zip_dir( + self, monkeypatch, pyramid_config, db_request + ): + """ + ZIP archives can contain directory "members". + These shouldn't cause a warning, as RECORD + only contains files, not directories. + """ + + user = UserFactory.create() + pyramid_config.testing_securitypolicy(identity=user) + db_request.user = user + db_request.user_agent = "warehouse-tests/6.6.6" + EmailFactory.create(user=user) + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + RoleFactory.create(user=user, project=project) + + temp_f = io.BytesIO() + project_name = project.normalized_name.replace("-", "_") + with zipfile.ZipFile(file=temp_f, mode="w") as zfp: + zfp.mkdir("some-dir/") # Directories! + zfp.mkdir(f"{project_name}-{release.version}.dist-info/") + + zfp.writestr(f"{project_name}-{release.version}.dist-info/METADATA", "") + zfp.writestr( + f"{project_name}-{release.version}.dist-info/RECORD", + dedent( + f"""\ + {project_name}-{release.version}.dist-info/METADATA, + {project_name}-{release.version}.dist-info/RECORD, + """, + ), + ) + + filename = "{}-{}-cp34-none-any.whl".format( + project.normalized_name.replace("-", "_"), + release.version, + ) + filebody = temp_f.getvalue() + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": release.version, + "filetype": "bdist_wheel", + "pyversion": "cp34", + "md5_digest": hashlib.md5(filebody).hexdigest(), + "content": pretend.stub( + filename=filename, + file=io.BytesIO(filebody), + type="application/zip", + ), + } + ) + + monkeypatch.setattr( + legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None) + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + }.get(svc) + + send_email = pretend.call_recorder(lambda *a, **kw: None) + monkeypatch.setattr(legacy, "send_wheel_record_mismatch_email", send_email) + + resp = legacy.file_upload(db_request) + + assert send_email.calls == [] + assert resp.status_code == 200 + + def test_upload_record_does_not_warn_windows_path_separators( + self, monkeypatch, pyramid_config, db_request + ): + """ + RECORD files can use '/' or '\' for path separators. + We should handle both and not send unnecessary emails. + """ + + user = UserFactory.create() + pyramid_config.testing_securitypolicy(identity=user) + db_request.user = user + db_request.user_agent = "warehouse-tests/6.6.6" + EmailFactory.create(user=user) + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + RoleFactory.create(user=user, project=project) + + temp_f = io.BytesIO() + project_name = project.normalized_name.replace("-", "_") + with zipfile.ZipFile(file=temp_f, mode="w") as zfp: + zfp.writestr(f"{project_name}-{release.version}.dist-info/METADATA", "") + zfp.writestr( + f"{project_name}-{release.version}.dist-info/RECORD", + dedent( + f"""\ + {project_name}-{release.version}.dist-info\\METADATA, + {project_name}-{release.version}.dist-info\\RECORD, + """, + ), + ) + + filename = "{}-{}-cp34-none-any.whl".format( + project.normalized_name.replace("-", "_"), + release.version, + ) + filebody = temp_f.getvalue() + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": release.version, + "filetype": "bdist_wheel", + "pyversion": "cp34", + "md5_digest": hashlib.md5(filebody).hexdigest(), + "content": pretend.stub( + filename=filename, + file=io.BytesIO(filebody), + type="application/zip", + ), + } + ) + + monkeypatch.setattr( + legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None) + ) + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + }.get(svc) + + send_email = pretend.call_recorder(lambda *a, **kw: None) + monkeypatch.setattr(legacy, "send_wheel_record_mismatch_email", send_email) + + resp = legacy.file_upload(db_request) + + assert send_email.calls == [] + assert resp.status_code == 200 + + def test_upload_fails_with_missing_metadata_wheel( + self, monkeypatch, pyramid_config, db_request + ): + user = UserFactory.create() + pyramid_config.testing_securitypolicy(identity=user) + db_request.user = user + EmailFactory.create(user=user) + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + RoleFactory.create(user=user, project=project) + + temp_f = io.BytesIO() + project_name = project.normalized_name.replace("-", "_") + with zipfile.ZipFile(file=temp_f, mode="w") as zfp: + zfp.writestr("some_file", "some_data") + zfp.writestr( + f"{project_name}-{release.version}.dist-info/RECORD", + dedent( + f"""\ + some_file, + {project_name}-{release.version}.dist-info/RECORD, + """, + ), + ) + + filename = "{}-{}-cp34-none-any.whl".format( + project_name, + release.version, + ) + filebody = temp_f.getvalue() + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": release.version, + "filetype": "bdist_wheel", + "pyversion": "cp34", + "md5_digest": hashlib.md5(filebody).hexdigest(), + "content": pretend.stub( + filename=filename, + file=io.BytesIO(filebody), + type="application/zip", + ), + } + ) + + monkeypatch.setattr( + legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None) + ) + + with pytest.raises(HTTPBadRequest) as excinfo: + legacy.file_upload(db_request) + + resp = excinfo.value + assert resp.status_code == 400 assert re.match( "400 Wheel .* does not contain the required METADATA file: .*", resp.status diff --git a/tests/unit/utils/test_zipfiles.py b/tests/unit/utils/test_zipfiles.py new file mode 100644 index 000000000000..6e64d5b8f363 --- /dev/null +++ b/tests/unit/utils/test_zipfiles.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 + +import io +import os +import pathlib +import struct + +import pytest + +from warehouse.forklift.legacy import _is_valid_dist_file +from warehouse.utils import zipfiles + +ZIPDATA_DIR = pathlib.Path(__file__).absolute().parent / "zipdata" + + +def zippath(filename: str): + return str(ZIPDATA_DIR / filename) + + +@pytest.mark.parametrize( + ("filename", "error"), + [ + ("reject/8bitcomment.zip", "Filename not in central directory"), + ("reject/cd_extra_entry.zip", "Duplicate filename in central directory"), + ("reject/cd_missing_entry.zip", "Filename not in central directory"), + ("reject/data_descriptor_bad_crc_0.zip", "Unknown record signature"), + ("reject/dupe_eocd.zip", "Truncated central directory"), + ( + "reject/eocd64_locator_mismatch.zip", + "Mis-matched EOCD64 record and locator offset", + ), + ("reject/eocd64_non_locator.zip", "Malformed zip file"), + ("reject/eocd64_without_eocd.zip", "Malformed zip file"), + ("reject/eocd64_without_locator.zip", "Malformed zip file"), + ("reject/missing_local_file.zip", "Missing filename in local headers"), + ("reject/extra3byte.zip", "Malformed zip file"), + ("reject/non_ascii_original_name.zip", "Filename not unicode"), + ("reject/not.zip", "File is not a zip file"), + ("reject/prefix.zip", "Unknown record signature"), + ("reject/second_unicode_extra.zip", "Filename not in central directory"), + ("reject/shortextra.zip", "Corrupt extra field 7075 (size=9)"), + ("reject/suffix_not_comment.zip", "Trailing data"), + ("reject/unicode_extra_chain.zip", "Filename not in central directory"), + ("reject/wheel-1.0-py3-none-any.whl", "Duplicate filename in local headers"), + ("reject/zip64_eocd_confusion.zip", "Filename not in central directory"), + ("reject/zip64_eocd_extensible_data.zip", "Bad offset for central directory"), + ("reject/zip64_extra_csize.zip", "Malformed zip file"), + ("reject/zip64_extra_too_long.zip", "Mis-matched data size"), + ( + "reject/zip64_extra_too_short.zip", + "Corrupt zip64 extra field. Compress size not found.", + ), + ("reject/zip64_extra_usize.zip", "Malformed zip file"), + ("reject/zipinzip.zip", "Filename not in central directory"), + ], +) +def test_bad_zips(filename, error): + result = zipfiles.validate_zipfile(zippath(filename)) + assert result[0] is False, error + assert result[1] == error + + # Also test as a ZIP provided as a dist + # is rejected if uploaded. The message + # might be different, as this function + # also checks ZIP validity. + result = _is_valid_dist_file(zippath(filename), "sdist") + assert result[0] is False + + +@pytest.mark.parametrize("filename", list(os.listdir(ZIPDATA_DIR / "accept"))) +def test_good_zips(filename): + result = zipfiles.validate_zipfile(zippath(f"accept/{filename}")) + assert result[0] is True + assert result[1] is None + + +def test_local_file_header(): + # Positive case! + header = struct.pack("+ This email is notifying you of an upcoming deprecation that we have + determined may affect you as a result of your recent upload to + '{{ project_name }}'. +
+
+ In the future, PyPI will require the file contents of all newly uploaded
+ wheel distributions to match their included RECORD file.
+ Specifically, your recent upload of '{{ filename }}' has file contents
+ that do not match the included RECORD file.
+
+ Any files already uploaded can remain in place as-is and do + not need to be updated or removed. +
++ In most cases, this can be resolved by upgrading the version of your build + tooling to a later version that correctly records wheel contents. You do + not need to remove the file. +
++ Please read this + PyPI blog post + for more information. If you have questions, you can email + admin@pypi.org to communicate with the + PyPI administrators. +
+{% endblock %} diff --git a/warehouse/templates/email/wheel-record-mismatch/body.txt b/warehouse/templates/email/wheel-record-mismatch/body.txt new file mode 100644 index 000000000000..51d412e1d330 --- /dev/null +++ b/warehouse/templates/email/wheel-record-mismatch/body.txt @@ -0,0 +1,19 @@ +{# SPDX-License-Identifier: Apache-2.0 -#} + +{% extends "email/_base/body.txt" %} + +{% block content %} + +This email is notifying you of an upcoming deprecation that we have determined may affect you as a result of your recent upload to '{{ project_name }}'. + +Specifically, your recent upload of '{{ filename }}' has file contents that do not match the included RECORD file. In the future, PyPI will require the file contents of all newly uploaded wheel distributions to match their included RECORD file. + +Any files already uploaded will remain in place as-is and do not need to be updated or removed. + +In most cases, this can be resolved by upgrading the version of your build tooling to a later version that correctly records wheel contents. You do not need to remove the file. + +Please read this PyPI blog post for more information: https://blog.pypi.org/posts/2025-08-07-wheel-archive-confusion-attacks + +If you have questions, you can email admin@pypi.org to communicate with the PyPI administrators. + +{% endblock %} diff --git a/warehouse/templates/email/wheel-record-mismatch/subject.txt b/warehouse/templates/email/wheel-record-mismatch/subject.txt new file mode 100644 index 000000000000..9fe5c6dc571c --- /dev/null +++ b/warehouse/templates/email/wheel-record-mismatch/subject.txt @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: Apache-2.0 -#} + +{% extends "email/_base/subject.txt" %} + +{% block subject %}Deprecation notice for recent wheel distribution upload to '{{ project_name }}'{% endblock %} diff --git a/warehouse/utils/zipfiles.py b/warehouse/utils/zipfiles.py new file mode 100644 index 000000000000..8ca10422f3bc --- /dev/null +++ b/warehouse/utils/zipfiles.py @@ -0,0 +1,312 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +import struct +import typing +import zipfile + +RECORD_SIG_CENTRAL_DIRECTORY = b"\x50\x4b\x01\x02" +RECORD_SIG_LOCAL_FILE = b"\x50\x4b\x03\x04" +RECORD_SIG_EOCD = b"\x50\x4b\x05\x06" +RECORD_SIG_EOCD64 = b"\x50\x4b\x06\x06" +RECORD_SIG_EOCD64_LOCATOR = b"\x50\x4b\x06\x07" +RECORD_SIG_DATA_DESCRIPTOR = b"\x50\x4b\x07\x08" + + +class InvalidZipFileError(Exception): + """Internal exception used by this module""" + + +def _seek_check(fp: typing.IO[bytes], amt: int, /) -> None: + """Call seek and check that the seeked amount + is correct. Returns True if the seeked amount + is less than what is expected. + """ + if amt < 0: # pragma: no cover + raise InvalidZipFileError("Negative offset") + fp.seek(amt, os.SEEK_CUR) + + +def _read_check(fp: typing.IO[bytes], amt: int, /) -> bytes: + """Read and assert there was enough data available.""" + if amt < 0: # pragma: no cover + raise InvalidZipFileError("Negative offset") + data = fp.read(amt) + if len(data) != amt: + raise InvalidZipFileError("Malformed zip file") + return data + + +def _handle_local_file_header( + fp: typing.IO[bytes], zipfile_files_and_sizes: dict[str, int] +) -> bytes: + """ + Parses the body of a Local File header. Returns + the contained filename field of the record. + + See section 4.3.7 of APPNOTE.TXT. + """ + data = _read_check(fp, 26) + gpbf, compress_method, compressed_size, filename_size, extra_size = struct.unpack( + "