From b8e3337e1f49118f7f68336eee2351b99489e0c3 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Mon, 6 Oct 2025 09:46:50 +0200 Subject: [PATCH] Replace urlretrieve with streaming download using urlopen Updated setup.py to replace urllib.request.urlretrieve with a streaming download helper based on urlopen. This avoids the drawbacks of urlretrieve, such as lack of streaming support and limited control over requests. Using urlopen allows downloads to be written in chunks, reduces memory usage, and makes it easier to extend the logic with custom headers, retries, or checksum verification in the future. This improves the reliability and security of fetching dependency tarballs during the build process. --- .github/workflows/manylinux.yml | 2 +- setup.py | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 550a8b3..04645e8 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -32,7 +32,7 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true - PYXMLSEC_LIBXML2_VERSION: 2.14.6 # Lock it to libxml2 2.14.6 until the issue with 2.15.x is resolved; e.g. https://github.com/lsh123/xmlsec/issues/948 + PYXMLSEC_LIBXML2_VERSION: 2.14.6 # Lock it to libxml2 2.14.6 to match it with lxml GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build diff --git a/setup.py b/setup.py index 94b49aa..014476c 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from distutils.version import StrictVersion as Version from pathlib import Path from urllib.parse import urljoin -from urllib.request import Request, urlcleanup, urlopen, urlretrieve +from urllib.request import Request, urlcleanup, urlopen from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig @@ -106,6 +106,16 @@ def latest_xmlsec_release(): return tar_gz['browser_download_url'] +def download_lib(url, filename): + req = Request(url, headers={'User-Agent': 'python-xmlsec build'}) + with urlopen(req) as r, open(filename, 'wb') as f: + while True: + chunk = r.read(8192) + if not chunk: + break + f.write(chunk) + + class CrossCompileInfo: def __init__(self, host, arch, compiler): self.host = host @@ -234,7 +244,7 @@ def prepare_static_build_win(self): else: self.info(f'Retrieving "{url}" to "{destfile}"') urlcleanup() # work around FTP bug 27973 in Py2.7.12+ - urlretrieve(url, str(destfile)) + download_lib(url, str(destfile)) for p in self.libs_dir.glob('*.zip'): with zipfile.ZipFile(str(p)) as f: @@ -297,7 +307,7 @@ def prepare_static_build(self, build_platform): else: url = f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{self.openssl_version}' self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) - urlretrieve(url, str(openssl_tar)) + download_lib(url, str(openssl_tar)) # fetch zlib zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) @@ -310,7 +320,7 @@ def prepare_static_build(self, build_platform): else: url = f'https://zlib.net/fossils/zlib-{self.zlib_version}.tar.gz' self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION={self.zlib_version}, downloading from {url}')) - urlretrieve(url, str(zlib_tar)) + download_lib(url, str(zlib_tar)) # fetch libiconv libiconv_tar = next(self.libs_dir.glob('libiconv*.tar.gz'), None) @@ -319,13 +329,13 @@ def prepare_static_build(self, build_platform): libiconv_tar = self.libs_dir / 'libiconv.tar.gz' if self.libiconv_version is None: url = latest_libiconv_release() - self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}')) + self.info('{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}')) else: url = f'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{self.libiconv_version}.tar.gz' self.info( - '{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}') + '{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}') ) - urlretrieve(url, str(libiconv_tar)) + download_lib(url, str(libiconv_tar)) # fetch libxml2 libxml2_tar = next(self.libs_dir.glob('libxml2*.tar.xz'), None) @@ -341,7 +351,7 @@ def prepare_static_build(self, build_platform): '{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION={self.libxml2_version}, downloading from {url}') ) libxml2_tar = self.libs_dir / 'libxml2.tar.xz' - urlretrieve(url, str(libxml2_tar)) + download_lib(url, str(libxml2_tar)) # fetch libxslt libxslt_tar = next(self.libs_dir.glob('libxslt*.tar.gz'), None) @@ -357,7 +367,7 @@ def prepare_static_build(self, build_platform): '{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION={self.libxslt_version}, downloading from {url}') ) libxslt_tar = self.libs_dir / 'libxslt.tar.gz' - urlretrieve(url, str(libxslt_tar)) + download_lib(url, str(libxslt_tar)) # fetch xmlsec1 xmlsec1_tar = next(self.libs_dir.glob('xmlsec1*.tar.gz'), None) @@ -372,7 +382,7 @@ def prepare_static_build(self, build_platform): '{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION={self.xmlsec1_version}, downloading from {url}') ) xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz' - urlretrieve(url, str(xmlsec1_tar)) + download_lib(url, str(xmlsec1_tar)) for file in (openssl_tar, zlib_tar, libiconv_tar, libxml2_tar, libxslt_tar, xmlsec1_tar): self.info(f'Unpacking {file.name}')