From 76555bbc3ee26bd3623f63ad365af5777397eddd Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:11:21 +0200 Subject: [PATCH 01/17] Remove requirement to update version manually from release checklist. --- .github/release_checklist.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/release_checklist.md b/.github/release_checklist.md index 71b59ae..b70ecfd 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -8,7 +8,6 @@ Release checklist from CHANGELOG.rst. - [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. - [ ] merge `main` branch back into `develop`. -- [ ] Add updated version number to develop. (`setup.py` and `src/zlib_ng/__init__.py`) - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. - [ ] Create a new release on github. From 0b329ff22c72c249dff67277075c0ec2906e054f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 25 Sep 2024 09:31:01 +0200 Subject: [PATCH 02/17] Add address sanitizer check to release checklist --- .github/release_checklist.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/release_checklist.md b/.github/release_checklist.md index b70ecfd..31c8744 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -3,6 +3,7 @@ Release checklist - [ ] Check [latest documentation](https://python-zlib-ng.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - [ ] Change current development version in `CHANGELOG.rst` to stable version. +- [ ] Check if the address sanitizer does not find any problems using `tox -e asan` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. From dbfdf8ada20d595bda6a665e0471410e1a581f42 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 16:27:32 +0200 Subject: [PATCH 03/17] Switch to setuptools-scm setup --- .github/workflows/ci.yml | 19 ++++++------- pyproject.toml | 60 ++++++++++++++++++++++++++++++++++++---- setup.py | 41 +-------------------------- 3 files changed, 64 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cb6dd8..e3b90ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2.2.1 with: - python-version: "3.8" + python-version: "3.9" - name: Install tox run: pip install tox - name: Lint @@ -39,10 +39,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2.2.1 with: - python-version: 3.8 + python-version: "3.9" - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} @@ -53,12 +53,11 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - - "3.13-dev" + - "3.13" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] @@ -66,9 +65,9 @@ jobs: - os: "macos-14" # For m1 macos python-version: "3.12" - os: "macos-13" # for x86 macos - python-version: "3.8" + python-version: "3.9" - os: "windows-latest" - python-version: "3.8" + python-version: "3.9" steps: - uses: actions/checkout@v4 with: @@ -96,7 +95,7 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.9" steps: - uses: actions/checkout@v4 with: @@ -197,7 +196,7 @@ jobs: - name: Build wheels run: cibuildwheel --output-dir dist env: - # Skip 32 bit, macosx_arm64 causes issues on cpython 3.8 + # Skip 32 bit, macosx_arm64 causes issues on cpython 3.9 CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_TEST_REQUIRES: "pytest" diff --git a/pyproject.toml b/pyproject.toml index 28ae41d..1d05808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,58 @@ [build-system] -requires = ["setuptools>=64", "versioningit>=1.1.0"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[tool.versioningit.vcs] -method="git" -default-tag = "v0.0.0" +[project] +name = "zlib-ng" +dynamic = ["version"] +description = "Drop-in replacement for zlib and gzip modules using zlib-ng" +license="PSF-2.0" +keywords=["zlib-ng", "zlib", "compression", "deflate", "gzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.9" # Because of setuptools version +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-zlib-ng" +urls.documentation = "python-zlib-ng.readthedocs.io" -[tool.versioningit.write] -file = "src/zlib_ng/_version.py" +[tool.setuptools_scm] +version_file = "src/zlib_ng/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["zlib_ng"] + +[tool.setuptools.package-data] +zlib_ng = ['*.pyi', 'py.typed', 'zlib-ng/LICENSE', 'zlib-ng/README.md'] +[tool.setuptools.exclude-package-data] +zlib_ng = [ + "*.c", + "*.h", + "zlib-ng/*/*", + "zlib-ng/*in", + "zlib-ng/.*", + "zlib-ng/*.map", + "zlib-ng/INDEX.md", + "zlib-ng/CMakeLists.txt", + "zlib-ng/PORTING.md", + "zlib-ng/configure", + "zlib-ng/*.empty", + "zlib-ng/FAQ.zlib", +] diff --git a/setup.py b/setup.py index 8959b04..f58cbf8 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,9 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioningit - ZLIB_NG_SOURCE = os.path.join("src", "zlib_ng", "zlib-ng") SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or @@ -125,43 +123,6 @@ def build_zlib_ng(): setup( - name="zlib-ng", - version=versioningit.get_version(), - description="Drop-in replacement for zlib and gzip modules using zlib-ng", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildZlibNGExt}, - license="PSF-2.0", - keywords="zlib-ng zlib compression deflate gzip", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'zlib_ng': [ - '*.pyi', 'py.typed', - # Include zlib-ng LICENSE and other relevant files with the binary distribution. - 'zlib-ng/LICENSE.md', 'zlib-ng/README.md']}, - url="https://github.com/pycompression/python-zlib-ng", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 4 - Beta", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # Earliest version still tested. ext_modules=EXTENSIONS ) From 845125ee41980090b8b66751d4fc54a48972b240 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 16:28:36 +0200 Subject: [PATCH 04/17] Use optimized build flags rather than appended flags because of setuptools change --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3b90ca..2c000ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,14 +217,14 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ZLIB_NG_BUILD_CACHE=True PYTHON_ZLIB_NG_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ZLIB_NG_BUILD_CACHE=True PYTHON_ZLIB_NG_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ZLIB_NG_BUILD_CACHE=True PYTHON_ZLIB_NG_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: From a189a3d7eed7394c7a1b8e6020b27ca1f0ee597b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 16:31:52 +0200 Subject: [PATCH 05/17] Add setuptools-scm change to CHANGELOG --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1b7cf28..8a2380b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version develop +----------------- ++ Python 3.8 is no longer supported. ++ Switched to setuptools-scm for building the package rather than versioningit. + version 0.5.1 ----------------- + Fix a bug where flushing in threaded mode did not write the data to the From 4912a4c0ec35f8cabf1a7f705eeb4d09b9f48e7e Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 16:33:17 +0200 Subject: [PATCH 06/17] Add test files to the source distribution --- CHANGELOG.rst | 1 + MANIFEST.in | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8a2380b..ffc0d0b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,7 @@ version develop ----------------- + Python 3.8 is no longer supported. + Switched to setuptools-scm for building the package rather than versioningit. ++ Test files are added to the source distribution. version 0.5.1 ----------------- diff --git a/MANIFEST.in b/MANIFEST.in index 6cde131..00f6a18 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,7 @@ graft src/zlib_ng/zlib-ng -prune tests prune docs prune benchmark_scripts prune .github -exclude tox.ini exclude requirements-docs.txt exclude codecov.yml exclude .readthedocs.yml From 54f8595c88233f625976fa906c8dcbfaccc7bb5f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 16:51:22 +0200 Subject: [PATCH 07/17] Fix documentation build. --- docs/conf.py | 3 --- requirements-docs.txt | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 528d846..61a5ca2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,9 +42,6 @@ # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' -html_theme_options = dict( - display_version=True, -) # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/requirements-docs.txt b/requirements-docs.txt index 051c278..65356bf 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ -sphinx -setuptools +# https://github.com/sphinx-doc/sphinx/issues/13415 +sphinx<8 sphinx-rtd-theme sphinx-argparse \ No newline at end of file From 6f68be903f4c2920fb5d493fb8fbae9e1f2270aa Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:14:27 +0200 Subject: [PATCH 08/17] Enable Python 3.14 support, disable Python 3.9 --- .github/workflows/ci.yml | 19 ++++++++++--------- CHANGELOG.rst | 3 ++- pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c000ea..b632581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2.2.1 with: - python-version: "3.9" + python-version: "3.10" - name: Install tox run: pip install tox - name: Lint @@ -39,10 +39,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2.2.1 with: - python-version: "3.9" + python-version: "3.10" - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} @@ -53,21 +53,21 @@ jobs: strategy: matrix: python-version: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - - "pypy-3.9" + - "3.14" - "pypy-3.10" + - "pypy-3.11" os: ["ubuntu-latest"] include: - os: "macos-14" # For m1 macos python-version: "3.12" - os: "macos-13" # for x86 macos - python-version: "3.9" + python-version: "3.10" - os: "windows-latest" - python-version: "3.9" + python-version: "3.10" steps: - uses: actions/checkout@v4 with: @@ -76,6 +76,7 @@ jobs: uses: actions/setup-python@v2.2.1 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Set MSVC developer prompt @@ -95,7 +96,7 @@ jobs: strategy: matrix: python_version: - - "3.9" + - "3.10" steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ffc0d0b..1559500 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,8 @@ Changelog version develop ----------------- -+ Python 3.8 is no longer supported. ++ Python 3.14 is supported. ++ Python 3.8 and 3.9 are no longer supported. + Switched to setuptools-scm for building the package rather than versioningit. + Test files are added to the source distribution. diff --git a/pyproject.toml b/pyproject.toml index 1d05808..6446a22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,11 @@ requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From a9f525fd631ff4669bca04bbcc6471abbebbe1fb Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:17:49 +0200 Subject: [PATCH 09/17] Update git checkouts for actions --- .github/workflows/ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b632581..9613aaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: with: submodules: recursive - name: Set up Python 3.10 - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install tox @@ -40,7 +40,7 @@ jobs: with: submodules: recursive - name: Set up Python 3.10 - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install tox and upgrade setuptools and pip @@ -73,14 +73,14 @@ jobs: with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install build dependencies (MacOS) run: brew install make @@ -101,7 +101,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - uses: uraimo/run-on-arch-action@v2.5.0 + - uses: uraimo/run-on-arch-action@v3 name: Build & run test with: arch: none @@ -146,7 +146,7 @@ jobs: - name: Install requirements (universal) run: conda install zlib-ng ${{ matrix.python_version}} tox - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Run tests (dynamic link) run: tox @@ -179,7 +179,7 @@ jobs: with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v2 # Some issues where caused by higher versions. name: Install Python - name: Install cibuildwheel twine build run: python -m pip install cibuildwheel twine build @@ -187,11 +187,11 @@ jobs: run: brew install make if: runner.os == 'macOS' - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} - uses: docker/setup-qemu-action@v1.0.1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Build wheels From 454a452d1f3b9ce6e132d1e28e040ad2d1b5c2aa Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 15:33:30 +0200 Subject: [PATCH 10/17] Add a test for correctly flushing data to disk --- tests/test_gzip_ng_threaded.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/test_gzip_ng_threaded.py b/tests/test_gzip_ng_threaded.py index b11ddb9..b976419 100644 --- a/tests/test_gzip_ng_threaded.py +++ b/tests/test_gzip_ng_threaded.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import zlib from pathlib import Path import pytest @@ -234,15 +235,29 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() test_file = tmp_path / "output.gz" with gzip_ng_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"1" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" f.write(b"2") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"12" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" f.write(b"3") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"123" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" From 175df5b63026f5e30405e062e490deb614ab8ccb Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 16:16:34 +0200 Subject: [PATCH 11/17] Fix flushing behavior for gzip_ng_threaded --- CHANGELOG.rst | 4 ++++ src/zlib_ng/gzip_ng_threaded.py | 17 ++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1559500..1c7130f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ version develop ----------------- + Python 3.14 is supported. + Python 3.8 and 3.9 are no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. + Switched to setuptools-scm for building the package rather than versioningit. + Test files are added to the source distribution. diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py index b9ec269..dee8be7 100644 --- a/src/zlib_ng/gzip_ng_threaded.py +++ b/src/zlib_ng/gzip_ng_threaded.py @@ -321,7 +321,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def _end_gzip_stream(self): + def flush(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -329,22 +329,17 @@ def _end_gzip_stream(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() - # Write an empty deflate block with a lost block marker. + self.raw.flush() + + def close(self): + if self._closed: + return self.raw.write(zlib_ng.compress(b"", wbits=-15)) trailer = struct.pack(" None: - if self._closed: - return - self._end_gzip_stream() self.stop() if self.exception: self.raw.close() From 3e393e0cc28a0f602e8909c2491d7e663f1274ce Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 9 Sep 2025 16:22:16 +0200 Subject: [PATCH 12/17] Update asan environment to better show errors --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 06b7f91..14409e4 100644 --- a/tox.ini +++ b/tox.ini @@ -23,8 +23,10 @@ commands = [testenv:asan] setenv= PYTHONDEVMODE=1 + PYTHONUNBUFFERED=1 PYTHONMALLOC=malloc - CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer + CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer -Og -g + ASAN_OPTIONS=log_path=asan_errors allowlist_externals=bash commands= bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from zlib_ng import zlib_ng" && pytest tests' From 67fe5ce17f4a63d8601e8bcef176a0fca2a21ed1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 10:47:35 +0200 Subject: [PATCH 13/17] Fix an issue where some tests failed because they ignored PYTHONPATH Thanks to @ellert --- CHANGELOG.rst | 1 + tests/test_gzip_compliance.py | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1c7130f..80b2095 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,7 @@ version develop are flushed using Z_SYNC_FLUSH. + Switched to setuptools-scm for building the package rather than versioningit. + Test files are added to the source distribution. ++ Fix an issue where some tests failed because they ignored PYTHONPATH. version 0.5.1 ----------------- diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index d938966..8ad5441 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -804,6 +804,19 @@ def test_decompress_stdin_stdout(self): self.assertEqual(err, b'') self.assertEqual(out, self.data) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run usung the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + @create_and_remove_directory(TEMPDIR) def test_decompress_infile_outfile(self): gzipname = os.path.join(TEMPDIR, 'testgzip.gz') @@ -812,7 +825,7 @@ def test_decompress_infile_outfile(self): with gzip.open(gzipname, mode='wb') as fp: fp.write(self.data) rc, out, err = assert_python_ok('-m', 'zlib_ng.gzip_ng', '-d', - gzipname) + gzipname, **{'_dummy': '1'}) with open(os.path.join(TEMPDIR, "testgzip"), "rb") as gunziped: self.assertEqual(gunziped.read(), self.data) @@ -824,7 +837,7 @@ def test_decompress_infile_outfile(self): def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'zlib_ng.gzip_ng', '-d', - 'thisisatest.out') + 'thisisatest.out', **{'_dummy': '1'}) self.assertIn(b"filename doesn't end in .gz: 'thisisatest.out'", err.strip()) self.assertEqual(rc, 1) @@ -849,7 +862,7 @@ def test_compress_infile_outfile_default(self): fp.write(self.data) rc, out, err = assert_python_ok('-m', 'zlib_ng.gzip_ng', - local_testgzip) + local_testgzip, **{'_dummy': '1'}) self.assertTrue(os.path.exists(gzipname)) self.assertEqual(out, b'') @@ -867,7 +880,8 @@ def test_compress_infile_outfile(self): fp.write(self.data) rc, out, err = assert_python_ok('-m', 'zlib_ng.gzip_ng', - compress_level, local_testgzip) + compress_level, local_testgzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(gzipname)) self.assertEqual(out, b'') @@ -877,7 +891,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'zlib_ng.gzip_ng', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -9/--best: not allowed with argument -1/--fast", err) @@ -885,7 +899,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'zlib_ng.gzip_ng', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument -1/--fast', err) From 402d8c9fda7b3e4ad163ecc542e8c8f046904570 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 10:48:21 +0200 Subject: [PATCH 14/17] Fix improper URL for readthedocs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6446a22..531017a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ "Operating System :: Microsoft :: Windows", ] urls.homepage = "https://github.com/pycompression/python-zlib-ng" -urls.documentation = "python-zlib-ng.readthedocs.io" +urls.documentation = "https://python-zlib-ng.readthedocs.io" [tool.setuptools_scm] version_file = "src/zlib_ng/_version.py" From 0baca0051cbf2cc27e442747a1fc4771fd9e5ade Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 11:02:59 +0200 Subject: [PATCH 15/17] Prepare release 1.0.0 --- CHANGELOG.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 80b2095..b063dd7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version develop +version 1.0.0 ----------------- +The library has been running without issues as a dependency in quite a few +projects and is now stable enough for the first major version. + + Python 3.14 is supported. + Python 3.8 and 3.9 are no longer supported. + Fix an issue where flushing using igzip_threaded caused a gzip end of stream From 5d4d68b9f239ea7a6023fdf5ef9756e2414348b2 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 11:03:40 +0200 Subject: [PATCH 16/17] Run on latest macos --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9613aaa..0bea2c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - "pypy-3.11" os: ["ubuntu-latest"] include: - - os: "macos-14" # For m1 macos + - os: "macos-latest" # For m1 macos python-version: "3.12" - os: "macos-13" # for x86 macos python-version: "3.10" @@ -129,7 +129,7 @@ jobs: os: - "ubuntu-latest" - "macos-13" - - "macos-14" + - "macos-latest" - "windows-latest" python_version: [ "python" ] include: @@ -167,7 +167,7 @@ jobs: os: - ubuntu-latest - macos-13 - - macos-14 + - macos-latest - windows-latest cibw_archs_linux: ["x86_64"] build_sdist: [true] From 2a3d6bfaecf0fd5ac47fcca00d5a1a0123085114 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 10 Sep 2025 11:06:16 +0200 Subject: [PATCH 17/17] Update bundled zlib-ng to 2.2.5 --- CHANGELOG.rst | 1 + src/zlib_ng/zlib-ng | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b063dd7..c067325 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,7 @@ version 1.0.0 The library has been running without issues as a dependency in quite a few projects and is now stable enough for the first major version. ++ Updated bundled zlib-ng to 2.2.5. + Python 3.14 is supported. + Python 3.8 and 3.9 are no longer supported. + Fix an issue where flushing using igzip_threaded caused a gzip end of stream diff --git a/src/zlib_ng/zlib-ng b/src/zlib_ng/zlib-ng index 2bc6688..4254390 160000 --- a/src/zlib_ng/zlib-ng +++ b/src/zlib_ng/zlib-ng @@ -1 +1 @@ -Subproject commit 2bc66887ddc0c50776811a27be68e34430d665e1 +Subproject commit 425439062b114a0f6cf625022c41d929c7e879f9