diff --git a/.github/release_checklist.md b/.github/release_checklist.md index 71b59ae..31c8744 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -3,12 +3,12 @@ Release checklist - [ ] Check [latest documentation](https://python-zlib-ng.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - [ ] Change current development version in `CHANGELOG.rst` to stable version. +- [ ] Check if the address sanitizer does not find any problems using `tox -e asan` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. - [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. - [ ] merge `main` branch back into `develop`. -- [ ] Add updated version number to develop. (`setup.py` and `src/zlib_ng/__init__.py`) - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. - [ ] Create a new release on github. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cb6dd8..0bea2c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10" - name: Install tox run: pip install tox - name: Lint @@ -39,10 +39,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 - uses: actions/setup-python@v2.2.1 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.10" - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} @@ -53,34 +53,34 @@ jobs: strategy: matrix: python-version: - - "3.8" - - "3.9" - "3.10" - "3.11" - "3.12" - - "3.13-dev" - - "pypy-3.9" + - "3.13" + - "3.14" - "pypy-3.10" + - "pypy-3.11" os: ["ubuntu-latest"] include: - - os: "macos-14" # For m1 macos + - os: "macos-latest" # For m1 macos python-version: "3.12" - os: "macos-13" # for x86 macos - python-version: "3.8" + python-version: "3.10" - os: "windows-latest" - python-version: "3.8" + python-version: "3.10" steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.2.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install build dependencies (MacOS) run: brew install make @@ -96,12 +96,12 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.10" steps: - uses: actions/checkout@v4 with: submodules: recursive - - uses: uraimo/run-on-arch-action@v2.5.0 + - uses: uraimo/run-on-arch-action@v3 name: Build & run test with: arch: none @@ -129,7 +129,7 @@ jobs: os: - "ubuntu-latest" - "macos-13" - - "macos-14" + - "macos-latest" - "windows-latest" python_version: [ "python" ] include: @@ -146,7 +146,7 @@ jobs: - name: Install requirements (universal) run: conda install zlib-ng ${{ matrix.python_version}} tox - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Run tests (dynamic link) run: tox @@ -167,7 +167,7 @@ jobs: os: - ubuntu-latest - macos-13 - - macos-14 + - macos-latest - windows-latest cibw_archs_linux: ["x86_64"] build_sdist: [true] @@ -179,7 +179,7 @@ jobs: with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v2 # Some issues where caused by higher versions. name: Install Python - name: Install cibuildwheel twine build run: python -m pip install cibuildwheel twine build @@ -187,17 +187,17 @@ jobs: run: brew install make if: runner.os == 'macOS' - name: Set MSVC developer prompt - uses: ilammy/msvc-dev-cmd@v1.6.0 + uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} - uses: docker/setup-qemu-action@v1.0.1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Build wheels run: cibuildwheel --output-dir dist env: - # Skip 32 bit, macosx_arm64 causes issues on cpython 3.8 + # Skip 32 bit, macosx_arm64 causes issues on cpython 3.9 CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_TEST_REQUIRES: "pytest" @@ -218,14 +218,14 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ZLIB_NG_BUILD_CACHE=True PYTHON_ZLIB_NG_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ZLIB_NG_BUILD_CACHE=True PYTHON_ZLIB_NG_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ZLIB_NG_BUILD_CACHE=True PYTHON_ZLIB_NG_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1b7cf28..c067325 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,22 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.0.0 +----------------- +The library has been running without issues as a dependency in quite a few +projects and is now stable enough for the first major version. + ++ Updated bundled zlib-ng to 2.2.5. ++ Python 3.14 is supported. ++ Python 3.8 and 3.9 are no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. ++ Switched to setuptools-scm for building the package rather than versioningit. ++ Test files are added to the source distribution. ++ Fix an issue where some tests failed because they ignored PYTHONPATH. + version 0.5.1 ----------------- + Fix a bug where flushing in threaded mode did not write the data to the diff --git a/MANIFEST.in b/MANIFEST.in index 6cde131..00f6a18 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,7 @@ graft src/zlib_ng/zlib-ng -prune tests prune docs prune benchmark_scripts prune .github -exclude tox.ini exclude requirements-docs.txt exclude codecov.yml exclude .readthedocs.yml diff --git a/docs/conf.py b/docs/conf.py index 528d846..61a5ca2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,9 +42,6 @@ # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' -html_theme_options = dict( - display_version=True, -) # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/pyproject.toml b/pyproject.toml index 28ae41d..531017a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,58 @@ [build-system] -requires = ["setuptools>=64", "versioningit>=1.1.0"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[tool.versioningit.vcs] -method="git" -default-tag = "v0.0.0" +[project] +name = "zlib-ng" +dynamic = ["version"] +description = "Drop-in replacement for zlib and gzip modules using zlib-ng" +license="PSF-2.0" +keywords=["zlib-ng", "zlib", "compression", "deflate", "gzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.9" # Because of setuptools version +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-zlib-ng" +urls.documentation = "https://python-zlib-ng.readthedocs.io" -[tool.versioningit.write] -file = "src/zlib_ng/_version.py" +[tool.setuptools_scm] +version_file = "src/zlib_ng/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["zlib_ng"] + +[tool.setuptools.package-data] +zlib_ng = ['*.pyi', 'py.typed', 'zlib-ng/LICENSE', 'zlib-ng/README.md'] +[tool.setuptools.exclude-package-data] +zlib_ng = [ + "*.c", + "*.h", + "zlib-ng/*/*", + "zlib-ng/*in", + "zlib-ng/.*", + "zlib-ng/*.map", + "zlib-ng/INDEX.md", + "zlib-ng/CMakeLists.txt", + "zlib-ng/PORTING.md", + "zlib-ng/configure", + "zlib-ng/*.empty", + "zlib-ng/FAQ.zlib", +] diff --git a/requirements-docs.txt b/requirements-docs.txt index 051c278..65356bf 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,4 @@ -sphinx -setuptools +# https://github.com/sphinx-doc/sphinx/issues/13415 +sphinx<8 sphinx-rtd-theme sphinx-argparse \ No newline at end of file diff --git a/setup.py b/setup.py index 8959b04..f58cbf8 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,9 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioningit - ZLIB_NG_SOURCE = os.path.join("src", "zlib_ng", "zlib-ng") SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or @@ -125,43 +123,6 @@ def build_zlib_ng(): setup( - name="zlib-ng", - version=versioningit.get_version(), - description="Drop-in replacement for zlib and gzip modules using zlib-ng", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildZlibNGExt}, - license="PSF-2.0", - keywords="zlib-ng zlib compression deflate gzip", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'zlib_ng': [ - '*.pyi', 'py.typed', - # Include zlib-ng LICENSE and other relevant files with the binary distribution. - 'zlib-ng/LICENSE.md', 'zlib-ng/README.md']}, - url="https://github.com/pycompression/python-zlib-ng", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 4 - Beta", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # Earliest version still tested. ext_modules=EXTENSIONS ) diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py index b9ec269..dee8be7 100644 --- a/src/zlib_ng/gzip_ng_threaded.py +++ b/src/zlib_ng/gzip_ng_threaded.py @@ -321,7 +321,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def _end_gzip_stream(self): + def flush(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -329,22 +329,17 @@ def _end_gzip_stream(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() - # Write an empty deflate block with a lost block marker. + self.raw.flush() + + def close(self): + if self._closed: + return self.raw.write(zlib_ng.compress(b"", wbits=-15)) trailer = struct.pack(" None: - if self._closed: - return - self._end_gzip_stream() self.stop() if self.exception: self.raw.close() diff --git a/src/zlib_ng/zlib-ng b/src/zlib_ng/zlib-ng index 2bc6688..4254390 160000 --- a/src/zlib_ng/zlib-ng +++ b/src/zlib_ng/zlib-ng @@ -1 +1 @@ -Subproject commit 2bc66887ddc0c50776811a27be68e34430d665e1 +Subproject commit 425439062b114a0f6cf625022c41d929c7e879f9 diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index d938966..8ad5441 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -804,6 +804,19 @@ def test_decompress_stdin_stdout(self): self.assertEqual(err, b'') self.assertEqual(out, self.data) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run usung the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + @create_and_remove_directory(TEMPDIR) def test_decompress_infile_outfile(self): gzipname = os.path.join(TEMPDIR, 'testgzip.gz') @@ -812,7 +825,7 @@ def test_decompress_infile_outfile(self): with gzip.open(gzipname, mode='wb') as fp: fp.write(self.data) rc, out, err = assert_python_ok('-m', 'zlib_ng.gzip_ng', '-d', - gzipname) + gzipname, **{'_dummy': '1'}) with open(os.path.join(TEMPDIR, "testgzip"), "rb") as gunziped: self.assertEqual(gunziped.read(), self.data) @@ -824,7 +837,7 @@ def test_decompress_infile_outfile(self): def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'zlib_ng.gzip_ng', '-d', - 'thisisatest.out') + 'thisisatest.out', **{'_dummy': '1'}) self.assertIn(b"filename doesn't end in .gz: 'thisisatest.out'", err.strip()) self.assertEqual(rc, 1) @@ -849,7 +862,7 @@ def test_compress_infile_outfile_default(self): fp.write(self.data) rc, out, err = assert_python_ok('-m', 'zlib_ng.gzip_ng', - local_testgzip) + local_testgzip, **{'_dummy': '1'}) self.assertTrue(os.path.exists(gzipname)) self.assertEqual(out, b'') @@ -867,7 +880,8 @@ def test_compress_infile_outfile(self): fp.write(self.data) rc, out, err = assert_python_ok('-m', 'zlib_ng.gzip_ng', - compress_level, local_testgzip) + compress_level, local_testgzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(gzipname)) self.assertEqual(out, b'') @@ -877,7 +891,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'zlib_ng.gzip_ng', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -9/--best: not allowed with argument -1/--fast", err) @@ -885,7 +899,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'zlib_ng.gzip_ng', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument -1/--fast', err) diff --git a/tests/test_gzip_ng_threaded.py b/tests/test_gzip_ng_threaded.py index b11ddb9..b976419 100644 --- a/tests/test_gzip_ng_threaded.py +++ b/tests/test_gzip_ng_threaded.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import zlib from pathlib import Path import pytest @@ -234,15 +235,29 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() test_file = tmp_path / "output.gz" with gzip_ng_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"1" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" f.write(b"2") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"12" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" f.write(b"3") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"123" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" diff --git a/tox.ini b/tox.ini index 06b7f91..14409e4 100644 --- a/tox.ini +++ b/tox.ini @@ -23,8 +23,10 @@ commands = [testenv:asan] setenv= PYTHONDEVMODE=1 + PYTHONUNBUFFERED=1 PYTHONMALLOC=malloc - CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer + CFLAGS=-lasan -fsanitize=address -fno-omit-frame-pointer -Og -g + ASAN_OPTIONS=log_path=asan_errors allowlist_externals=bash commands= bash -c 'export LD_PRELOAD=$(gcc -print-file-name=libasan.so) && printenv LD_PRELOAD && python -c "from zlib_ng import zlib_ng" && pytest tests'