[feature] Add support for zlib files #741
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: '**' | |
| tags-ignore: '**' | |
| pull_request: | |
| jobs: | |
| Static-Code-Checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install pip Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --user fusepy pytest | |
| - name: Style Check With Black | |
| run: | | |
| python3 -m pip install black | |
| black -q --diff --line-length 120 --skip-string-normalization ratarmount.py tests/*.py core/ratarmountcore/*.py > black.diff | |
| if [ -s black.diff ]; then | |
| cat black.diff | |
| exit 123 | |
| fi | |
| - name: Lint With Flake8 | |
| run: | | |
| python3 -m pip install flake8 | |
| flake8 --config tests/.flake8 *.py tests/*.py core/ratarmountcore/[^_]*.py | |
| - name: Lint With Pylint | |
| run: | | |
| python3 -m pip install pylint | |
| pylint --rcfile tests/.pylintrc *.py tests/*.py core/ratarmountcore/*.py | tee pylint.log | |
| ! 'egrep' ': E[0-9]{4}: ' pylint.log | |
| - name: Lint With Pytype | |
| run: | | |
| python3 -m pip install pytype | |
| pytype -d import-error -P$( cd core && pwd ):$( pwd ) ratarmount.py core/ratarmountcore/*.py | |
| - name: Lint With Mypy | |
| run: | | |
| yes | python3 -m pip install --upgrade-strategy eager --upgrade types-dataclasses mypy | |
| mypy --config-file tests/.mypy.ini *.py core/ratarmountcore/*.py | |
| yes | python3 -m pip uninstall types-dataclasses | |
| - name: Lint With ShellCheck | |
| run: | | |
| sudo apt-get -y install shellcheck | |
| shellcheck -e SC2064 tests/*.sh | |
| Tests: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [macos-latest, ubuntu-latest] | |
| # ToDo: Add windows-latest but it requires a lot of setup of the dependencies! | |
| # Maybe only test ratarmount-core without most dependencies after I have split that off. | |
| # Oldest and newest versions should be enough. Python versions are supported 5 years from release date. | |
| # 3.5 was released 2015-09-13 and end-of-life was 2020-09-13 | |
| # 3.6 was released 2016-12-23 and end-of-life was 2021-12-23 | |
| # 3.7 was released 2018-06-27 and end-of-life was 2023-06-27 | |
| # 3.8 was released 2019-10-14 and end-of-life will be 2024-10-14 | |
| # 3.9 was released 2020-10-05 and end-of-life will be 2025-10-25 | |
| # 3.10 was released 2021-10-04 and end-of-life will be 2026-10-04 | |
| # 3.11 was released 2022-10-24 and end-of-life will be 2027-10 | |
| # 3.12 was released 2023-10-02 and end-of-life will be 2028-10 | |
| # 3.13 is to be released 2024-10 | |
| python-version: ['3.7', '3.12'] | |
| defaults: | |
| run: | |
| # This is especially important for windows because it seems to default to powershell | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Print System Information | |
| run: | | |
| echo "uname -a: $( uname -a )" | |
| echo "Shell: $SHELL" | |
| echo "Cores: $( nproc )" | |
| echo "Mount points:"; mount | |
| - uses: msys2/setup-msys2@v2 | |
| if: startsWith( matrix.os, 'windows' ) | |
| with: | |
| install: gcc make liblzma-devel libzstd-devel zlib-devel | |
| - name: Install Dependencies (Linux) | |
| if: startsWith( matrix.os, 'ubuntu' ) | |
| run: | | |
| # Libarchive calls the grzip, lrzip, lzop binaries for lrzip support. Others, such as bzip2, gzip, lz4, lzma, | |
| # zstd, may also call external binaries depending on how libarchive was compiled! | |
| # https://github.com/libarchive/libarchive/blob/ad5a0b542c027883d7069f6844045e6788c7d70c/libarchive/ | |
| # archive_read_support_filter_lrzip.c#L68 | |
| sudo apt-get -y install fuse bzip2 pbzip2 pixz zstd unar lrzip lzop | |
| set -x | |
| - name: Install Dependencies (MacOS) | |
| if: startsWith( matrix.os, 'macos' ) | |
| run: | | |
| # coreutils is required for the tests written in shell, e.g., for the 'realpath' command | |
| # unar is required for rar tests with passwords. By default, bsdtar is installed but that is the only | |
| # one of the three supported tools (the third is unrar) that does not support passwords. | |
| # And the error message is atrocious: | |
| # cmdline.extend(args) | |
| # TypeError: 'NoneType' object is not iterable | |
| brew install macfuse coreutils pixz pbzip2 zstd unar libarchive lrzip lzop | |
| # Add brew installation binary folder to PATH so that command line tools like zstd can be found | |
| export PATH="$PATH:/usr/local/bin" | |
| - name: Install pip Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade wheel | |
| python3 -m pip install --upgrade setuptools | |
| python3 -m pip install --upgrade-strategy eager --upgrade cython twine build zstandard fusepy cffi | |
| - name: Test Startup With Only One Compression Dependency | |
| run: | | |
| for module in indexed_gzip indexed_zstd lzmaffi python-xz; do | |
| python3 -m pip install --upgrade "$module" | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| # indexed_zstd=1.2.0 did segfault here! | |
| python3 ratarmount.py README.md || [ $? != 139 ] | |
| python3 -m pip uninstall --yes "$module" | |
| done | |
| python3 -m pip install --upgrade 'git+https://github.com/mxmlnkn/indexed_bzip2.git@master#egginfo=rapidgzip&subdirectory=python/rapidgzip' | |
| - name: Test ratarmountcore Installation From Tarball | |
| working-directory: core | |
| run: | | |
| python3 -m build | |
| twine check dist/* | |
| python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full] | |
| - name: Test Installation From Tarball | |
| run: | | |
| python3 -m build | |
| twine check dist/* | |
| python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full] | |
| - name: Test Installation From Source | |
| run: | | |
| python3 -m pip install .[full] | |
| - name: Test Simple Startup | |
| run: | | |
| ratarmount --help | |
| ratarmount --version | |
| - name: Test Simple Mount | |
| # macOS 11+ is too uptight about "security" and is not able to fully load the macfuse kernel extension. | |
| # https://github.com/actions/runner-images/issues/4731 | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| ratarmount tests/single-file.tar mimi | |
| ls -la mimi | |
| sleep 1s | |
| # MacOS does not have fusermount! | |
| ratarmount -u mimi | |
| - name: Test Startup Without Compression Dependencies | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| python3 ratarmount.py tests/simple.bz2 || [ $? != 139 ] | |
| - name: Unit Tests | |
| run: | | |
| python3 -m pip install pytest pytest-xdist | |
| for file in core/tests/test_*.py tests/test_*.py; do | |
| case "$file" in | |
| "tests/test_cli.py") | |
| if ! uname | 'grep' -q -i darwin; then | |
| # First off, n=auto seems to use the physical cores and ignores virtual ones. | |
| # Secondly, these tests scale much better than the others because most time is spent waiting for | |
| # the FUSE mount point to appear or disappear, which doesn't seem to be bottlenecked by CPU usage. | |
| python3 -X dev -W ignore::DeprecationWarning:fuse -u \ | |
| -c "import pytest, re, sys; sys.exit(pytest.console_main())" \ | |
| -n 16 --disable-warnings "$testFile" || returnError "$LINENO" 'pytest failed!' | |
| fi | |
| ;; | |
| "core/tests/test_AutoMountLayer.py"\ | |
| |"core/tests/test_BlockParallelReaders.py"\ | |
| |"core/tests/test_LibarchiveMountSource.py"\ | |
| |"core/tests/test_SQLiteIndexedTar.py") | |
| echo "$file" # pytest-xdist seems to omit the test file name | |
| pytest -n auto --disable-warnings "$file" | |
| ;; | |
| *) | |
| # Fusepy warns about usage of use_ns because the implicit behavior is deprecated. | |
| # But there has been no development to fusepy for 4 years, so I think it should be fine to ignore. | |
| pytest --disable-warnings "$file" | |
| esac | |
| done | |
| - name: Regression Tests | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| python3 tests/tests.py | |
| bash tests/runtests.sh | |
| - name: Module tests without fusepy | |
| run: | | |
| python3 -m pip uninstall -y fusepy | |
| python3 tests/tests.py |