diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 48d181e..ce6d514 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,30 +1,28 @@ # Continous Integration Workflows -This package implements different workflows for CI. +This package implements different workflows for CI, based on our organisation's common workflows. They are organised as follows. ### Documentation The `documentation` workflow triggers on any push to master, builds the documentation and pushes it to the `gh-pages` branch (if the build is successful). -It runs on `ubuntu-latest` and our lowest supported Python version, `Python 3.7`. ### Testing Suite Tests are ensured in the `tests` workflow, which triggers on all pushes. -It runs on a matrix of all supported operating systems (ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, windows-latest and macos-latest) for all supported Python versions (currently `3.7`, `3.8`, `3.9` and `3.10`). +It runs on a matrix of all supported operating systems for all supported Python versions. ### Test Coverage Test coverage is calculated in the `coverage` wokflow, which triggers on pushes to `master` and any push to a `pull request`. -It runs on `ubuntu-latest` & the lowest supported Python version (`Python 3.7`), and reports the coverage results of the test suite to `CodeClimate`, - +It reports the coverage results of the test suite to `CodeClimate`. ### Regular Testing A `cron` workflow triggers every Monday at 3am (UTC time) and runs the full testing suite, on all available operating systems and supported Python versions. -It also runs on `Python 3.x` so that newly released Python versions that would break tests are automatically detected. +It also runs on `Python 3.x` so that newly released Python versions that would break tests are automatically included. ### Publishing -Publishing to `PyPI` is done through the `publish` workflow, which triggers anytime a `release` is made of the Github repository. -It builds a `wheel`, checks it, and pushes to `PyPI` if checks are successful. \ No newline at end of file +Publishing to `PyPI` is done through the `publish` workflow, which triggers anytime a `release` is made of the GitHub repository. +It builds a `wheel`, checks it, and pushes to `PyPI` if checks are successful. diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 54f4306..b9ffe3a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,10 +1,6 @@ # Runs all tests and pushes coverage report to codeclimate name: Coverage -defaults: - run: - shell: bash - on: # Runs on all push events to master branch and any push related to a pull request push: branches: @@ -13,59 +9,7 @@ on: # Runs on all push events to master branch and any push related to a pull r jobs: coverage: - name: ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: # only lowest supported Python on latest ubuntu - os: [ubuntu-latest] - python-version: [3.7] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: '**/setup.py' - - - name: Get full Python version - id: full-python-version - run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - name: Upgrade pip, setuptools and wheel - run: python -m pip install --upgrade pip setuptools wheel - - - name: Install package - run: python -m pip install '.[test]' - - - name: Set up env for CodeClimate (push) - run: | - echo "GIT_BRANCH=${GITHUB_REF/refs\/heads\//}" >> $GITHUB_ENV - echo "GIT_COMMIT_SHA=$GITHUB_SHA" >> $GITHUB_ENV - if: github.event_name == 'push' - - - name: Set up env for CodeClimate (pull_request) - env: - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - echo "GIT_BRANCH=$GITHUB_HEAD_REF" >> $GITHUB_ENV - echo "GIT_COMMIT_SHA=$PR_HEAD_SHA" >> $GITHUB_ENV - if: github.event_name == 'pull_request' - - - name: Prepare CodeClimate binary - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - run: | - curl -LSs 'https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64' >./cc-test-reporter; - chmod +x ./cc-test-reporter - ./cc-test-reporter before-build - - name: Run all tests - run: python -m pytest --cov-report xml --cov=sdds - - - name: Push Coverage to CodeClimate - if: ${{ success() }} # only if tests were successful - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - run: ./cc-test-reporter after-build \ No newline at end of file + uses: pylhc/.github/.github/workflows/coverage.yml@master + with: + src-dir: sdds + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 92f14c4..66cf544 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,43 +1,13 @@ -# Runs all tests on master everyday at 10 am (UTC time) +# Runs all tests on master on Mondays at 3 am (UTC time) name: Cron Testing -defaults: - run: - shell: bash -on: # Runs on master branch on Mondays at 3am UTC time +on: schedule: - cron: '* 3 * * mon' jobs: - tests: - name: ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-18.04, ubuntu-20.04,, ubuntu-22.04, macos-latest, windows-latest] - # Make sure to escape 3.10 with quotes so it doesn't get interpreted as float 3.1 by GA's parser - python-version: [3.7, 3.8, 3.9, "3.10", 3.x] # crons should always run latest python hence 3.x - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: '**/setup.py' - - - name: Get full Python version - id: full-python-version - run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - name: Upgrade pip, setuptools and wheel - run: python -m pip install --upgrade pip setuptools wheel - - - name: Install package - run: python -m pip install '.[test]' - - - name: Run all tests - run: python -m pytest \ No newline at end of file + tests: + uses: pylhc/.github/.github/workflows/cron.yml@master + with: + extra-dependencies: test diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index e548f81..ace3140 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,51 +1,14 @@ # Build documentation +# The build is uploaded as artifact if the triggering event is a push for a pull request +# The build is published to github pages if the triggering event is a push to the master branch (PR merge) name: Build and upload documentation -defaults: - run: - shell: bash - -on: # Runs on any push event to master +on: # Runs on any push event in a PR or any push event to master + pull_request: push: branches: - 'master' jobs: documentation: - name: ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: # only lowest supported Python on latest ubuntu - os: [ubuntu-latest] - python-version: [3.7] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: '**/setup.py' - - - name: Get full Python version - id: full-python-version - run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - name: Upgrade pip, setuptools and wheel - run: python -m pip install --upgrade pip setuptools wheel - - - name: Install package - run: python -m pip install '.[doc]' - - - name: Build documentation - run: python -m sphinx -b html doc ./doc_build -d ./doc_build - - - name: Upload documentation to gh-pages - if: ${{ success() }} - uses: JamesIves/github-pages-deploy-action@3.7.1 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: doc_build + uses: pylhc/.github/.github/workflows/documentation.yml@master \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9bdca5b..bc21fe3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,50 +1,11 @@ # Publishes to PyPI upon creation of a release name: Upload Package to PyPI -defaults: - run: - shell: bash - on: # Runs everytime a release is added to the repository release: types: [created] jobs: deploy: - name: ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: # only lowest supported Python on latest ubuntu - os: [ubuntu-latest] - python-version: [3.7] - - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: '**/setup.py' - - - name: Get full Python version - id: full-python-version - run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - name: Upgrade pip, setuptools, wheel, build and twine - run: python -m pip install --upgrade pip setuptools wheel build twine - - - name: Build and check build - run: | - python -m build - twine check dist/* - - - name: Publish - if: ${{ success() }} - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - twine upload dist/* + uses: pylhc/.github/.github/workflows/publish.yml@master + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7fdc322..74b02fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,42 +1,17 @@ # Runs all tests -name: Tests +name: All Tests defaults: run: shell: bash -on: [push] # Runs on all push events to any branch - +on: # Runs on any push event to any branch except master (the coverage workflow takes care of that) + push: + branches-ignore: + - 'master' jobs: tests: - name: ${{ matrix.os }} / ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest] - # Make sure to escape 3.10 with quotes so it doesn't get interpreted as float 3.1 by GA's parser - python-version: [3.7, 3.8, 3.9, "3.10"] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: '**/setup.py' - - - name: Get full Python version - id: full-python-version - run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") - - - name: Upgrade pip, setuptools and wheel - run: python -m pip install --upgrade pip setuptools wheel - - - name: Install package - run: python -m pip install '.[test]' - - - name: Run basic tests - run: python -m pytest \ No newline at end of file + uses: pylhc/.github/.github/workflows/tests.yml@master + with: + extra-dependencies: test \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b182b..c7f49f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # SDDS-Files Changelog +## Version 0.4 +- Added: + - Support for reading gzipped compressed file and arbitrary compression formats if the opener abstraction is provided. + ## Version 0.3 - Added: - little endian support. diff --git a/doc/conf.py b/doc/conf.py index a09664f..ca3b849 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -88,7 +88,7 @@ def about_package(init_posixpath: pathlib.Path) -> dict: # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -113,7 +113,7 @@ def about_package(init_posixpath: pathlib.Path) -> dict: 'collapse_navigation': False, 'display_version': True, 'logo_only': True, - 'navigation_depth': 2, + 'navigation_depth': 3, } html_logo = '_static/img/omc_logo.svg' diff --git a/doc/modules/index.rst b/doc/modules/index.rst index 625c623..7e4c284 100644 --- a/doc/modules/index.rst +++ b/doc/modules/index.rst @@ -1,15 +1,18 @@ SDDS Modules -************************** +************ .. automodule:: sdds.classes :members: + :noindex: .. automodule:: sdds.reader :members: + :noindex: .. automodule:: sdds.writer :members: + :noindex: diff --git a/sdds/__init__.py b/sdds/__init__.py index 0454a13..7fc940b 100644 --- a/sdds/__init__.py +++ b/sdds/__init__.py @@ -1,12 +1,12 @@ """Exposes SddsFile, read_sdds and write_sdds directly in sdds namespace.""" -from sdds.writer import write_sdds -from sdds.reader import read_sdds from sdds.classes import SddsFile +from sdds.reader import read_sdds +from sdds.writer import write_sdds __title__ = "sdds" __description__ = "SDDS file handling." __url__ = "https://github.com/pylhc/sdds" -__version__ = "0.3.1" +__version__ = "0.4.0" __author__ = "pylhc" __author_email__ = "pylhc@github.com" __license__ = "MIT" diff --git a/sdds/reader.py b/sdds/reader.py index 6befcea..69d1709 100644 --- a/sdds/reader.py +++ b/sdds/reader.py @@ -5,36 +5,94 @@ This module contains the reading functionality of ``sdds``. It provides a high-level function to read SDDS files in different formats, and a series of helpers. """ +import gzip +import io +import os import pathlib import struct import sys -from typing import IO, Any, List, Optional, Generator, Dict, Union, Tuple, Callable, Type +from collections.abc import Callable +from contextlib import AbstractContextManager, contextmanager +from functools import partial +from typing import IO, Any, Dict, Generator, List, Optional, Tuple, Type, Union import numpy as np -from sdds.classes import (SddsFile, Column, Parameter, Definition, Array, Data, Description, - ENCODING, NUMTYPES_CAST, NUMTYPES_SIZES, get_dtype_str) +from sdds.classes import (ENCODING, NUMTYPES_CAST, NUMTYPES_SIZES, Array, + Column, Data, Definition, Description, Parameter, + SddsFile, get_dtype_str) +# ----- Providing Opener Abstractions for the Reader ----- # -def read_sdds(file_path: Union[pathlib.Path, str], endianness: str = None) -> SddsFile: +# On Python 3.8, we cannot subscript contextlib.AbstractContextManager or collections.abc.Callable, +# which became possible with PEP 585 in Python 3.9. We will check for the runtime version and simply +# not subscript if running on 3.8. The cost here is degraded typing. +# TODO: remove this conditional once Python 3.8 has reached EoL and we drop support for it +if sys.version_info < (3, 9, 0): # we're running on 3.8, which is our lowest supported + OpenerType = Callable +else: + OpenerType = Callable[[os.PathLike], AbstractContextManager[io.BufferedIOBase]] + +binary_open = partial(open, mode="rb") # default opening mode, simple sdds files +gzip_open = partial(gzip.open, mode="rb") # for gzip-compressed sdds files + + +# ----- Reader Function ----- # + +def read_sdds(file_path: Union[pathlib.Path, str], endianness: str = None, opener: OpenerType = binary_open) -> SddsFile: """ - Reads SDDS file from the specified ``file_path``. + Reads an SDDS file from the specified ``file_path``. Args: file_path (Union[pathlib.Path, str]): `Path` object to the input SDDS file. Can be a `string`, in which case it will be cast to a `Path` object. - endianness (str): Endianness of the file. Either 'big' or 'little'. - If not given, the endianness is either extracted from - the comments in the header of the file (if present) - or determined by the machine you are running on. - Binary files written by this package are all big-endian, - and contain a comment in the file. + endianness (str): Endianness of the file, either 'big' or 'little'. If not given, + the endianness is either extracted from the comments in the header of the file + (if present) or determined by the machine you are running on. Binary files + written by this package are all big-endian, and contain a comment in the file. + opener (OpenerType): Callable to open the SDDS file. Uses `open(file, mode="rb")` by + default. One can use provided openers for specific format or bring their own, see + the examples below. Returns: An `SddsFile` object containing the loaded data. + + Examples: + + To read a typical file, one can use the default options: + + .. code-block:: python + + import sdds + + data = sdds.read("some/location/to/file.sdds") + + + To read a ``gzip``-compressed file, use the provided opener function: + + .. code-block:: python + + import sdds + from sdds.reader import gzip_open + + data = sdds.read("some/location/to/file.sdds.gz", opener=gzip_open) + + To read another specific compression format, bring your own opener abstraction. + It should take a single parameter for the path-like object pointing to the file, + and return a context manager providing byte-data of the file. For instance the + `gzip_opener` from the example above is built as `functools.partial(gzip.open, mode="rb")`. + + .. code-block:: python + + import sdds + from functools import partial + from relevant_module import opening_function + + your_opener = partial(opening_function, some_option) + data = sdds.read("some/location/to/file.sdds.extension", opener=your_opener) """ file_path = pathlib.Path(file_path) - with file_path.open("rb") as inbytes: + with opener(file_path) as inbytes: if endianness is None: endianness = _get_endianness(inbytes) version, definition_list, description, data = _read_header(inbytes) diff --git a/setup.py b/setup.py index 4737748..2924abe 100644 --- a/setup.py +++ b/setup.py @@ -58,10 +58,10 @@ def about_package(init_posixpath: pathlib.Path) -> dict: "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries :: Python Modules", ], diff --git a/tests/inputs/test_file.sdds.gz b/tests/inputs/test_file.sdds.gz new file mode 100644 index 0000000..629e632 Binary files /dev/null and b/tests/inputs/test_file.sdds.gz differ diff --git a/tests/test_sdds.py b/tests/test_sdds.py index 8ca048c..ca388fe 100644 --- a/tests/test_sdds.py +++ b/tests/test_sdds.py @@ -1,26 +1,19 @@ +import io import os import pathlib -import io import struct import sys from typing import Dict -import pytest import numpy as np -from sdds.reader import ( - read_sdds, - _gen_words, - _get_def_as_dict, - _read_header, - _read_data, - _sort_definitions, -) -from sdds.writer import write_sdds, _sdds_def_as_str -from sdds.classes import (Parameter, Column, Array, - SddsFile, Definition, Include, Data, - Description, get_dtype_str, NUMTYPES, - NUMTYPES_CAST) +import pytest +from sdds.classes import (NUMTYPES, NUMTYPES_CAST, Array, Column, Data, + Definition, Description, Include, Parameter, + SddsFile, get_dtype_str) +from sdds.reader import (_gen_words, _get_def_as_dict, _read_data, + _read_header, _sort_definitions, gzip_open, read_sdds) +from sdds.writer import _sdds_def_as_str, write_sdds CURRENT_DIR = pathlib.Path(__file__).parent @@ -47,6 +40,27 @@ def test_sdds_write_read_str_input(self, _sdds_file_str, tmp_file): assert np.all(value == new_val) +class TestReadGzippedFiles: + def test_sdds_read_gzipped_file_pathlib(self, _sdds_gzipped_file_pathlib, tmp_file): + original = read_sdds(_sdds_gzipped_file_pathlib, opener=gzip_open) + write_sdds(original, tmp_file) + new = read_sdds(tmp_file) + for definition, value in original: + new_def, new_val = new[definition.name] + assert new_def.name == definition.name + assert new_def.type == definition.type + assert np.all(value == new_val) + + def test_sdds_read_gzipped_file_str(self, _sdds_gzipped_file_str, tmp_file): + original = read_sdds(_sdds_gzipped_file_str, opener=gzip_open) + write_sdds(original, tmp_file) + new = read_sdds(tmp_file) + for definition, value in original: + new_def, new_val = new[definition.name] + assert new_def.name == definition.name + assert new_def.type == definition.type + assert np.all(value == new_val) + class TestEndianness: def test_sdds_read_little_endian(self, _sdds_file_little_endian): read_sdds(_sdds_file_little_endian) @@ -277,7 +291,7 @@ def test_get_dtype(self): assert get_dtype_str(name).endswith(format_) -# Helpers +# ----- Helpers ----- # def _write_read_header(): original = Parameter(name="param1", type="str") @@ -298,6 +312,8 @@ def _header_from_dict(d: Dict[str, Dict[str, str]]) -> str: ) + ", &end\n" +# ----- Fixtures ----- # + @pytest.fixture() def _sdds_file_pathlib() -> pathlib.Path: return CURRENT_DIR / "inputs" / "test_file.sdds" @@ -308,6 +324,16 @@ def _sdds_file_str() -> str: return os.path.join(os.path.dirname(__file__), "inputs", "test_file.sdds") +@pytest.fixture() +def _sdds_gzipped_file_pathlib() -> pathlib.Path: + return CURRENT_DIR / "inputs" / "test_file.sdds.gz" + + +@pytest.fixture() +def _sdds_gzipped_file_str() -> str: + return os.path.join(os.path.dirname(__file__), "inputs", "test_file.sdds.gz") + + @pytest.fixture() def _sdds_file_little_endian() -> pathlib.Path: return CURRENT_DIR / "inputs" / "test_file_little_endian.sdds"