diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml deleted file mode 100644 index f5bcec90..00000000 --- a/.azure-pipelines.yml +++ /dev/null @@ -1,167 +0,0 @@ -trigger: - branches: - include: - - '*' - tags: - include: - - '*' - -stages: -- stage: static - displayName: Static Analysis - jobs: - - job: checks - displayName: static code analysis - pool: - vmImage: ubuntu-20.04 - steps: - # Run syntax validation on a shallow clone using oldest and latest Python - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.8 - - - bash: python .azure-pipelines/syntax-validation.py - displayName: Syntax validation (3.8) - - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.11 - - - bash: python .azure-pipelines/syntax-validation.py - displayName: Syntax validation (3.11) - - # Run flake8 validation on a shallow clone - - bash: | - pip install --disable-pip-version-check flake8 - python .azure-pipelines/flake8-validation.py - displayName: Flake8 validation - -- stage: build - displayName: Build - dependsOn: - jobs: - - job: build - displayName: build package - pool: - vmImage: ubuntu-20.04 - steps: - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.11 - - - bash: | - pip install --disable-pip-version-check collective.checkdocs wheel - displayName: Install dependencies - - - bash: | - set -ex - python setup.py sdist bdist_wheel - mkdir -p dist/pypi - shopt -s extglob - mv -v dist/!(pypi) dist/pypi - git archive HEAD | gzip > dist/repo-source.tar.gz - ls -laR dist - displayName: Build python package - - - task: PublishBuildArtifacts@1 - displayName: Store artifact - inputs: - pathToPublish: dist/ - artifactName: package - - - bash: python setup.py checkdocs - displayName: Check package description - -- stage: tests - displayName: Run unit tests - dependsOn: - - static - - build - jobs: - - job: linux - pool: - vmImage: ubuntu-20.04 - strategy: - matrix: - python38: - PYTHON_VERSION: 3.8 - python39: - PYTHON_VERSION: 3.9 - python310: - PYTHON_VERSION: 3.10 - python311: - PYTHON_VERSION: 3.11 - steps: - - template: .azure-pipelines/ci.yml - - job: macOS - pool: - vmImage: macOS-latest - strategy: - matrix: - python38: - PYTHON_VERSION: 3.8 - python39: - PYTHON_VERSION: 3.9 - python310: - PYTHON_VERSION: 3.10 - python311: - PYTHON_VERSION: 3.11 - steps: - - template: .azure-pipelines/ci.yml - - job: windows - pool: - vmImage: windows-latest - strategy: - matrix: - python38: - PYTHON_VERSION: 3.8 - python39: - PYTHON_VERSION: 3.9 - python310: - PYTHON_VERSION: 3.10 - python311: - PYTHON_VERSION: 3.11 - steps: - - template: .azure-pipelines/ci.yml - -- stage: deploy - displayName: Publish release - dependsOn: - - tests - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) - jobs: - - job: pypi - displayName: Publish pypi release - pool: - vmImage: ubuntu-20.04 - steps: - - checkout: none - - - task: UsePythonVersion@0 - displayName: Set up python - inputs: - versionSpec: 3.11 - - - task: DownloadBuildArtifacts@0.206.0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - - - script: | - pip install --disable-pip-version-check twine - displayName: Install twine - - - task: TwineAuthenticate@1 - displayName: Set up credentials - inputs: - pythonUploadServiceConnection: pypi-zocalo - - - bash: | - python -m twine upload -r pypi-zocalo --config-file $(PYPIRC_PATH) $(System.ArtifactsDirectory)/package/pypi/*.tar.gz $(System.ArtifactsDirectory)/package/pypi/*.whl - displayName: Publish package diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml deleted file mode 100644 index e9f5e37c..00000000 --- a/.azure-pipelines/ci.yml +++ /dev/null @@ -1,38 +0,0 @@ -steps: -- checkout: none - -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(PYTHON_VERSION)' - displayName: 'Use Python $(PYTHON_VERSION)' - -- task: DownloadBuildArtifacts@0.206.0 - displayName: Get pre-built package - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'package' - downloadPath: '$(System.ArtifactsDirectory)' - -- task: ExtractFiles@1.200.0 - displayName: Checkout sources - inputs: - archiveFilePatterns: "$(System.ArtifactsDirectory)/package/repo-source.tar.gz" - destinationFolder: "$(Pipeline.Workspace)/src" - -- bash: | - set -eux - pip install --disable-pip-version-check wheel - pip install --disable-pip-version-check -r "$(Pipeline.Workspace)/src/requirements_dev.txt" pytest-azurepipelines - pip install --no-deps --disable-pip-version-check -e "$(Pipeline.Workspace)/src" - displayName: Install package - -- bash: PYTHONDEVMODE=1 pytest -v -ra --cov=zocalo --cov-report=xml --cov-branch - displayName: Run tests - workingDirectory: $(Pipeline.Workspace)/src - -- bash: bash <(curl -s https://codecov.io/bash) -t $(CODECOV_TOKEN) -n "Python $(PYTHON_VERSION) $(Agent.OS)" - displayName: Publish coverage stats - continueOnError: True - workingDirectory: $(Pipeline.Workspace)/src - timeoutInMinutes: 2 diff --git a/.azure-pipelines/flake8-validation.py b/.azure-pipelines/flake8-validation.py deleted file mode 100644 index 7b997fab..00000000 --- a/.azure-pipelines/flake8-validation.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import os -import subprocess - -# Flake8 validation -failures = 0 -try: - flake8 = subprocess.run( - [ - "flake8", - "--exit-zero", - ], - capture_output=True, - check=True, - encoding="latin-1", - timeout=300, - ) -except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - print( - "##vso[task.logissue type=error;]flake8 validation failed with", - str(e.__class__.__name__), - ) - print(e.stdout) - print(e.stderr) - print("##vso[task.complete result=Failed;]flake8 validation failed") - exit() -for line in flake8.stdout.split("\n"): - if ":" not in line: - continue - filename, lineno, column, error = line.split(":", maxsplit=3) - errcode, error = error.strip().split(" ", maxsplit=1) - filename = os.path.normpath(filename) - failures += 1 - print( - f"##vso[task.logissue type=error;sourcepath={filename};" - f"linenumber={lineno};columnnumber={column};code={errcode};]" + error - ) - -if failures: - print(f"##vso[task.logissue type=warning]Found {failures} flake8 violation(s)") - print(f"##vso[task.complete result=Failed;]Found {failures} flake8 violation(s)") diff --git a/.azure-pipelines/syntax-validation.py b/.azure-pipelines/syntax-validation.py deleted file mode 100644 index 2d74948a..00000000 --- a/.azure-pipelines/syntax-validation.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import ast -import os -import sys - -print("Python", sys.version, "\n") - -failures = 0 - -for base, _, files in os.walk("."): - for f in files: - if not f.endswith(".py"): - continue - filename = os.path.normpath(os.path.join(base, f)) - try: - with open(filename) as fh: - ast.parse(fh.read()) - except SyntaxError as se: - failures += 1 - print( - f"##vso[task.logissue type=error;sourcepath={filename};" - f"linenumber={se.lineno};columnnumber={se.offset};]" - f"SyntaxError: {se.msg}" - ) - print(" " + se.text + " " * se.offset + "^") - print(f"SyntaxError: {se.msg} in {filename} line {se.lineno}") - print() - -if failures: - print(f"##vso[task.logissue type=warning]Found {failures} syntax error(s)") - print(f"##vso[task.complete result=Failed;]Found {failures} syntax error(s)") diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fea9f883..080ac2b2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -3,9 +3,9 @@ current_version = 0.30.1 commit = True tag = True -[bumpversion:file:setup.cfg] -search = version = {current_version} -replace = version = {new_version} +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" [bumpversion:file:src/zocalo/__init__.py] search = __version__ = "{current_version}" diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 00000000..88e33eb1 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,78 @@ +name: Build and Test + +on: + push: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest build + - name: Build distribution + run: | + python -m build + pip install dist/*.whl + - uses: actions/upload-artifact@v3 + with: + path: ./dist/* + + test: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest dist/*.whl -r requirements_dev.txt pytest-md pytest-emoji + - name: Run pytest + uses: pavelzw/pytest-action@b09a85cd1831cbaae76125fcae4a1e4b137ef026 # v2.1.3 + with: + click-to-expand: false + emoji: true + job-summary: true + verbose: false + custom-arguments: "-v -ra --cov=zocalo --cov-report=xml --cov-branch" + custom-pytest: "PYTHONDEVMODE=1 pytest" + report-title: "Test Report" + - name: Upload Coverage to Codecov + uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4 + + pypi-publish: + name: Upload release to PyPI + needs: test + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/watchdir + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@b7f401de30cb6434a1e19f805ff006643653240e diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 493a9c0b..94a30b84 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,7 +2,7 @@ name: "CodeQL" on: push: - branches: [ "main", "master" ] + branches: ["main"] pull_request: branches: [ "main" ] schedule: diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 00000000..28384f47 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,16 @@ +name: "Linting" + +on: + push: + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Ruff Check + uses: jpetrucciani/ruff-check@33f3725e751a187bb666b7d7a7094d38a2df12db # 0.0.239 diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index f689824b..00000000 --- a/.mypy.ini +++ /dev/null @@ -1,15 +0,0 @@ -[mypy] -mypy_path=src - -[mypy-dlstbx.*] -ignore_missing_imports=True -[mypy-graypy] -ignore_missing_imports=True -[mypy-graypy.handler] -ignore_missing_imports=True -[mypy-marshmallow] -ignore_missing_imports=True -[mypy-pkg_resources] -ignore_missing_imports=True -[mypy-yaml.*] -ignore_missing_imports=True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56f622dc..78e5e456 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: # Syntax validation and some basic sanity checks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.4.0 hooks: - id: check-merge-conflict - id: check-ast @@ -11,41 +11,26 @@ repos: args: ['--maxkb=200'] - id: check-yaml -# Automatically sort imports -- repo: https://github.com/PyCQA/isort - rev: 5.12.0 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.286 hooks: - - id: isort - args: [ - '-a', 'from __future__ import annotations', # 3.7-3.11 - '--rm', 'from __future__ import absolute_import', # -3.0 - '--rm', 'from __future__ import division', # -3.0 - '--rm', 'from __future__ import generator_stop', # -3.7 - '--rm', 'from __future__ import generators', # -2.3 - '--rm', 'from __future__ import nested_scopes', # -2.2 - '--rm', 'from __future__ import print_function', # -3.0 - '--rm', 'from __future__ import unicode_literals', # -3.0 - '--rm', 'from __future__ import with_statement', # -2.6 - ] + - id: ruff + args: [--fix, --show-fixes, --exit-non-zero-on-fix] # Automatic source code formatting - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.7.0 hooks: - id: black args: [--safe, --quiet] -# Linting -- repo: https://github.com/PyCQA/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - additional_dependencies: ['flake8-comprehensions==3.5.0'] - # Type checking - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.910 + rev: v1.5.1 hooks: - id: mypy files: 'src/.*\.py$' - additional_dependencies: [types-requests] + additional_dependencies: + - types-requests + - types-pkg_resources + - types-pyyaml diff --git a/docs/conf.py b/docs/conf.py index 8555a581..4ca4b95f 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,9 +22,10 @@ import os import sys +import sphinx_rtd_theme # noqa: F401 - install theme + sys.path.insert(0, os.path.abspath("..")) -import sphinx_rtd_theme # noqa; F401 - install theme # -- General configuration --------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index de680538..a0a3b075 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,113 @@ [build-system] -requires = ["setuptools >= 40.6.0", "wheel"] +requires = ["setuptools>=68.0"] build-backend = "setuptools.build_meta" +[project] +name = "zocalo" +version = "0.30.1" +description = "Infrastructure components for automated data processing at Diamond Light Source" +readme = "README.rst" +requires-python = ">=3.8" +license = { file = "LICENSE" } +authors = [ + { name = "Diamond Light Source - Data Analysis Group", email = "dataanalysis@diamond.ac.uk" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "PyYAML", + "graypy>=1.0", + "marshmallow", + "requests", + "pydantic<2", + "setuptools", + "workflows>=2.14", +] + +[project.urls] +Bug-Tracker = "https://github.com/DiamondLightSource/python-zocalo/issues" +Changelog = "https://github.com/DiamondLightSource/python-zocalo/blob/main/HISTORY.rst" +Documentation = "https://github.com/DiamondLightSource/python-zocalo" +Download = "https://github.com/DiamondLightSource/python-zocalo/releases" +GitHub = "https://github.com/DiamondLightSource/python-zocalo" + +[project.entry-points] +"libtbx.precommit" = { zocalo = "zocalo" } +"zocalo.wrappers" = { dummy = "zocalo.wrapper:DummyWrapper" } + +[project.entry-points."libtbx.dispatcher.script"] +"zocalo.configure_rabbitmq" = "zocalo.configure_rabbitmq" +"zocalo.dlq_check" = "zocalo.dlq_check" +"zocalo.dlq_purge" = "zocalo.dlq_purge" +"zocalo.dlq_reinject" = "zocalo.dlq_reinject" +"zocalo.go" = "zocalo.go" +"zocalo.queue_drain" = "zocalo.queue_drain" +"zocalo.service" = "zocalo.service" +"zocalo.shutdown" = "zocalo.shutdown" +"zocalo.wrap" = "zocalo.wrap" + +[project.entry-points."workflows.services"] +Dispatcher = "zocalo.service.dispatcher:Dispatcher" +JSONLines = "zocalo.service.jsonlines:JSONLines" +Mailer = "zocalo.service.mailer:Mailer" +Schlockmeister = "zocalo.service.schlockmeister:Schlockmeister" + +[project.entry-points."zocalo.configuration.plugins"] +graylog = "zocalo.configuration.plugin_graylog:Graylog" +jmx = "zocalo.configuration.plugin_jmx:JMX" +logging = "zocalo.configuration.plugin_logging:Logging" +rabbitmqapi = "zocalo.configuration.plugin_rabbitmqapi:RabbitAPI" +slurm = "zocalo.configuration.plugin_slurm:Slurm" +smtp = "zocalo.configuration.plugin_smtp:SMTP" +storage = "zocalo.configuration.plugin_storage:Storage" + +[project.scripts] +"zocalo.configure_rabbitmq" = "zocalo.cli.configure_rabbitmq:run" +"zocalo.dlq_check" = "zocalo.cli.dlq_check:run" +"zocalo.dlq_purge" = "zocalo.cli.dlq_purge:run" +"zocalo.dlq_reinject" = "zocalo.cli.dlq_reinject:run" +"zocalo.go" = "zocalo.cli.go:run" +"zocalo.pickup" = "zocalo.cli.pickup:run" +"zocalo.queue_drain" = "zocalo.cli.queue_drain:run" +"zocalo.service" = "zocalo.service:start_service" +"zocalo.shutdown" = "zocalo.cli.shutdown:run" +"zocalo.wrap" = "zocalo.cli.wrap:run" + [tool.isort] -profile="black" +profile = "black" [tool.pytest.ini_options] required_plugins = "pytest-mock requests-mock" +testpaths = [ + "tests", +] + +[tool.ruff] +line-length = 88 +ignore = ["E501", "E741"] +select = ["C4", "E", "F", "W", "I"] +unfixable = ["F841"] + +[tool.ruff.isort] +required-imports = ["from __future__ import annotations"] + +[tool.mypy] +mypy_path = "src" + + +[[tool.mypy.overrides]] +module = [ + "graypy", + "graypy.handler", +] +ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 75e55bf1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,104 +0,0 @@ -[metadata] -name = zocalo -version = 0.30.1 -description = Infrastructure components for automated data processing at Diamond Light Source -author = Diamond Light Source - Data Analysis Group -author_email = dataanalysis@diamond.ac.uk -license = BSD -license_file = LICENSE -classifiers = - Development Status :: 4 - Beta - Intended Audience :: Developers - License :: OSI Approved :: BSD License - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Operating System :: OS Independent - Topic :: Software Development :: Libraries :: Python Modules -project_urls = - Bug-Tracker = https://github.com/DiamondLightSource/python-zocalo/issues - Changelog = https://github.com/DiamondLightSource/python-zocalo/blob/main/HISTORY.rst - Documentation = https://github.com/DiamondLightSource/python-zocalo - Download = https://github.com/DiamondLightSource/python-zocalo/releases - GitHub = https://github.com/DiamondLightSource/python-zocalo - -[options] -include_package_data = True -install_requires = - PyYAML - graypy>=1.0 - marshmallow - requests - pydantic<2 - setuptools - workflows>=2.14 -packages = find: -package_dir = - =src -python_requires = >=3.8 -zip_safe = False - -[options.entry_points] -console_scripts = - zocalo.configure_rabbitmq = zocalo.cli.configure_rabbitmq:run - zocalo.dlq_check = zocalo.cli.dlq_check:run - zocalo.dlq_purge = zocalo.cli.dlq_purge:run - zocalo.dlq_reinject = zocalo.cli.dlq_reinject:run - zocalo.go = zocalo.cli.go:run - zocalo.pickup = zocalo.cli.pickup:run - zocalo.queue_drain = zocalo.cli.queue_drain:run - zocalo.service = zocalo.service:start_service - zocalo.shutdown = zocalo.cli.shutdown:run - zocalo.wrap = zocalo.cli.wrap:run -libtbx.dispatcher.script = - zocalo.configure_rabbitmq = zocalo.configure_rabbitmq - zocalo.dlq_check = zocalo.dlq_check - zocalo.dlq_purge = zocalo.dlq_purge - zocalo.dlq_reinject = zocalo.dlq_reinject - zocalo.go = zocalo.go - zocalo.queue_drain = zocalo.queue_drain - zocalo.service = zocalo.service - zocalo.shutdown = zocalo.shutdown - zocalo.wrap = zocalo.wrap -libtbx.precommit = - zocalo = zocalo -workflows.services = - Dispatcher = zocalo.service.dispatcher:Dispatcher - JSONLines = zocalo.service.jsonlines:JSONLines - Mailer = zocalo.service.mailer:Mailer - Schlockmeister = zocalo.service.schlockmeister:Schlockmeister -zocalo.configuration.plugins = - graylog = zocalo.configuration.plugin_graylog:Graylog - jmx = zocalo.configuration.plugin_jmx:JMX - logging = zocalo.configuration.plugin_logging:Logging - rabbitmqapi = zocalo.configuration.plugin_rabbitmqapi:RabbitAPI - slurm = zocalo.configuration.plugin_slurm:Slurm - smtp = zocalo.configuration.plugin_smtp:SMTP - storage = zocalo.configuration.plugin_storage:Storage -zocalo.wrappers = - dummy = zocalo.wrapper:DummyWrapper - -[options.packages.find] -where = src - -[flake8] -# Black disagrees with flake8 on a few points. Ignore those. -ignore = E203, E266, E501, W503 -# E203 whitespace before ':' -# E266 too many leading '#' for block comment -# E501 line too long -# W503 line break before binary operator - -max-line-length = 88 - -select = - E401,E711,E712,E713,E714,E721,E722,E901, - F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901, - W191,W291,W292,W293,W602,W603,W604,W605,W606, - # flake8-comprehensions, https://github.com/adamchainz/flake8-comprehensions - C4, - -[aliases] -test = pytest diff --git a/setup.py b/setup.py deleted file mode 100644 index 67fc7640..00000000 --- a/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python - -from __future__ import annotations - -from setuptools import setup - -_rst_section = "\n--------" - -with open("README.rst") as readme_file: - readme = readme_file.read() - - # Only take text up to the first section - if _rst_section in readme: - readme = readme.split(_rst_section)[0] - readme = "\n".join(readme.split("\n")[:-1]) - -with open("HISTORY.rst") as history_file: - history = history_file.read() - - # Only take text from up to 6 sections - _history_split = history.split(_rst_section) - if len(_history_split) > 7: - history = _rst_section.join(_history_split[:7]) - history = "\n".join(history.split("\n")[:-1]) - -setup( - long_description=readme + "\n\n" + history, -) diff --git a/src/zocalo/cli/configure_rabbitmq.py b/src/zocalo/cli/configure_rabbitmq.py index 5ef8a9c4..a5398c3e 100644 --- a/src/zocalo/cli/configure_rabbitmq.py +++ b/src/zocalo/cli/configure_rabbitmq.py @@ -19,9 +19,11 @@ PermissionSpec, PolicySpec, QueueSpec, + UserSpec, + VHostSpec, + hash_password, ) from zocalo.util.rabbitmq import RabbitMQAPI as _RabbitMQAPI -from zocalo.util.rabbitmq import UserSpec, VHostSpec, hash_password logger = logging.getLogger("zocalo.cli.configure_rabbitmq") @@ -409,7 +411,6 @@ def run(): yaml_data = yaml.safe_load(in_file) try: - if args.user_config: _configure_users(api, args.user_config) diff --git a/src/zocalo/cli/dlq_check.py b/src/zocalo/cli/dlq_check.py index 9fdc9bab..c0efacc1 100644 --- a/src/zocalo/cli/dlq_check.py +++ b/src/zocalo/cli/dlq_check.py @@ -14,7 +14,9 @@ # -def check_dlq(zc: zocalo.configuration.Configuration, namespace: str = None) -> dict: +def check_dlq( + zc: zocalo.configuration.Configuration, namespace: str | None = None +) -> dict: """Monitor ActiveMQ queue activity.""" jmx = JMXAPI(zc) if namespace: @@ -48,7 +50,7 @@ def extract_queue_name(namestring): def check_dlq_rabbitmq( - zc: zocalo.configuration.Configuration, namespace: str = None + zc: zocalo.configuration.Configuration, namespace: str | None = None ) -> dict: rmq = RabbitMQAPI.from_zocalo_configuration(zc) return { diff --git a/src/zocalo/configuration/__init__.py b/src/zocalo/configuration/__init__.py index c9bae5fa..74f54bb4 100644 --- a/src/zocalo/configuration/__init__.py +++ b/src/zocalo/configuration/__init__.py @@ -311,7 +311,7 @@ def _read_configuration_yaml(configuration: str) -> dict: f"Invalid YAML configuration: circular environment definitions for {environment_aliases}" ) - plugin_fields = {} + plugin_fields: dict[str, mm.fields.Field] = {} for key in yaml_dict: if key in ConfigSchema().fields: continue diff --git a/src/zocalo/service/__init__.py b/src/zocalo/service/__init__.py index 45923964..14e6702c 100644 --- a/src/zocalo/service/__init__.py +++ b/src/zocalo/service/__init__.py @@ -29,7 +29,9 @@ def setup_logging(self): # Always enable logging to console try: - from dlstbx.util.colorstreamhandler import ColorStreamHandler + from dlstbx.util.colorstreamhandler import ( + ColorStreamHandler, # type: ignore + ) self.console = ColorStreamHandler() except ImportError: diff --git a/src/zocalo/service/jsonlines.py b/src/zocalo/service/jsonlines.py index 06b7a0a8..df1dcb1b 100644 --- a/src/zocalo/service/jsonlines.py +++ b/src/zocalo/service/jsonlines.py @@ -17,8 +17,9 @@ class JSONLines(CommonService): # Logger name _logger_name = "zocalo.service.jsonlines" - def initializing(self): + _data: dict[Path, list[tuple[dict, dict]]] + def initializing(self): self._register_idle(1, self.process_messages) workflows.recipe.wrap_subscribe( self._transport, @@ -30,7 +31,7 @@ def initializing(self): prefetch_count=100, ) self._lock = threading.Lock() - self._data: dict[Path, list[tuple(dict, dict)]] = {} + self._data = {} def receive_msg( self, rw: workflows.recipe.RecipeWrapper, header: dict, message: dict diff --git a/src/zocalo/service/schlockmeister.py b/src/zocalo/service/schlockmeister.py index 2fc1384d..417ed56a 100644 --- a/src/zocalo/service/schlockmeister.py +++ b/src/zocalo/service/schlockmeister.py @@ -179,9 +179,9 @@ def update_subscriptions(self): if self.known_queues[destination].get("subscription"): continue real_subscriber_count = sum( - map( - lambda k: k not in self.known_instances, - self.known_queues[destination]["subscribers"], + ( + k not in self.known_instances + for k in self.known_queues[destination]["subscribers"] ) ) if real_subscriber_count: @@ -205,9 +205,9 @@ def garbage_collect(self): for destination in queues: if self.known_queues[destination].get("subscription"): real_subscriber_count = sum( - map( - lambda k: k not in self.known_instances, - self.known_queues[destination]["subscribers"], + ( + k not in self.known_instances + for k in self.known_queues[destination]["subscribers"] ) ) if not real_subscriber_count: diff --git a/src/zocalo/util/rabbitmq.py b/src/zocalo/util/rabbitmq.py index 1df85ef5..741cff54 100644 --- a/src/zocalo/util/rabbitmq.py +++ b/src/zocalo/util/rabbitmq.py @@ -706,7 +706,10 @@ def health_checks(self) -> Tuple[Dict[str, Any], Dict[str, str]]: return success, failure def get( - self, endpoint: str, params: Dict[str, Any] = None, timeout: float | None = None + self, + endpoint: str, + params: Dict[str, Any] | None = None, + timeout: float | None = None, ) -> requests.Response: return self._session.get( f"{self._url}/{endpoint}", params=params, timeout=timeout @@ -715,8 +718,8 @@ def get( def put( self, endpoint: str, - params: Dict[str, Any] = None, - json: Dict[str, Any] = None, + params: Dict[str, Any] | None = None, + json: Dict[str, Any] | None = None, timeout: float | None = None, ) -> requests.Response: return self._session.put( @@ -735,7 +738,10 @@ def post( ) def delete( - self, endpoint: str, params: Dict[str, Any] = None, timeout: float | None = None + self, + endpoint: str, + params: Dict[str, Any] | None = None, + timeout: float | None = None, ) -> requests.Response: return self._session.delete( f"{self._url}/{endpoint}", params=params, timeout=timeout @@ -759,9 +765,10 @@ def bindings( if destination_type is not None: endpoint = f"{endpoint}/e/{source}/{destination_type}/{destination}" dest_map = {"queue": "q", "exchange": "e"} - conv = ( - lambda key, value: dest_map[value] if key == "destination_type" else value - ) + + def conv(key, value): + return dest_map[value] if key == "destination_type" else value + return [ BindingInfo(**{_r[0]: conv(_r[0], _r[1]) for _r in r.items()}) for r in self.get(endpoint).json() @@ -791,11 +798,10 @@ def bindings_delete( endpoint = f"bindings/{vhost}/e/{source}/{destination_type}/{destination}" if properties_key is None: dest_map = {"queue": "q", "exchange": "e"} - conv = ( - lambda key, value: dest_map[value] - if key == "destination_type" - else value - ) + + def conv(key, value): + return dest_map[value] if key == "destination_type" else value + props = [ BindingInfo( **{_r[0]: conv(_r[0], _r[1]) for _r in r.items()} diff --git a/src/zocalo/util/slurm/__init__.py b/src/zocalo/util/slurm/__init__.py index ed090e67..6fefb0cc 100644 --- a/src/zocalo/util/slurm/__init__.py +++ b/src/zocalo/util/slurm/__init__.py @@ -44,7 +44,10 @@ def from_zocalo_configuration(cls, zc: zocalo.configuration.Configuration): ) def get( - self, endpoint: str, params: dict[str, Any] = None, timeout: float | None = None + self, + endpoint: str, + params: dict[str, Any] | None = None, + timeout: float | None = None, ) -> requests.Response: response = self.session.get( f"{self.url}/{endpoint}", params=params, timeout=timeout @@ -55,8 +58,8 @@ def get( def put( self, endpoint: str, - params: dict[str, Any] = None, - json: dict[str, Any] = None, + params: dict[str, Any] | None = None, + json: dict[str, Any] | None = None, timeout: float | None = None, ) -> requests.Response: response = self.session.put( @@ -79,7 +82,10 @@ def post( return response def delete( - self, endpoint: str, params: dict[str, Any] = None, timeout: float | None = None + self, + endpoint: str, + params: dict[str, Any] | None = None, + timeout: float | None = None, ) -> requests.Response: response = self.session.delete( f"{self.url}/{endpoint}", params=params, timeout=timeout diff --git a/src/zocalo/wrapper.py b/src/zocalo/wrapper.py index cf65c8b3..27ed4875 100644 --- a/src/zocalo/wrapper.py +++ b/src/zocalo/wrapper.py @@ -4,13 +4,13 @@ import threading from typing import Any, Callable +import workflows.services.common_service import workflows.util import zocalo class BaseWrapper: - _logger_name = "zocalo.wrapper" # The logger can be accessed via self.log def __init__(self, *args, **kwargs): @@ -60,7 +60,6 @@ def config(self): class DummyWrapper(BaseWrapper): - _logger_name = "zocalo.wrapper.DummyWrapper" def run(self): diff --git a/tests/cli/test_dlq_purge.py b/tests/cli/test_dlq_purge.py index d32ad5eb..e77fb1ea 100644 --- a/tests/cli/test_dlq_purge.py +++ b/tests/cli/test_dlq_purge.py @@ -5,9 +5,8 @@ from unittest import mock import workflows.transport -from workflows.transport.common_transport import CommonTransport - import zocalo.cli.dlq_purge as dlq_purge +from workflows.transport.common_transport import CommonTransport def gen_header_activemq(i): diff --git a/tests/cli/test_dlq_reinject.py b/tests/cli/test_dlq_reinject.py index 2770e78a..5298de2e 100644 --- a/tests/cli/test_dlq_reinject.py +++ b/tests/cli/test_dlq_reinject.py @@ -8,7 +8,6 @@ import workflows.transport from workflows.transport.common_transport import CommonTransport - from zocalo.cli.dlq_reinject import run diff --git a/tests/cli/test_pickup.py b/tests/cli/test_pickup.py index a4e37433..7519ae00 100644 --- a/tests/cli/test_pickup.py +++ b/tests/cli/test_pickup.py @@ -8,9 +8,8 @@ import pytest import workflows.transport -from workflows.transport.common_transport import CommonTransport - import zocalo.configuration +from workflows.transport.common_transport import CommonTransport from zocalo.cli import pickup diff --git a/tests/cli/test_queue_drain.py b/tests/cli/test_queue_drain.py index 98439521..886fb6c8 100644 --- a/tests/cli/test_queue_drain.py +++ b/tests/cli/test_queue_drain.py @@ -4,7 +4,6 @@ import workflows.transport from workflows.transport.common_transport import CommonTransport - from zocalo.cli.queue_drain import run diff --git a/tests/cli/test_shutdown.py b/tests/cli/test_shutdown.py index 3e21d8ff..d05d9276 100644 --- a/tests/cli/test_shutdown.py +++ b/tests/cli/test_shutdown.py @@ -6,7 +6,6 @@ import workflows.transport from workflows.transport.common_transport import CommonTransport from workflows.util import generate_unique_host_id - from zocalo.cli.shutdown import run diff --git a/tests/configuration/test_configuration.py b/tests/configuration/test_configuration.py index 7f54a04f..3ab1643b 100644 --- a/tests/configuration/test_configuration.py +++ b/tests/configuration/test_configuration.py @@ -7,7 +7,6 @@ from unittest import mock import pytest - import zocalo.configuration sample_configuration = """ diff --git a/tests/configuration/test_plugin_graylog.py b/tests/configuration/test_plugin_graylog.py index 2f8700a7..e38fcadf 100644 --- a/tests/configuration/test_plugin_graylog.py +++ b/tests/configuration/test_plugin_graylog.py @@ -3,7 +3,6 @@ from unittest import mock import pytest - import zocalo.configuration sample_configuration = """ diff --git a/tests/configuration/test_plugin_jmx.py b/tests/configuration/test_plugin_jmx.py index ef3135db..547cc86a 100644 --- a/tests/configuration/test_plugin_jmx.py +++ b/tests/configuration/test_plugin_jmx.py @@ -1,7 +1,6 @@ from __future__ import annotations import pytest - import zocalo.configuration sample_configuration = """ diff --git a/tests/configuration/test_plugin_logging.py b/tests/configuration/test_plugin_logging.py index 75826db4..8cf4fcbd 100644 --- a/tests/configuration/test_plugin_logging.py +++ b/tests/configuration/test_plugin_logging.py @@ -4,7 +4,6 @@ from unittest import mock import pytest - import zocalo.configuration import zocalo.configuration.plugin_logging diff --git a/tests/configuration/test_plugin_slurm.py b/tests/configuration/test_plugin_slurm.py index be4947c0..c24e298b 100644 --- a/tests/configuration/test_plugin_slurm.py +++ b/tests/configuration/test_plugin_slurm.py @@ -1,7 +1,6 @@ from __future__ import annotations import pytest - import zocalo.configuration sample_configuration = """ diff --git a/tests/configuration/test_plugin_smtp.py b/tests/configuration/test_plugin_smtp.py index 50b5f707..3b4f23aa 100644 --- a/tests/configuration/test_plugin_smtp.py +++ b/tests/configuration/test_plugin_smtp.py @@ -1,7 +1,6 @@ from __future__ import annotations import pytest - import zocalo.configuration sample_configuration = """ diff --git a/tests/service/test_dispatcher.py b/tests/service/test_dispatcher.py index 1b3182fe..1befdd3b 100644 --- a/tests/service/test_dispatcher.py +++ b/tests/service/test_dispatcher.py @@ -4,10 +4,9 @@ from unittest import mock import pytest +import zocalo.configuration from workflows.recipe import Recipe from workflows.transport.offline_transport import OfflineTransport - -import zocalo.configuration from zocalo.service.dispatcher import Dispatcher diff --git a/tests/service/test_jsonlines.py b/tests/service/test_jsonlines.py index af7fb909..9db2a84e 100644 --- a/tests/service/test_jsonlines.py +++ b/tests/service/test_jsonlines.py @@ -4,7 +4,6 @@ from workflows.recipe.wrapper import RecipeWrapper from workflows.transport.offline_transport import OfflineTransport - from zocalo.service.jsonlines import JSONLines diff --git a/tests/service/test_mailer.py b/tests/service/test_mailer.py index 83361ff9..bdf711ed 100644 --- a/tests/service/test_mailer.py +++ b/tests/service/test_mailer.py @@ -3,10 +3,9 @@ from unittest import mock import pytest +import zocalo.configuration from workflows.recipe.wrapper import RecipeWrapper from workflows.transport.offline_transport import OfflineTransport - -import zocalo.configuration from zocalo.service.mailer import Mailer diff --git a/tests/test_zocalo.py b/tests/test_zocalo.py index ad0ae3ad..38b81275 100644 --- a/tests/test_zocalo.py +++ b/tests/test_zocalo.py @@ -5,7 +5,6 @@ from __future__ import annotations import pytest - import zocalo diff --git a/tests/util/test_rabbitmq.py b/tests/util/test_rabbitmq.py index a5c00c5a..7797a77f 100644 --- a/tests/util/test_rabbitmq.py +++ b/tests/util/test_rabbitmq.py @@ -3,7 +3,6 @@ import re import pytest - import zocalo.configuration import zocalo.util.rabbitmq as rabbitmq diff --git a/tests/util/test_slurm.py b/tests/util/test_slurm.py index 3a9ae5b8..83e39ec4 100644 --- a/tests/util/test_slurm.py +++ b/tests/util/test_slurm.py @@ -1,7 +1,6 @@ from __future__ import annotations import pytest - import zocalo.configuration from zocalo.util import slurm