From 979785cdee3819f0e4b76e7a96dd42f9d05a63dc Mon Sep 17 00:00:00 2001 From: Lucas Bickel Date: Mon, 31 Jan 2022 21:52:55 +0100 Subject: [PATCH 1/2] chore: linting and cleanup --- .flake8 | 33 +++ .github/dependabot.yml | 26 ++ .github/workflows/lint-and-test.yaml | 36 +++ .isort.cfg | 7 + .pre-commit-config.yaml | 32 +++ .travis.yml | 19 -- .travis/rpm.sh | 11 - .vscode/settings.json | 3 + LICENSE | 2 +- README.md | 15 +- renovate.json | 5 - requirements-dev.txt | 17 ++ requirements.txt | 6 +- setup.cfg | 5 + setup.py | 33 ++- suisa_sendemeldung.spec | 108 -------- suisa_sendemeldung/acrclient.py | 37 ++- suisa_sendemeldung/suisa_sendemeldung.py | 332 ++++++++++++++--------- 18 files changed, 406 insertions(+), 321 deletions(-) create mode 100644 .flake8 create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/lint-and-test.yaml create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml delete mode 100644 .travis.yml delete mode 100755 .travis/rpm.sh create mode 100644 .vscode/settings.json delete mode 100644 renovate.json create mode 100644 requirements-dev.txt create mode 100644 setup.cfg delete mode 100644 suisa_sendemeldung.spec diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..6cc391dc --- /dev/null +++ b/.flake8 @@ -0,0 +1,33 @@ +[flake8] +ignore = + # whitespace before ':' + E203, + # too many leading ### in a block comment + E266, + # line too long (managed by black) + E501, + # Line break occurred before a binary operator (this is not PEP8 compatible) + W503, + # Missing docstring in public module + D100, + # Missing docstring in public class + D101, + # Missing docstring in public method + D102, + # Missing docstring in public function + D103, + # Missing docstring in public package + D104, + # Missing docstring in magic method + D105, + # Missing docstring in public package + D106, + # Missing docstring in __init__ + D107, + # needed because of https://github.com/ambv/black/issues/144 + D202, + # other string does contain unindexed parameters + P103 +max-line-length = 80 +exclude = migrations snapshots +max-complexity = 10 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..98d193aa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,26 @@ +version: 2 +updates: + # Update pip dependencies + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "feat(deps): " + prefix-development: "chore(deps): " + open-pull-requests-limit: 20 + # Update Dockerfile + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "feat: " + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + commit-message: + prefix: "chore(ci): " + open-pull-requests-limit: 10 diff --git a/.github/workflows/lint-and-test.yaml b/.github/workflows/lint-and-test.yaml new file mode 100644 index 00000000..36b82268 --- /dev/null +++ b/.github/workflows/lint-and-test.yaml @@ -0,0 +1,36 @@ +name: Lint and Test + +on: + push: + branches-ignore: + - main + - gh-pages + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + + - name: Setup Python + uses: actions/setup-python@v2 + + - run: pip install -r requirements-dev.txt + + - name: Run pre-commit + uses: pre-commit/action@v2.0.3 + + pytest: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + + - name: Setup Python + uses: actions/setup-python@v2 + + - run: pip install -r requirements-dev.txt + + - run: pytest diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..6f8d4f1f --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,7 @@ +[settings] +known_first_party=nowplaying +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +combine_as_imports=True +line_length=88 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..7ad2b1e9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +repos: + - repo: local + hooks: + - id: black + name: black + language: system + entry: black + types: [python] + - id: isort + name: isort + language: system + entry: isort + types: [python] + - id: flake8 + name: flake8 + language: system + entry: flake8 + types: [python] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: trailing-whitespace + exclude: ^src/api/client.js$ + - id: end-of-file-fixer + exclude: ^src/api/client.js$ + - id: check-symlinks + - id: check-merge-conflict + - id: check-case-conflict + - id: detect-aws-credentials + args: + - --allow-missing-credentials + - id: detect-private-key diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 542c7136..00000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: minimal - -services: - - docker - -jobs: - include: - - stage: packaging tests - name: "Docker" - before_install: - - docker pull radiorabe/suisa_sendemeldung - script: - - docker build -t suisa_sendemeldung --cache-from radiorabe/suisa_sendemeldung . - - docker run --rm -ti suisa_sendemeldung -h - - stage: packaging tests - name: "RPM" - before_install: - - docker pull quay.io/hairmare/centos_rpmdev - script: docker run --rm -ti -v `pwd`:'/git' quay.io/hairmare/centos_rpmdev /git/.travis/rpm.sh diff --git a/.travis/rpm.sh b/.travis/rpm.sh deleted file mode 100755 index f925c988..00000000 --- a/.travis/rpm.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# RPM build wrapper for suisa_sendemeldung, runs inside the build container on travis-ci - -set -xe - -chown root:root suisa_sendemeldung.spec - -create-source-tarball.sh /git suisa_sendemeldung-master.tar.gz - -build-rpm-package.sh suisa_sendemeldung.spec diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..b7368caa --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "black" +} diff --git a/LICENSE b/LICENSE index ab602974..b442934b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 +Copyright (c) 2018 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index ef484a4d..7f31b0d8 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,6 @@ ACRCloud client that fetches data on our playout history and formats them in a C ## Installation -### [Open Build Service](https://openbuildservice.org/) - -There are pre-built binary packages for CentOS 7 available on [Radio RaBe's OBS sendemeldungs package repository](https://build.opensuse.org/project/show/home:radiorabe:sendemeldung), which can be installed as follows: - -```bash -curl -o /etc/yum.repos.d/home:radiorabe:sendemeldung.repo \ - http://download.opensuse.org/repositories/home:/radiorabe:/sendemeldung/CentOS_7/home:radiorabe:sendemeldung.repo - -yum install suisa_sendemeldung -``` - -### Docker - You can build a Docker image using the included [Dockerfile](Dockerfile): ```bash @@ -89,7 +76,7 @@ optional arguments: --filename FILENAME file to write to (default: _.csv) [env var: FILENAME] --stdout also print to stdout [env var: STDOUT] ``` - + ## Configuration You can configure this script either with a configuration file (default is `suisa_sendemeldung.conf`), environment variables or command line arguments as shown above. diff --git a/renovate.json b/renovate.json deleted file mode 100644 index f45d8f11..00000000 --- a/renovate.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "extends": [ - "config:base" - ] -} diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..45870256 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,17 @@ +-r requirements.txt +black==22.1.0 +flake8==4.0.1 +flake8-debugger==4.0.0 +flake8-docstrings==1.6.0 +flake8-isort==4.1.1 +flake8-string-format==0.3.0 +flake8-tuple==0.4.1 +isort==5.10.1 +mock==4.0.3 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-env==0.6.2 +pytest-pylint==0.18.0 +twine==3.7.1 +types-requests==2.27.7 +wheel==0.37.1 diff --git a/requirements.txt b/requirements.txt index 143c0853..ba7d4c33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -ConfigArgParse -pytz -requests +ConfigArgParse==1.5.3 +pytz==2021.3 +requests==2.27.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..b09f331c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +description-file = README.md + +[tool:pytest] +addopts = --doctest-modules --cov=suisa_sendemeldung --pylint diff --git a/setup.py b/setup.py index efea4717..a33e484e 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,24 @@ -from setuptools import setup +"""Set up suisa_sendemeldung.""" +from setuptools import setup -with open('requirements.txt') as f: +with open("requirements.txt", encoding="utf-8") as f: requirements = f.read().splitlines() -setup(name='suisa_sendemeldung', - description='ACRCloud client for SUISA reporting', - url='http://github.com/radiorabe/suisa_reporting', - author='RaBe IT-Reaktion', - author_email='it@rabe.ch', - license='MIT', - install_requires=requirements, - packages=['suisa_sendemeldung'], - entry_points = { - 'console_scripts': ['suisa_sendemeldung=suisa_sendemeldung.suisa_sendemeldung:main'], - }, - zip_safe=True) +setup( + name="suisa_sendemeldung", + description="ACRCloud client for SUISA reporting", + url="http://github.com/radiorabe/suisa_reporting", + author="RaBe IT-Reaktion", + author_email="it@rabe.ch", + license="MIT", + install_requires=requirements, + packages=["suisa_sendemeldung"], + entry_points={ + "console_scripts": [ + "suisa_sendemeldung=suisa_sendemeldung.suisa_sendemeldung:main" + ], + }, + zip_safe=True, +) diff --git a/suisa_sendemeldung.spec b/suisa_sendemeldung.spec deleted file mode 100644 index dd4e231b..00000000 --- a/suisa_sendemeldung.spec +++ /dev/null @@ -1,108 +0,0 @@ -# -# spec file for package suisa_sendemeldung -# -# Copyright (c) 2018 Radio Bern RaBe -# http://www.rabe.ch -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Please submit enhancements, bugfixes or comments via GitHub: -# https://github.com/radiorabe/suisa_sendemeldung -# -%global srcname suisa_sendemeldung - -%{?el7:%global python3_pkgversion 36} - -Name: %{srcname} -Version: master -Release: 0%{?dist} -Summary: ACRCloud client for SUISA reporting - -License: MIT -URL: https://github.com/radiorabe/suisa_sendemeldung -Source0: https://github.com/radiorabe/suisa_sendemeldung/archive/%{version}/%{name}-%{version}.tar.gz - -BuildArch: noarch -BuildRequires: python%{python3_pkgversion}-configargparse -BuildRequires: python%{python3_pkgversion}-devel -BuildRequires: python%{python3_pkgversion}-pytz -BuildRequires: python%{python3_pkgversion}-requests -BuildRequires: python%{python3_pkgversion}-setuptools -BuildRequires: python3-devel -%{?systemd_requires} -BuildRequires: systemd -Requires(pre): shadow-utils -Requires: python%{python3_pkgversion}-configargparse -Requires: python%{python3_pkgversion}-pytz -Requires: python%{python3_pkgversion}-requests -%{?python_enable_dependency_generator} - -%description -ACRCloud client that fetches data on our playout history and -formats them in a CSV file format containing the data (like -Track, Title and, ISRC) requested by SUISA. -Also takes care of sending the report to SUISA via email for -hands-off operations. - -%prep -%autosetup -n %{srcname}-%{version} - -%build -%py3_build - -%install -%py3_install -install -d %{buildroot}%{_unitdir} -install etc/systemd/%{srcname}.* %{buildroot}%{_unitdir} -install -d %{buildroot}%{_sysconfdir} -install etc/%{srcname}.conf %{buildroot}%{_sysconfdir} -install -d %{buildroot}%{_sysconfdir}/sysconfig -install etc/sysconfig/%{srcname} %{buildroot}%{_sysconfdir}/sysconfig - -%check -%{__python3} setup.py test - -%pre -getent group %{srcname} >/dev/null || groupadd -r %{srcname} -getent passwd %{srcname} >/dev/null || \ - useradd -r -g %{srcname} -d %{python3_sitelib}/%{srcname}/ -s /sbin/nologin \ - -c "SUISA ACRClient account" %{srcname} -exit 0 - -%post -%systemd_post %{srcname}.service -%systemd_post %{srcname}.timer - -%preun -%systemd_preun %{srcname}.timer -%systemd_preun %{srcname}.service - -%postun -%systemd_postun %{srcname}.timer -%systemd_postun %{srcname}.service - -%files -%license LICENSE -%doc README.md -%{python3_sitelib}/%{srcname}/ -%{python3_sitelib}/%{srcname}-*.egg-info/ -%{_bindir}/%{srcname} -%{_unitdir}/%{srcname}.* -%config(noreplace)%{_sysconfdir}/%{srcname}.conf -%config(noreplace)%{_sysconfdir}/sysconfig/%{srcname} diff --git a/suisa_sendemeldung/acrclient.py b/suisa_sendemeldung/acrclient.py index cad85649..1871dbae 100644 --- a/suisa_sendemeldung/acrclient.py +++ b/suisa_sendemeldung/acrclient.py @@ -1,28 +1,29 @@ -"""module containing the ACRCloud client""" +"""module containing the ACRCloud client.""" from datetime import date, datetime, timedelta import pytz import requests + class ACRClient: - """ACRCloud client to fetch metadata + """ACRCloud client to fetch metadata. Args: access_key: The access key for ACRCloud. """ + # format of timestamp in api answer - TS_FMT = '%Y-%m-%d %H:%M:%S' + TS_FMT = "%Y-%m-%d %H:%M:%S" # timezone of ACRCloud - ACR_TIMEZONE = 'UTC' + ACR_TIMEZONE = "UTC" def __init__(self, access_key): self.access_key = access_key - self.default_date = date.today()-timedelta(days=1) - self.url = ('https://api.acrcloud.com/v1/' - 'monitor-streams/{stream_id}/results') + self.default_date = date.today() - timedelta(days=1) def get_data(self, stream_id, requested_date=None, timezone=ACR_TIMEZONE): """Fetch metadata from ACRCloud for `stream_id`. + Args: stream_id: The ID of the stream. requested_date (optional): The date of the entries you want (default: yesterday). @@ -34,28 +35,26 @@ def get_data(self, stream_id, requested_date=None, timezone=ACR_TIMEZONE): if requested_date is None: requested_date = self.default_date url_params = dict( - access_key=self.access_key, - date=requested_date.strftime('%Y%m%d') + access_key=self.access_key, date=requested_date.strftime("%Y%m%d") ) - url = self.url.format(stream_id=stream_id) + url = f"https://api.acrcloud.com/v1/monitor-streams/{stream_id}/results" response = requests.get(url=url, params=url_params) response.raise_for_status() data = response.json() for entry in data: - metadata = entry.get('metadata') - ts_utc = pytz.utc.localize(datetime.strptime(metadata.get('timestamp_utc'), - ACRClient.TS_FMT)) + metadata = entry.get("metadata") + ts_utc = pytz.utc.localize( + datetime.strptime(metadata.get("timestamp_utc"), ACRClient.TS_FMT) + ) ts_local = ts_utc.astimezone(pytz.timezone(timezone)) - metadata.update({ - 'timestamp_local': ts_local.strftime(ACRClient.TS_FMT) - }) + metadata.update({"timestamp_local": ts_local.strftime(ACRClient.TS_FMT)}) return data def get_interval_data(self, stream_id, start, end, timezone=ACR_TIMEZONE): - """Get data specified by interval from start to end + """Get data specified by interval from start to end. Args: stream_id: The ID of the stream. @@ -94,8 +93,8 @@ def get_interval_data(self, stream_id, start, end, timezone=ACR_TIMEZONE): # if timestamps are localized we will have to removed the unneeded entries. if trim: for entry in reversed(data): - metadata = entry.get('metadata') - timestamp = metadata.get('timestamp_local') + metadata = entry.get("metadata") + timestamp = metadata.get("timestamp_local") timestamp_date = datetime.strptime(timestamp, ACRClient.TS_FMT).date() if timestamp_date < start or timestamp_date > end: data.remove(entry) diff --git a/suisa_sendemeldung/suisa_sendemeldung.py b/suisa_sendemeldung/suisa_sendemeldung.py index e8fdb6ed..87e7c703 100755 --- a/suisa_sendemeldung/suisa_sendemeldung.py +++ b/suisa_sendemeldung/suisa_sendemeldung.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 """ -ACRCloud client that fetches data on our playout history and formats them in a CSV file format +SUISA Sendemeldung bugs SUISA with email once per month. + +Fetches data on our playout history and formats them in a CSV file format containing the data (like Track, Title and ISRC) requested by SUISA. Also takes care of sending the report to SUISA via email for hands-off operations. """ from csv import writer -from datetime import datetime, date, timedelta +from datetime import date, datetime, timedelta from email.encoders import encode_base64 from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -17,16 +19,13 @@ from configargparse import ArgumentParser -try: - # this only works when the program has been installed - from suisa_sendemeldung.acrclient import ACRClient -except ModuleNotFoundError: - from acrclient import ACRClient +from .acrclient import ACRClient def validate_arguments(parser, args): - """Validate the arguments provided to the script. After this function we are sure that there - are no conflicts in the arguments + """Validate the arguments provided to the script. + + After this function we are sure that there are no conflicts in the arguments. Arguments: parser: the ArgumentParser to use for throwing errors @@ -35,26 +34,29 @@ def validate_arguments(parser, args): msgs = [] # check length of access_key if not len(args.access_key) == 32: - msgs.append('wrong format on access_key, expected 32 characters but got {}.' - .format(len(args.access_key))) + msgs.append( + f"wrong format on access_key, expected 32 characters but got {len(args.access_key)}" + ) # check length of stream_id if not len(args.stream_id) == 9: - msgs.append('wrong format on stream_id, expected 9 characters but got {}.' - .format(len(args.stream_id))) + msgs.append( + f"wrong format on stream_id, expected 9 characters but got {len(args.stream_id)}" + ) # one output option has to be set if not (args.csv or args.email or args.stdout): - msgs.append('no output option has been set, specify one of --csv, --email or --stdout') + msgs.append( + "no output option has been set, specify one of --csv, --email or --stdout" + ) # last_month is in conflict with start_date and end_date if args.last_month and (args.start_date or args.end_date): - msgs.append('argument --last_month not allowed with --start_date or --end_date') + msgs.append("argument --last_month not allowed with --start_date or --end_date") # exit if there are error messages if msgs: - parser.error('\n- ' + '\n- '.join(msgs)) + parser.error("\n- " + "\n- ".join(msgs)) -def get_arguments(parser): - """Setup the provided ArgumentParser (a configargparse.ArgumentParser object) with arguments - and return arguments +def get_arguments(parser: ArgumentParser): + """Create :class:`ArgumentParser` with arguments. Arguments: parser: the parser to add arguments @@ -62,46 +64,96 @@ def get_arguments(parser): Returns: args: the parsed args from the parser """ - parser.add_argument('--access_key', env_var='ACCESS_KEY', - help='the access key for ACRCloud (required)', required=True) - parser.add_argument('--stream_id', env_var='STREAM_ID', - help='the id of the stream at ACRCloud (required)', required=True) - parser.add_argument('--csv', env_var='CSV', help='create a csv file', action='store_true') - parser.add_argument('--email', env_var='EMAIL', help='send an email', action='store_true') - parser.add_argument('--email_from', env_var='EMAIL_FROM', help='the sender of the email') - parser.add_argument('--email_to', env_var='EMAIL_TO', help='the recipients of the email') - parser.add_argument('--email_cc', env_var='EMAIL_CC', help='the cc recipients of the email') - parser.add_argument('--email_bcc', env_var='EMAIL_BCC', help='the bcc recipients of the email') - parser.add_argument('--email_server', env_var='EMAIL_SERVER', - help='the smtp server to send the mail with') - parser.add_argument('--email_login', env_var='EMAIL_LOGIN', - help='the username to logon to the smtp server (default: email_from)') - parser.add_argument('--email_pass', env_var='EMAIL_PASS', - help='the password for the smtp server') - parser.add_argument('--email_subject', env_var='EMAIL_SUBJECT', help='the subject of the email', - default='SUISA Sendemeldung') - parser.add_argument('--email_text', env_var='EMAIL_TEXT', - help='the text of the email', default='') - parser.add_argument('--start_date', env_var='START_DATE', - help='the start date of the interval in format YYYY-MM-DD (default: 30 days\ - before end_date)') - parser.add_argument('--end_date', env_var='END_DATE', - help='the end date of the interval in format YYYY-MM-DD (default: today)') - parser.add_argument('--last_month', env_var='LAST_MONTH', action='store_true', - help='download data of whole last month') - parser.add_argument('--timezone', env_var='TIMEZONE', help='set the timezone for localization', - default='UTC') - parser.add_argument('--filename', env_var='FILENAME', - help='file to write to (default: _.csv)') - parser.add_argument('--stdout', env_var='STDOUT', help='also print to stdout', - action='store_true') + parser.add_argument( + "--access_key", + env_var="ACCESS_KEY", + help="the access key for ACRCloud (required)", + required=True, + ) + parser.add_argument( + "--stream_id", + env_var="STREAM_ID", + help="the id of the stream at ACRCloud (required)", + required=True, + ) + parser.add_argument( + "--csv", env_var="CSV", help="create a csv file", action="store_true" + ) + parser.add_argument( + "--email", env_var="EMAIL", help="send an email", action="store_true" + ) + parser.add_argument( + "--email_from", env_var="EMAIL_FROM", help="the sender of the email" + ) + parser.add_argument( + "--email_to", env_var="EMAIL_TO", help="the recipients of the email" + ) + parser.add_argument( + "--email_cc", env_var="EMAIL_CC", help="the cc recipients of the email" + ) + parser.add_argument( + "--email_bcc", env_var="EMAIL_BCC", help="the bcc recipients of the email" + ) + parser.add_argument( + "--email_server", + env_var="EMAIL_SERVER", + help="the smtp server to send the mail with", + ) + parser.add_argument( + "--email_login", + env_var="EMAIL_LOGIN", + help="the username to logon to the smtp server (default: email_from)", + ) + parser.add_argument( + "--email_pass", env_var="EMAIL_PASS", help="the password for the smtp server" + ) + parser.add_argument( + "--email_subject", + env_var="EMAIL_SUBJECT", + help="the subject of the email", + default="SUISA Sendemeldung", + ) + parser.add_argument( + "--email_text", env_var="EMAIL_TEXT", help="the text of the email", default="" + ) + parser.add_argument( + "--start_date", + env_var="START_DATE", + help="the start date of the interval in format YYYY-MM-DD (default: 30 days\ + before end_date)", + ) + parser.add_argument( + "--end_date", + env_var="END_DATE", + help="the end date of the interval in format YYYY-MM-DD (default: today)", + ) + parser.add_argument( + "--last_month", + env_var="LAST_MONTH", + action="store_true", + help="download data of whole last month", + ) + parser.add_argument( + "--timezone", + env_var="TIMEZONE", + help="set the timezone for localization", + default="UTC", + ) + parser.add_argument( + "--filename", + env_var="FILENAME", + help="file to write to (default: _.csv)", + ) + parser.add_argument( + "--stdout", env_var="STDOUT", help="also print to stdout", action="store_true" + ) args = parser.parse_args() validate_arguments(parser, args) return args def parse_date(args): - """Parse date from args + """Parse date from args. Arguments: args: the arguments provided to the script @@ -120,12 +172,12 @@ def parse_date(args): start_date = end_date.replace(day=1) else: if args.end_date: - end_date = datetime.strptime(args.end_date, '%Y-%m-%d').date() + end_date = datetime.strptime(args.end_date, "%Y-%m-%d").date() else: # if no end_date was set, default to today end_date = date.today() if args.start_date: - start_date = datetime.strptime(args.start_date, '%Y-%m-%d').date() + start_date = datetime.strptime(args.start_date, "%Y-%m-%d").date() else: # if no start_date was set, default to 30 days before end_date start_date = end_date - timedelta(days=30) @@ -133,7 +185,7 @@ def parse_date(args): def parse_filename(args, start_date): - """Parse filename from args and start_date + """Parse filename from args and start_date. Arguments: args: the arguments provided to the script @@ -145,14 +197,14 @@ def parse_filename(args, start_date): filename = args.filename # depending on date args either append the month or the start_date elif args.last_month: - filename = (__file__.replace('.py', '_{}.csv').format(start_date.strftime('%B'))) + filename = __file__.replace(".py", "_{}.csv").format(start_date.strftime("%B")) else: - filename = __file__.replace('.py', '_{}.csv').format(start_date) + filename = __file__.replace(".py", "_{}.csv").format(start_date) return filename def check_duplicate(entry_a, entry_b): - """Check if two entries are duplicates by checking their acrid in all music items + """Check if two entries are duplicates by checking their acrid in all music items. Arguments: entry_a: first entry @@ -162,22 +214,22 @@ def check_duplicate(entry_a, entry_b): True if the entries are duplicates, False otherwise """ try: - entry_a = entry_a['metadata']['music'] + entry_a = entry_a["metadata"]["music"] except KeyError: - entry_a = entry_a['metadata']['custom_files'] + entry_a = entry_a["metadata"]["custom_files"] try: - entry_b = entry_b['metadata']['music'] + entry_b = entry_b["metadata"]["music"] except KeyError: - entry_b = entry_b['metadata']['custom_files'] + entry_b = entry_b["metadata"]["custom_files"] for music_a in entry_a: for music_b in entry_b: - if music_a['acrid'] == music_b['acrid']: + if music_a["acrid"] == music_b["acrid"]: return True return False def merge_duplicates(data): - """Merge consecutive entries into one if they are duplicates + """Merge consecutive entries into one if they are duplicates. Arguments: data: The data provided by ACRClient @@ -189,8 +241,10 @@ def merge_duplicates(data): mark = [] for entry in data[1:]: if check_duplicate(prev, entry): - prev['metadata']['played_duration'] = prev['metadata']['played_duration'] + \ - entry['metadata']['played_duration'] + prev["metadata"]["played_duration"] = ( + prev["metadata"]["played_duration"] + + entry["metadata"]["played_duration"] + ) # mark entry for removal mark.append(entry) else: @@ -202,7 +256,7 @@ def merge_duplicates(data): def get_csv(data): - """Create SUISA compatible csv data + """Create SUISA compatible csv data. Arguments: data: To data to create csv from @@ -210,66 +264,74 @@ def get_csv(data): Returns: csv: The converted data """ - header = ['Sendedatum', 'Sendezeit', 'Sendedauer', 'Titel', 'Künstler', 'ISRC', 'Label'] + header = [ + "Sendedatum", + "Sendezeit", + "Sendedauer", + "Titel", + "Künstler", + "ISRC", + "Label", + ] csv = StringIO() - csv.write('sep=,\n') + csv.write("sep=,\n") - csv_writer = writer(csv, dialect='excel') + csv_writer = writer(csv, dialect="excel") csv_writer.writerow(header) for entry in data: - metadata = entry.get('metadata') + metadata = entry.get("metadata") # parse timestamp - timestamp = datetime.strptime(metadata.get('timestamp_local'), ACRClient.TS_FMT) + timestamp = datetime.strptime(metadata.get("timestamp_local"), ACRClient.TS_FMT) - ts_date = timestamp.strftime('%d/%m/%y') - ts_time = timestamp.strftime('%H:%M:%S') - duration = timedelta(seconds=metadata.get('played_duration')) + ts_date = timestamp.strftime("%d/%m/%y") + ts_time = timestamp.strftime("%H:%M:%S") + duration = timedelta(seconds=metadata.get("played_duration")) try: - music = metadata.get('music')[0] + music = metadata.get("music")[0] except TypeError: - music = metadata.get('custom_files')[0] - title = music.get('title') - if music.get('artists') is not None: - artist = ', '.join([a.get('name') for a in music.get('artists')]) - elif music.get('artist') is not None: - artist = music.get('artist') - elif music.get('Artist') is not None: - """ - Uppercase is a hack needed for Jun 2021 since there is a 'wrong' entry in the database. - Going forward the record will be available as 'artist' in lowercase. - """ - artist = music.get('Artist') + music = metadata.get("custom_files")[0] + title = music.get("title") + if music.get("artists") is not None: + artist = ", ".join([a.get("name") for a in music.get("artists")]) + elif music.get("artist") is not None: + artist = music.get("artist") + elif music.get("Artist") is not None: + # Uppercase is a hack needed for Jun 2021 since there is a 'wrong' entry + # in the database. Going forward the record will be available as 'artist' + # in lowercase. + # @TODO remove once is waaaay in the past + artist = music.get("Artist") else: - artist = '' - if music.get('external_ids') and len(music.get('external_ids')) > 0: - isrc = music.get('external_ids').get('isrc') - elif music.get('isrc'): - isrc = music.get('isrc') + artist = "" + if music.get("external_ids") and len(music.get("external_ids")) > 0: + isrc = music.get("external_ids").get("isrc") + elif music.get("isrc"): + isrc = music.get("isrc") else: - isrc = '' - label = music.get('label') + isrc = "" + label = music.get("label") csv_writer.writerow([ts_date, ts_time, duration, title, artist, isrc, label]) return csv.getvalue() def write_csv(filename, csv): - """Write contents of `csv` to file + """Write contents of `csv` to file. Arguments: filename: The file to write to. csv: The data to write to `filename`. """ - with open(filename, mode='w') as csvfile: + with open(filename, mode="w", encoding="utf-8") as csvfile: csvfile.write(csv) # reducing the arguments even more does not seem practical -#pylint: disable-msg=too-many-arguments,invalid-name +# pylint: disable-msg=too-many-arguments,invalid-name def create_message(sender, recipient, subject, text, filename, csv, cc=None, bcc=None): - """Create email message + """Create email message. Arguments: sender: The sender of the email. Login will be made with this user. @@ -280,29 +342,30 @@ def create_message(sender, recipient, subject, text, filename, csv, cc=None, bcc csv: The attachment data. """ msg = MIMEMultipart() - msg['From'] = sender - msg['To'] = recipient + msg["From"] = sender + msg["To"] = recipient if cc: - msg['Cc'] = cc + msg["Cc"] = cc if bcc: - msg['Bcc'] = bcc - msg['Date'] = formatdate(localtime=True) - msg['Subject'] = subject + msg["Bcc"] = bcc + msg["Date"] = formatdate(localtime=True) + msg["Subject"] = subject # set body msg.attach(MIMEText(text)) # attach csv - part = MIMEBase('text', 'csv') - part.set_payload(csv.encode('utf-8')) + part = MIMEBase("text", "csv") + part.set_payload(csv.encode("utf-8")) encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="{}"'.format(basename(filename))) + part.add_header( + "Content-Disposition", f'attachment; filename="{basename(filename)}' + ) msg.attach(part) return msg -#pylint: enable-msg=too-many-arguments,invalid-name -def send_message(msg, server='127.0.0.1', login=None, password=None): - """Send email +def send_message(msg, server="127.0.0.1", login=None, password=None): + """Send email. Arguments: msg: The message to send (an email.messag.Message object) @@ -315,44 +378,59 @@ def send_message(msg, server='127.0.0.1', login=None, password=None): if login: smtp.login(login, password) else: - smtp.login(msg['From'], password) + smtp.login(msg["From"], password) smtp.send_message(msg) def main(): - """main function""" - default_config_file = basename(__file__).replace('.py', '.conf') + """Entrypoint for SUISA Sendemeldung .""" + default_config_file = basename(__file__).replace(".py", ".conf") # config file in /etc gets overriden by the one in $HOME which gets overriden by the one in the # current directory default_config_files = [ - '/etc/' + default_config_file, - expanduser('~') + '/' + default_config_file, - default_config_file + "/etc/" + default_config_file, + expanduser("~") + "/" + default_config_file, + default_config_file, ] parser = ArgumentParser( - default_config_files=default_config_files, - description='ACRCloud client for SUISA reporting @ RaBe.') + default_config_files=default_config_files, + description="ACRCloud client for SUISA reporting @ RaBe.", + ) args = get_arguments(parser) start_date, end_date = parse_date(args) filename = parse_filename(args, start_date) client = ACRClient(args.access_key) - data = client.get_interval_data(args.stream_id, start_date, end_date, timezone=args.timezone) + data = client.get_interval_data( + args.stream_id, start_date, end_date, timezone=args.timezone + ) csv = get_csv(merge_duplicates(data)) if args.email: email_subject = start_date.strftime(args.email_subject) email_text = start_date.strftime(args.email_text) - email_text = email_text.replace('\\n', '\n') - msg = create_message(args.email_from, args.email_to, email_subject, email_text, filename, - csv, cc=args.email_cc, bcc=args.email_bcc) - send_message(msg, server=args.email_server, - login=args.email_login, password=args.email_pass) + email_text = email_text.replace("\\n", "\n") + msg = create_message( + args.email_from, + args.email_to, + email_subject, + email_text, + filename, + csv, + cc=args.email_cc, + bcc=args.email_bcc, + ) + send_message( + msg, + server=args.email_server, + login=args.email_login, + password=args.email_pass, + ) if args.csv: write_csv(filename, csv) if args.stdout: print(csv) -if __name__ == '__main__': +if __name__ == "__main__": # pragma: no cover main() From ecb47dc68c27db82c4fc54e19a00eb194f92202f Mon Sep 17 00:00:00 2001 From: Lucas Bickel Date: Fri, 4 Feb 2022 13:50:48 +0100 Subject: [PATCH 2/2] chore: use editorconfig instead of .vscode dir For this to work with VS Code users need to install the editorconfig.editorconfig extension --- .editorconfig | 2 ++ .vscode/settings.json | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 .editorconfig delete mode 100644 .vscode/settings.json diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..edc849a5 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,2 @@ +[*.py] +profile = black diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index b7368caa..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.formatting.provider": "black" -}