From b581d32c7da1f45ce6596eca4f4b9941db821c73 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 01:16:55 +0000 Subject: [PATCH 01/11] Rename cmd and lib files. --- cmd_consume.py => tklamq_consume.py | 0 cmd_declare.py => tklamq_declare.py | 0 {tklamq => tklamq_lib}/__init__.py | 0 {tklamq => tklamq_lib}/amqp.py | 0 {tklamq => tklamq_lib}/crypto.py | 0 cmd_publish.py => tklamq_publish.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename cmd_consume.py => tklamq_consume.py (100%) rename cmd_declare.py => tklamq_declare.py (100%) rename {tklamq => tklamq_lib}/__init__.py (100%) rename {tklamq => tklamq_lib}/amqp.py (100%) rename {tklamq => tklamq_lib}/crypto.py (100%) rename cmd_publish.py => tklamq_publish.py (100%) diff --git a/cmd_consume.py b/tklamq_consume.py similarity index 100% rename from cmd_consume.py rename to tklamq_consume.py diff --git a/cmd_declare.py b/tklamq_declare.py similarity index 100% rename from cmd_declare.py rename to tklamq_declare.py diff --git a/tklamq/__init__.py b/tklamq_lib/__init__.py similarity index 100% rename from tklamq/__init__.py rename to tklamq_lib/__init__.py diff --git a/tklamq/amqp.py b/tklamq_lib/amqp.py similarity index 100% rename from tklamq/amqp.py rename to tklamq_lib/amqp.py diff --git a/tklamq/crypto.py b/tklamq_lib/crypto.py similarity index 100% rename from tklamq/crypto.py rename to tklamq_lib/crypto.py diff --git a/cmd_publish.py b/tklamq_publish.py similarity index 100% rename from cmd_publish.py rename to tklamq_publish.py From 4188d53ec3a72b9fe4fee925e34590c1ca396505 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 01:17:57 +0000 Subject: [PATCH 02/11] Move to pybuild Debian packaging. --- Makefile | 83 ------------------------------------------- debian/control | 12 +++---- debian/copyright | 7 ++-- debian/rules | 9 ++--- debian/tklamq.install | 5 +++ setup.py | 76 ++++++--------------------------------- 6 files changed, 28 insertions(+), 164 deletions(-) delete mode 100644 Makefile create mode 100644 debian/tklamq.install diff --git a/Makefile b/Makefile deleted file mode 100644 index 511b036..0000000 --- a/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# standard Python project Makefile -progname = $(shell awk '/^Source/ {print $$2}' debian/control) -name= - -prefix = /usr/local -PATH_BIN = $(prefix)/bin -PATH_INSTALL_LIB = $(prefix)/lib/$(progname) -PATH_DIST := $(progname)-$(shell date +%F) - -all: help - -debug: - $(foreach v, $V, $(warning $v = $($v))) - @true - -dist: clean - -mkdir -p $(PATH_DIST) - - -cp -a .git .gitignore $(PATH_DIST) - -cp -a *.sh *.c *.py Makefile pylib/ libexec* $(PATH_DIST) - - tar jcvf $(PATH_DIST).tar.bz2 $(PATH_DIST) - rm -rf $(PATH_DIST) - -### Extendable targets - -# target: help -help: - @echo '=== Targets:' - @echo 'install [ prefix=path/to/usr ] # default: prefix=$(value prefix)' - @echo 'uninstall [ prefix=path/to/usr ]' - @echo - @echo 'clean' - @echo - @echo 'dist # create distribution tarball' - -# DRY macros -truepath = $(shell echo $1 | sed -e 's/^debian\/$(progname)//') -libpath = $(call truepath,$(PATH_INSTALL_LIB))/$$(basename $1) -subcommand = $(progname)-$$(echo $1 | sed 's|.*/||; s/^cmd_//; s/_/-/g; s/.py$$//') -echo-do = echo $1; $1 - -# first argument: code we execute if there is just one executable module -# second argument: code we execute if there is more than on executable module -define with-py-executables - @modules=$$(find -maxdepth 1 -type f -name '*.py' -perm -100); \ - modules_len=$$(echo $$modules | wc -w); \ - if [ $$modules_len = 1 ]; then \ - module=$$modules; \ - $(call echo-do, $1); \ - elif [ $$modules_len -gt 1 ]; then \ - for module in $$modules; do \ - $(call echo-do, $2); \ - done; \ - fi; -endef - -# target: install -install: - @echo - @echo \*\* CONFIG: prefix = $(prefix) \*\* - @echo - - install -d $(PATH_BIN) $(PATH_INSTALL_LIB) - python setup.py install --prefix $(prefix) --install-layout=deb - cp cmd_*.py $(PATH_INSTALL_LIB) - - $(call with-py-executables, \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(progname), \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(call subcommand, $$module)) - -# target: uninstall -uninstall: - rm -rf $(PATH_INSTALL_LIB) - - $(call with-py-executables, \ - rm -f $(PATH_BIN)/$(progname), \ - rm -f $(PATH_BIN)/$(call subcommand, $$module)) - -# target: clean -clean: - rm -f *.pyc *.pyo _$(progname) - rm -rf build diff --git a/debian/control b/debian/control index a19145f..fc852f3 100644 --- a/debian/control +++ b/debian/control @@ -1,11 +1,12 @@ Source: tklamq Section: misc Priority: optional -Maintainer: Alon Swartz +Maintainer: Jeremy Davis Build-Depends: debhelper (>= 10), dh-python, - python-all (>= 2.6.6-3~), + dh-exec, + python3-all (>= 3.6~), Standards-Version: 4.0.0 Package: tklamq @@ -13,8 +14,7 @@ Architecture: any Depends: ${misc:Depends}, ${python:Depends}, - python-amqplib, - python-kombu, - python-crypto, - python-simplejson, + python3-amqplib, + python3-kombu, + python3-crypto, Description: TurnKey Linux Advanced Message Queue (AMQ) client diff --git a/debian/copyright b/debian/copyright index c6ee89a..8f72cad 100644 --- a/debian/copyright +++ b/debian/copyright @@ -2,11 +2,12 @@ Author: Alon Swartz License: - Copyright (C) 2010 Alon Swartz + Copyright (C) 2010-2021 Alon Swartz + Copyright (C) 2022 TurnKey GNU/Linux This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or + the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -19,4 +20,4 @@ License: Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA On Debian and Ubuntu systems, the complete text of the GNU General Public -License can be found in /usr/share/common-licenses/GPL file. +License can be found in /usr/share/common-licenses/GPL-3 file. diff --git a/debian/rules b/debian/rules index 500f8be..88627e4 100755 --- a/debian/rules +++ b/debian/rules @@ -1,9 +1,6 @@ -#! /usr/bin/make -f +#!/usr/bin/make -f -include /usr/share/dpkg/pkg-info.mk +export DEB_BUILD_MAINT_OPTIONS = hardening=+all %: - dh $@ --with python2 --buildsystem=makefile - -override_dh_auto_install: - dh_auto_install -- prefix=debian/$(DEB_SOURCE)/usr + dh $@ --buildsystem=pybuild diff --git a/debian/tklamq.install b/debian/tklamq.install new file mode 100644 index 0000000..089e3d6 --- /dev/null +++ b/debian/tklamq.install @@ -0,0 +1,5 @@ +#!/usr/bin/dh-exec + +tklamq_consume.py => /usr/bin/tklamq-consume +tklamq_declare.py => /usr/bin/tklamq-declare +tklamq_publish.py => /usr/bin/tklamq-publish diff --git a/setup.py b/setup.py index 399c887..4f8ce2d 100644 --- a/setup.py +++ b/setup.py @@ -1,69 +1,13 @@ -# Copyright (c) 2010 Alon Swartz - all rights reserved - -import re -import os.path -import commands +#!/usr/bin/python3 from distutils.core import setup -class ExecError(Exception): - pass - -def _getoutput(command): - status, output = commands.getstatusoutput(command) - if status != 0: - raise ExecError() - return output - -def get_version(): - if not os.path.exists("debian/changelog"): - version = _getoutput("autoversion HEAD") - return version - else: - output = _getoutput("dpkg-parsechangelog") - version = [ line.split(" ")[1] - for line in output.split("\n") - if line.startswith("Version:") ][0] - return version - -def parse_control(control): - """parse control fields -> dict""" - d = {} - for line in control.split("\n"): - if not line or line[0] == " ": - continue - line = line.strip() - i = line.index(':') - key = line[:i] - val = line[i + 2:] - d[key] = val - - return d - -def parse_email(email): - m = re.match(r'(.*)\s*<(.*)>', email.strip()) - if m: - name, address = m.groups() - else: - name = "" - address = email - - return name.strip(), address.strip() - -def main(): - control_fields = parse_control(file("debian/control").read()) - maintainer = control_fields['Maintainer'] - maintainer_name, maintainer_email = parse_email(maintainer) - - setup(packages = ['tklamq'], - # non-essential meta-data - name=control_fields['Source'], - version=get_version(), - maintainer=maintainer_name, - maintainer_email=maintainer_email, - description=control_fields['Description']) - -if __name__=="__main__": - main() - - +setup( + name="tklamq", + version="0.10", + author="Jeremy Davis", + author_email="jeremy@turnkeylinux.org", + url="https://github.com/turnkeylinux/tklamq", + packages=["tklamq_lib"], + scripts=["tklamq_consume.py", "tklamq_declare.py", "tklamq_publish.py"] +) From d50026a382b94a8616ad04435b1f8ee33a12a493 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 01:32:10 +0000 Subject: [PATCH 03/11] Run 2to3 over code. --- tklamq_consume.py | 8 ++++---- tklamq_declare.py | 4 ++-- tklamq_lib/amqp.py | 6 +++--- tklamq_publish.py | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tklamq_consume.py b/tklamq_consume.py index fbf0126..e0e57a6 100755 --- a/tklamq_consume.py +++ b/tklamq_consume.py @@ -15,12 +15,12 @@ from tklamq.amqp import connect, decode_message def usage(): - print >> sys.stderr, "Syntax: %s " % sys.argv[0] - print >> sys.stderr, __doc__, env_doc + print("Syntax: %s " % sys.argv[0], file=sys.stderr) + print(__doc__, env_doc, file=sys.stderr) sys.exit(1) def fatal(s): - print >> sys.stderr, "error: " + str(s) + print("error: " + str(s), file=sys.stderr) sys.exit(1) def decrypt_callback(message_data, message): @@ -31,7 +31,7 @@ def decrypt_callback(message_data, message): fatal('TKLAMQ_SECRET not specified, cannot decrypt cipher text') sender, content, timestamp = decode_message(message_data, secret) - print content + print(content) message.ack() diff --git a/tklamq_declare.py b/tklamq_declare.py index 2b0c755..45d6112 100755 --- a/tklamq_declare.py +++ b/tklamq_declare.py @@ -15,11 +15,11 @@ def usage(): syntax = "Syntax: %s " % sys.argv[0] - print >> sys.stderr, syntax, __doc__, env_doc + print(syntax, __doc__, env_doc, file=sys.stderr) sys.exit(1) def fatal(s): - print >> sys.stderr, "error: " + str(s) + print("error: " + str(s), file=sys.stderr) sys.exit(1) def main(): diff --git a/tklamq_lib/amqp.py b/tklamq_lib/amqp.py index 6399ae5..c5eda14 100644 --- a/tklamq_lib/amqp.py +++ b/tklamq_lib/amqp.py @@ -18,7 +18,7 @@ from kombu.connection import BrokerConnection from kombu.compat import Publisher, Consumer -from crypto import encrypt, decrypt +from .crypto import encrypt, decrypt class Error(Exception): pass @@ -99,7 +99,7 @@ def publish(self, exchange, routing_key, message, def _consume_callback(message_data, message): """default consume callback if not specified""" - print message_data + print(message_data) message.ack() def connect(): @@ -156,7 +156,7 @@ def decode_message(message_data, secret=None): """ sender = str(message_data['sender']) content = base64.urlsafe_b64decode(str(message_data['content'])) - timestamp = datetime(*map(lambda f: int(f), message_data['timestamp-utc'])) + timestamp = datetime(*[int(f) for f in message_data['timestamp-utc']]) if message_data['encrypted']: content = decrypt(content, secret) diff --git a/tklamq_publish.py b/tklamq_publish.py index 74fe794..2ea01b0 100755 --- a/tklamq_publish.py +++ b/tklamq_publish.py @@ -25,15 +25,15 @@ def usage(e=None): if e: - print >> sys.stderr, "error: " + str(e) + print("error: " + str(e), file=sys.stderr) - print >> sys.stderr, "Syntax: %s [-opts] " % sys.argv[0] - print >> sys.stderr, "Message is stdin" - print >> sys.stderr, __doc__, env_doc + print("Syntax: %s [-opts] " % sys.argv[0], file=sys.stderr) + print("Message is stdin", file=sys.stderr) + print(__doc__, env_doc, file=sys.stderr) sys.exit(1) def fatal(s): - print >> sys.stderr, "error: " + str(s) + print("error: " + str(s), file=sys.stderr) sys.exit(1) def main(): @@ -41,7 +41,7 @@ def main(): opts, args = getopt.gnu_getopt(sys.argv[1:], 'i:s:e:jh', ['input=', 'sender=', 'encrypt', 'json', 'non-persistent']) - except getopt.GetoptError, e: + except getopt.GetoptError as e: usage(e) inputfile = None From 5610ff0bfcddb3b8f82d1522765bdd80b9b3e7c5 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 01:48:14 +0000 Subject: [PATCH 04/11] Update copyright and python shebang. --- tklamq_consume.py | 5 +++-- tklamq_declare.py | 5 +++-- tklamq_lib/amqp.py | 3 ++- tklamq_lib/crypto.py | 3 ++- tklamq_publish.py | 5 +++-- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tklamq_consume.py b/tklamq_consume.py index e0e57a6..ce18f5d 100755 --- a/tklamq_consume.py +++ b/tklamq_consume.py @@ -1,5 +1,6 @@ -#!/usr/bin/python -# Copyright (c) 2010 Alon Swartz - all rights reserved +#!/usr/bin/python3 +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved """ Arguments: diff --git a/tklamq_declare.py b/tklamq_declare.py index 45d6112..4e20e70 100755 --- a/tklamq_declare.py +++ b/tklamq_declare.py @@ -1,5 +1,6 @@ -#!/usr/bin/python -# Copyright (c) 2010 Alon Swartz - all rights reserved +#!/usr/bin/python3 +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved """ Arguments: diff --git a/tklamq_lib/amqp.py b/tklamq_lib/amqp.py index c5eda14..4e72949 100644 --- a/tklamq_lib/amqp.py +++ b/tklamq_lib/amqp.py @@ -1,4 +1,5 @@ -# Copyright (c) 2010 Alon Swartz - all rights reserved +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved """ Environment variables: diff --git a/tklamq_lib/crypto.py b/tklamq_lib/crypto.py index bab6712..9a12cff 100644 --- a/tklamq_lib/crypto.py +++ b/tklamq_lib/crypto.py @@ -1,4 +1,5 @@ -# Copyright (c) 2010 Alon Swartz - all rights reserved +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved from Crypto.Cipher import AES from hashlib import sha1 diff --git a/tklamq_publish.py b/tklamq_publish.py index 2ea01b0..b9d7e40 100755 --- a/tklamq_publish.py +++ b/tklamq_publish.py @@ -1,5 +1,6 @@ -#!/usr/bin/python -# Copyright (c) 2010 Alon Swartz - all rights reserved +#!/usr/bin/python3 +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved """ Arguments: From 88ae008fcec365bd86f38d3101777266574a1909 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 01:49:55 +0000 Subject: [PATCH 05/11] Update dependency: pycrypto -> pycryptodome. --- debian/control | 2 +- tklamq_lib/crypto.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index fc852f3..4c3e13c 100644 --- a/debian/control +++ b/debian/control @@ -16,5 +16,5 @@ Depends: ${python:Depends}, python3-amqplib, python3-kombu, - python3-crypto, + python3-pycryptodome, Description: TurnKey Linux Advanced Message Queue (AMQ) client diff --git a/tklamq_lib/crypto.py b/tklamq_lib/crypto.py index 9a12cff..4c36f1e 100644 --- a/tklamq_lib/crypto.py +++ b/tklamq_lib/crypto.py @@ -1,7 +1,7 @@ # Copyright (c) 2010-2021 Alon Swartz - all rights reserved # Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved -from Crypto.Cipher import AES +from Cryptodome.Cipher import AES from hashlib import sha1 class CheckSumError(Exception): From b835cddb5f866c26aa364d7b60ccdf9a192b37bb Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 02:03:17 +0000 Subject: [PATCH 06/11] Code layout tidy up. --- tklamq_consume.py | 8 ++++++-- tklamq_declare.py | 6 ++++-- tklamq_lib/amqp.py | 28 ++++++++++++++++++---------- tklamq_lib/crypto.py | 9 ++++++--- tklamq_publish.py | 5 ++++- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/tklamq_consume.py b/tklamq_consume.py index ce18f5d..f078bff 100755 --- a/tklamq_consume.py +++ b/tklamq_consume.py @@ -15,15 +15,18 @@ from tklamq.amqp import __doc__ as env_doc from tklamq.amqp import connect, decode_message + def usage(): print("Syntax: %s " % sys.argv[0], file=sys.stderr) print(__doc__, env_doc, file=sys.stderr) sys.exit(1) + def fatal(s): print("error: " + str(s), file=sys.stderr) sys.exit(1) + def decrypt_callback(message_data, message): encrypted = message_data['encrypted'] secret = os.getenv('TKLAMQ_SECRET', None) @@ -36,15 +39,16 @@ def decrypt_callback(message_data, message): message.ack() + def main(): if not len(sys.argv) == 2: usage() queue = sys.argv[1] - + conn = connect() conn.consume(queue, callback=decrypt_callback) + if __name__ == "__main__": main() - diff --git a/tklamq_declare.py b/tklamq_declare.py index 4e20e70..17bece6 100755 --- a/tklamq_declare.py +++ b/tklamq_declare.py @@ -14,21 +14,24 @@ from tklamq.amqp import __doc__ as env_doc from tklamq.amqp import connect + def usage(): syntax = "Syntax: %s " % sys.argv[0] print(syntax, __doc__, env_doc, file=sys.stderr) sys.exit(1) + def fatal(s): print("error: " + str(s), file=sys.stderr) sys.exit(1) + def main(): if not len(sys.argv) == 5: usage() exchange, exchange_type, binding, queue = sys.argv[1:] - if not exchange_type in ('direct', 'topic', 'fanout'): + if exchange_type not in ('direct', 'topic', 'fanout'): fatal("Invalid exchange type") conn = connect() @@ -37,4 +40,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/tklamq_lib/amqp.py b/tklamq_lib/amqp.py index 4e72949..0cd652b 100644 --- a/tklamq_lib/amqp.py +++ b/tklamq_lib/amqp.py @@ -21,22 +21,24 @@ from .crypto import encrypt, decrypt -class Error(Exception): + +class TklAmqError(Exception): pass + class Connection: def __init__(self, hostname, port, vhost, userid, password): """connects to broker and provides convenience methods""" self.broker = BrokerConnection(hostname=hostname, port=port, userid=userid, password=password, virtual_host=vhost) - + def __del__(self): self.broker.close() def declare(self, exchange, exchange_type, binding="", queue=""): """declares the exchange, the queue and binds the queue to the exchange - + exchange - exchange name exchange_type - direct, topic, fanout binding - binding to queue (optional) @@ -44,7 +46,8 @@ def declare(self, exchange, exchange_type, binding="", queue=""): """ if (binding and not queue) or (queue and not binding): if queue and not exchange_type == "fanout": - raise Error("binding and queue are not mutually exclusive") + raise TklAmqError( + "binding and queue are not mutually exclusive") consumer = Consumer(connection=self.broker, exchange=exchange, exchange_type=exchange_type, @@ -54,9 +57,10 @@ def declare(self, exchange, exchange_type, binding="", queue=""): def consume(self, queue, limit=None, callback=None, auto_declare=False): """consume messages in queue - + queue - name of queue - limit - amount of messages to iterate through (default: no limit) + limit - amount of messages to iterate through + (default: no limit) callback - method to call when a new message is received must take two arguments: message_data, message @@ -80,12 +84,14 @@ def consume(self, queue, limit=None, callback=None, auto_declare=False): def publish(self, exchange, routing_key, message, auto_declare=False, persistent=True): """publish a message to exchange using routing_key - + exchange - name of exchange - routing_key - interpretation of routing key depends on exchange type + routing_key - interpretation of routing key depends on exchange + type message - message content to send auto_declare - automatically declare the exchange (default: false) - persistent - store message on disk as well as memory (default: True) + persistent - store message on disk as well as memory + (default: True) """ delivery_mode = 2 if not persistent: @@ -98,11 +104,13 @@ def publish(self, exchange, routing_key, message, publisher.send(message, delivery_mode=delivery_mode) publisher.close() + def _consume_callback(message_data, message): """default consume callback if not specified""" print(message_data) message.ack() + def connect(): """convenience method using environment variables""" BROKER_HOST = os.getenv('BROKER_HOST', 'localhost') @@ -144,6 +152,7 @@ def encode_message(sender, content, secret=None): return message + def decode_message(message_data, secret=None): """decode message envelope args @@ -164,4 +173,3 @@ def decode_message(message_data, secret=None): content = json.loads(content) return sender, content, timestamp - diff --git a/tklamq_lib/crypto.py b/tklamq_lib/crypto.py index 4c36f1e..34aa090 100644 --- a/tklamq_lib/crypto.py +++ b/tklamq_lib/crypto.py @@ -4,9 +4,11 @@ from Cryptodome.Cipher import AES from hashlib import sha1 -class CheckSumError(Exception): + +class TklAmqCheckSumError(Exception): pass + def _lazysecret(secret, blocksize=32, padding='}'): """pads secret if not legal AES block size (16, 24, 32)""" if not len(secret) in (16, 24, 32): @@ -14,6 +16,7 @@ def _lazysecret(secret, blocksize=32, padding='}'): return secret + def encrypt(plaintext, secret, lazy=True, checksum=True): """encrypt plaintext with secret plaintext - content to encrypt @@ -30,6 +33,7 @@ def encrypt(plaintext, secret, lazy=True, checksum=True): return encobj.encrypt(plaintext) + def decrypt(ciphertext, secret, lazy=True, checksum=True): """decrypt ciphertext with secret ciphertext - encrypted content to decrypt @@ -45,7 +49,6 @@ def decrypt(ciphertext, secret, lazy=True, checksum=True): if checksum: digest, plaintext = (plaintext[-20:], plaintext[:-20]) if not digest == sha1(plaintext).digest(): - raise CheckSumError("checksum mismatch") + raise TklAmqCheckSumError("checksum mismatch") return plaintext - diff --git a/tklamq_publish.py b/tklamq_publish.py index b9d7e40..fa3fd48 100755 --- a/tklamq_publish.py +++ b/tklamq_publish.py @@ -24,6 +24,7 @@ from tklamq.amqp import __doc__ as env_doc from tklamq.amqp import connect, encode_message + def usage(e=None): if e: print("error: " + str(e), file=sys.stderr) @@ -33,10 +34,12 @@ def usage(e=None): print(__doc__, env_doc, file=sys.stderr) sys.exit(1) + def fatal(s): print("error: " + str(s), file=sys.stderr) sys.exit(1) + def main(): try: opts, args = getopt.gnu_getopt(sys.argv[1:], 'i:s:e:jh', @@ -95,6 +98,6 @@ def main(): conn = connect() conn.publish(exchange, routing_key, message, persistent=opt_persistent) + if __name__ == "__main__": main() - From fafc5c81a45a921de2cb9c79f9b89a1f6845293e Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 05:53:09 +0000 Subject: [PATCH 07/11] Port to use argparse instead of getopt. --- tklamq_consume.py | 37 ++++++------ tklamq_declare.py | 53 ++++++++--------- tklamq_lib/amqp.py | 5 +- tklamq_publish.py | 143 +++++++++++++++++++++------------------------ 4 files changed, 112 insertions(+), 126 deletions(-) diff --git a/tklamq_consume.py b/tklamq_consume.py index f078bff..27f1c82 100755 --- a/tklamq_consume.py +++ b/tklamq_consume.py @@ -1,25 +1,13 @@ #!/usr/bin/python3 # Copyright (c) 2010-2021 Alon Swartz - all rights reserved # Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved -""" -Arguments: - - queue queue to consume messages from - -If message content is encrypted, TKLAMQ_SECRET will be used as decryption key -""" import os import sys +import argparse -from tklamq.amqp import __doc__ as env_doc -from tklamq.amqp import connect, decode_message - - -def usage(): - print("Syntax: %s " % sys.argv[0], file=sys.stderr) - print(__doc__, env_doc, file=sys.stderr) - sys.exit(1) +from tklamq_lib.amqp import __doc__ as env_doc +from tklamq_lib.amqp import connect, decode_message def fatal(s): @@ -41,13 +29,22 @@ def decrypt_callback(message_data, message): def main(): - if not len(sys.argv) == 2: - usage() - - queue = sys.argv[1] + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + prog="tklamq-consume", + description="Consumes messages from a queue", + epilog=(f"{env_doc}" + " TKLAMQ_SECRET" + " decryption key (required if encrypted)") + ) + parser.add_argument( + "queue", + help="queue to consume messages from", + ) + args = parser.parse_args() conn = connect() - conn.consume(queue, callback=decrypt_callback) + conn.consume(args.queue, callback=decrypt_callback) if __name__ == "__main__": diff --git a/tklamq_declare.py b/tklamq_declare.py index 17bece6..a0ae3d4 100755 --- a/tklamq_declare.py +++ b/tklamq_declare.py @@ -1,41 +1,42 @@ #!/usr/bin/python3 # Copyright (c) 2010-2021 Alon Swartz - all rights reserved # Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved -""" -Arguments: - - exchange name of exchange - exchange_type exchange type (direct, topic, fanout) - binding queue binding (used by routing_key) - queue queue to bind to exchange using binding -""" import sys +import argparse + from tklamq.amqp import __doc__ as env_doc from tklamq.amqp import connect -def usage(): - syntax = "Syntax: %s " % sys.argv[0] - print(syntax, __doc__, env_doc, file=sys.stderr) - sys.exit(1) - - -def fatal(s): - print("error: " + str(s), file=sys.stderr) - sys.exit(1) - - def main(): - if not len(sys.argv) == 5: - usage() - - exchange, exchange_type, binding, queue = sys.argv[1:] - if exchange_type not in ('direct', 'topic', 'fanout'): - fatal("Invalid exchange type") + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + prog="tklamq-declare" + description="Declares an exchange/binding/queue", + epilog=env_doc, + ) + parser.add_argument( + "exchange", + help="name of exchange", + ) + parser.add_argument( + "exchange-type", + choices=['direct', 'topic', 'fanout'], + help="exchange type (direct, topic, fanout)", + ) + parser.add_argument( + "binding", + help="queue binding (used by routing_key)", + ) + parser.add_argument( + "queue", + help="queue to bind to exchange using binding", + ) + args = parser.parse_args() conn = connect() - conn.declare(exchange, exchange_type, binding, queue) + conn.declare(args.exchange, args.exchange_type, args.binding, args.queue) if __name__ == "__main__": diff --git a/tklamq_lib/amqp.py b/tklamq_lib/amqp.py index 0cd652b..3e9c0ff 100644 --- a/tklamq_lib/amqp.py +++ b/tklamq_lib/amqp.py @@ -2,8 +2,7 @@ # Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved """ -Environment variables: - +environment variables: BROKER_HOST default: localhost BROKER_PORT default: 5672 BROKER_USER default: guest @@ -13,7 +12,7 @@ import os import base64 -import simplejson as json +import json from datetime import datetime from kombu.connection import BrokerConnection diff --git a/tklamq_publish.py b/tklamq_publish.py index fa3fd48..683948d 100755 --- a/tklamq_publish.py +++ b/tklamq_publish.py @@ -1,38 +1,14 @@ #!/usr/bin/python3 # Copyright (c) 2010-2021 Alon Swartz - all rights reserved # Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved -""" -Arguments: - - exchange name of exchange - routing_key interpretation of routing key depends on exchange type - -Options: - - -i --input=PATH content to send (- for stdin) - -j --json treat input as encoded json - -e --encrypt encrypt message using secret TKLAMQ_SECRET - -s --sender= message sender - --non-persistent only store message in memory (not to disk) -""" import os import sys -import getopt -import simplejson as json - -from tklamq.amqp import __doc__ as env_doc -from tklamq.amqp import connect, encode_message - +import argparse +import json -def usage(e=None): - if e: - print("error: " + str(e), file=sys.stderr) - - print("Syntax: %s [-opts] " % sys.argv[0], file=sys.stderr) - print("Message is stdin", file=sys.stderr) - print(__doc__, env_doc, file=sys.stderr) - sys.exit(1) +from tklamq_lib.amqp import __doc__ as env_doc +from tklamq_lib.amqp import connect, encode_message def fatal(s): @@ -40,63 +16,76 @@ def fatal(s): sys.exit(1) -def main(): - try: - opts, args = getopt.gnu_getopt(sys.argv[1:], 'i:s:e:jh', - ['input=', 'sender=', 'encrypt', 'json', 'non-persistent']) - - except getopt.GetoptError as e: - usage(e) - - inputfile = None - sender = None - opt_json = False - opt_encrypt = False - opt_persistent = True - for opt, val in opts: - if opt == '-h': - usage() +def file_exists(path: str) -> str: + #if path != '-' or not os.path.exists(path): + if not os.path.exists(path): + raise TypeError(f"File not found ({path})") + return path - if opt in ('-i', '--input'): - inputfile = val - - if opt in ('-s', '--sender'): - sender = val - - if opt in ('-e', '--encrypt'): - opt_encrypt = True - - if opt in ('-j', '--json'): - opt_json = True - - if opt == "--non-persistent": - opt_persistent = False - - if not len(args) == 2: - usage() - - secret = os.getenv('TKLAMQ_SECRET', None) - if opt_encrypt and not secret: - fatal('TKLAMQ_SECRET not specified, cannot encrypt') - - # unset secret if encryption was not specified - if not opt_encrypt: - secret = None +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + prog="tklamq-publish", + description="Publishes a message to exchange/routing_key", + epilog=env_doc, + ) + parser.add_argument( + "exchange", + help="name of exchange", + ) + parser.add_argument( + "routing_key", + help="interpretation of routing key depends on exchange type", + ) + parser.add_argument( + "--input", "-i", + type=file_exists, + #help="content to send (path/to/file or '-' for stdin)", + help="content to send (path/to/file)", + ) + parser.add_argument( + "--json", "-j", + action="store_true", + help="treat input as encoded json", + ) + parser.add_argument( + "--encrypt", "-e", + action="store_true", + help="encrypt message using secret TKLAMQ_SECRET", + ) + parser.add_argument( + "--sender", "-s", + help="message sender", + ) + parser.add_argument( + "--non-persistent", + dest="persistent", + action="store_false", # store persistance + help="only store message in memory (not to disk)", + ) + args = parser.parse_args() + + secret = None content = '' - if inputfile == '-': - content = sys.stdin.read() - elif inputfile: - content = file(inputfile).read() + if args.encrypt: + secret = os.getenv('TKLAMQ_SECRET', None) + if not secret: + fatal('TKLAMQ_SECRET not specified, cannot encrypt') + + if args.input: + #if args.input == '-': + with open(args.input) as fob: + content = fob.read() - if opt_json: + if args.json: content = json.loads(content) - exchange, routing_key = args - message = encode_message(sender, content, secret=secret) + message = encode_message(args.sender, content, secret=secret) conn = connect() - conn.publish(exchange, routing_key, message, persistent=opt_persistent) + conn.publish(args.exchange, args.routing_key, + message, persistent=args.persistent) if __name__ == "__main__": From 35b0e70d8faed1c82a50f2f815dac1d839795c5f Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 06:04:05 +0000 Subject: [PATCH 08/11] Minor bugfix --- tklamq_declare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tklamq_declare.py b/tklamq_declare.py index a0ae3d4..e6f0292 100755 --- a/tklamq_declare.py +++ b/tklamq_declare.py @@ -12,7 +12,7 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - prog="tklamq-declare" + prog="tklamq-declare", description="Declares an exchange/binding/queue", epilog=env_doc, ) From 6710d47a2b08b1755e08cec46c5b3571966cfb40 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 06:07:17 +0000 Subject: [PATCH 09/11] Make packaging work (not sure why rename isn't working?). --- debian/control | 1 - debian/tklamq.install | 8 +++----- tklamq_consume.py => tklamq-consume | 0 tklamq_declare.py => tklamq-declare | 0 tklamq_publish.py => tklamq-publish | 0 5 files changed, 3 insertions(+), 6 deletions(-) rename tklamq_consume.py => tklamq-consume (100%) rename tklamq_declare.py => tklamq-declare (100%) rename tklamq_publish.py => tklamq-publish (100%) diff --git a/debian/control b/debian/control index 4c3e13c..c10b241 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,6 @@ Maintainer: Jeremy Davis Build-Depends: debhelper (>= 10), dh-python, - dh-exec, python3-all (>= 3.6~), Standards-Version: 4.0.0 diff --git a/debian/tklamq.install b/debian/tklamq.install index 089e3d6..bf74900 100644 --- a/debian/tklamq.install +++ b/debian/tklamq.install @@ -1,5 +1,3 @@ -#!/usr/bin/dh-exec - -tklamq_consume.py => /usr/bin/tklamq-consume -tklamq_declare.py => /usr/bin/tklamq-declare -tklamq_publish.py => /usr/bin/tklamq-publish +tklamq-consume /usr/bin/ +tklamq-declare /usr/bin/ +tklamq-publish /usr/bin/ diff --git a/tklamq_consume.py b/tklamq-consume similarity index 100% rename from tklamq_consume.py rename to tklamq-consume diff --git a/tklamq_declare.py b/tklamq-declare similarity index 100% rename from tklamq_declare.py rename to tklamq-declare diff --git a/tklamq_publish.py b/tklamq-publish similarity index 100% rename from tklamq_publish.py rename to tklamq-publish From 71c9ed7e63108a3b26eb769b51924ebeecc77aef Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 06:09:43 +0000 Subject: [PATCH 10/11] Tweak build deps. --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index c10b241..e0ffe01 100644 --- a/debian/control +++ b/debian/control @@ -12,7 +12,7 @@ Package: tklamq Architecture: any Depends: ${misc:Depends}, - ${python:Depends}, + ${python3:Depends}, python3-amqplib, python3-kombu, python3-pycryptodome, From b997e3c16d64fc6137502c76566b5320623f6f53 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Mon, 7 Feb 2022 06:12:09 +0000 Subject: [PATCH 11/11] more pkging tweaks. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4f8ce2d..1633993 100644 --- a/setup.py +++ b/setup.py @@ -9,5 +9,5 @@ author_email="jeremy@turnkeylinux.org", url="https://github.com/turnkeylinux/tklamq", packages=["tklamq_lib"], - scripts=["tklamq_consume.py", "tklamq_declare.py", "tklamq_publish.py"] + scripts=["tklamq-consume", "tklamq-declare", "tklamq-publish"] )