diff --git a/Makefile b/Makefile deleted file mode 100644 index 511b036..0000000 --- a/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# standard Python project Makefile -progname = $(shell awk '/^Source/ {print $$2}' debian/control) -name= - -prefix = /usr/local -PATH_BIN = $(prefix)/bin -PATH_INSTALL_LIB = $(prefix)/lib/$(progname) -PATH_DIST := $(progname)-$(shell date +%F) - -all: help - -debug: - $(foreach v, $V, $(warning $v = $($v))) - @true - -dist: clean - -mkdir -p $(PATH_DIST) - - -cp -a .git .gitignore $(PATH_DIST) - -cp -a *.sh *.c *.py Makefile pylib/ libexec* $(PATH_DIST) - - tar jcvf $(PATH_DIST).tar.bz2 $(PATH_DIST) - rm -rf $(PATH_DIST) - -### Extendable targets - -# target: help -help: - @echo '=== Targets:' - @echo 'install [ prefix=path/to/usr ] # default: prefix=$(value prefix)' - @echo 'uninstall [ prefix=path/to/usr ]' - @echo - @echo 'clean' - @echo - @echo 'dist # create distribution tarball' - -# DRY macros -truepath = $(shell echo $1 | sed -e 's/^debian\/$(progname)//') -libpath = $(call truepath,$(PATH_INSTALL_LIB))/$$(basename $1) -subcommand = $(progname)-$$(echo $1 | sed 's|.*/||; s/^cmd_//; s/_/-/g; s/.py$$//') -echo-do = echo $1; $1 - -# first argument: code we execute if there is just one executable module -# second argument: code we execute if there is more than on executable module -define with-py-executables - @modules=$$(find -maxdepth 1 -type f -name '*.py' -perm -100); \ - modules_len=$$(echo $$modules | wc -w); \ - if [ $$modules_len = 1 ]; then \ - module=$$modules; \ - $(call echo-do, $1); \ - elif [ $$modules_len -gt 1 ]; then \ - for module in $$modules; do \ - $(call echo-do, $2); \ - done; \ - fi; -endef - -# target: install -install: - @echo - @echo \*\* CONFIG: prefix = $(prefix) \*\* - @echo - - install -d $(PATH_BIN) $(PATH_INSTALL_LIB) - python setup.py install --prefix $(prefix) --install-layout=deb - cp cmd_*.py $(PATH_INSTALL_LIB) - - $(call with-py-executables, \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(progname), \ - ln -fs $(call libpath, $$module) $(PATH_BIN)/$(call subcommand, $$module)) - -# target: uninstall -uninstall: - rm -rf $(PATH_INSTALL_LIB) - - $(call with-py-executables, \ - rm -f $(PATH_BIN)/$(progname), \ - rm -f $(PATH_BIN)/$(call subcommand, $$module)) - -# target: clean -clean: - rm -f *.pyc *.pyo _$(progname) - rm -rf build diff --git a/cmd_consume.py b/cmd_consume.py deleted file mode 100755 index fbf0126..0000000 --- a/cmd_consume.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010 Alon Swartz - all rights reserved -""" -Arguments: - - queue queue to consume messages from - -If message content is encrypted, TKLAMQ_SECRET will be used as decryption key -""" - -import os -import sys - -from tklamq.amqp import __doc__ as env_doc -from tklamq.amqp import connect, decode_message - -def usage(): - print >> sys.stderr, "Syntax: %s " % sys.argv[0] - print >> sys.stderr, __doc__, env_doc - sys.exit(1) - -def fatal(s): - print >> sys.stderr, "error: " + str(s) - sys.exit(1) - -def decrypt_callback(message_data, message): - encrypted = message_data['encrypted'] - secret = os.getenv('TKLAMQ_SECRET', None) - - if encrypted and not secret: - fatal('TKLAMQ_SECRET not specified, cannot decrypt cipher text') - - sender, content, timestamp = decode_message(message_data, secret) - print content - - message.ack() - -def main(): - if not len(sys.argv) == 2: - usage() - - queue = sys.argv[1] - - conn = connect() - conn.consume(queue, callback=decrypt_callback) - -if __name__ == "__main__": - main() - diff --git a/cmd_declare.py b/cmd_declare.py deleted file mode 100755 index 2b0c755..0000000 --- a/cmd_declare.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010 Alon Swartz - all rights reserved -""" -Arguments: - - exchange name of exchange - exchange_type exchange type (direct, topic, fanout) - binding queue binding (used by routing_key) - queue queue to bind to exchange using binding -""" - -import sys -from tklamq.amqp import __doc__ as env_doc -from tklamq.amqp import connect - -def usage(): - syntax = "Syntax: %s " % sys.argv[0] - print >> sys.stderr, syntax, __doc__, env_doc - sys.exit(1) - -def fatal(s): - print >> sys.stderr, "error: " + str(s) - sys.exit(1) - -def main(): - if not len(sys.argv) == 5: - usage() - - exchange, exchange_type, binding, queue = sys.argv[1:] - if not exchange_type in ('direct', 'topic', 'fanout'): - fatal("Invalid exchange type") - - conn = connect() - conn.declare(exchange, exchange_type, binding, queue) - - -if __name__ == "__main__": - main() - diff --git a/cmd_publish.py b/cmd_publish.py deleted file mode 100755 index 74fe794..0000000 --- a/cmd_publish.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010 Alon Swartz - all rights reserved -""" -Arguments: - - exchange name of exchange - routing_key interpretation of routing key depends on exchange type - -Options: - - -i --input=PATH content to send (- for stdin) - -j --json treat input as encoded json - -e --encrypt encrypt message using secret TKLAMQ_SECRET - -s --sender= message sender - --non-persistent only store message in memory (not to disk) -""" - -import os -import sys -import getopt -import simplejson as json - -from tklamq.amqp import __doc__ as env_doc -from tklamq.amqp import connect, encode_message - -def usage(e=None): - if e: - print >> sys.stderr, "error: " + str(e) - - print >> sys.stderr, "Syntax: %s [-opts] " % sys.argv[0] - print >> sys.stderr, "Message is stdin" - print >> sys.stderr, __doc__, env_doc - sys.exit(1) - -def fatal(s): - print >> sys.stderr, "error: " + str(s) - sys.exit(1) - -def main(): - try: - opts, args = getopt.gnu_getopt(sys.argv[1:], 'i:s:e:jh', - ['input=', 'sender=', 'encrypt', 'json', 'non-persistent']) - - except getopt.GetoptError, e: - usage(e) - - inputfile = None - sender = None - opt_json = False - opt_encrypt = False - opt_persistent = True - for opt, val in opts: - if opt == '-h': - usage() - - if opt in ('-i', '--input'): - inputfile = val - - if opt in ('-s', '--sender'): - sender = val - - if opt in ('-e', '--encrypt'): - opt_encrypt = True - - if opt in ('-j', '--json'): - opt_json = True - - if opt == "--non-persistent": - opt_persistent = False - - if not len(args) == 2: - usage() - - secret = os.getenv('TKLAMQ_SECRET', None) - if opt_encrypt and not secret: - fatal('TKLAMQ_SECRET not specified, cannot encrypt') - - # unset secret if encryption was not specified - if not opt_encrypt: - secret = None - - content = '' - if inputfile == '-': - content = sys.stdin.read() - elif inputfile: - content = file(inputfile).read() - - if opt_json: - content = json.loads(content) - - exchange, routing_key = args - message = encode_message(sender, content, secret=secret) - - conn = connect() - conn.publish(exchange, routing_key, message, persistent=opt_persistent) - -if __name__ == "__main__": - main() - diff --git a/debian/control b/debian/control index a19145f..e0ffe01 100644 --- a/debian/control +++ b/debian/control @@ -1,20 +1,19 @@ Source: tklamq Section: misc Priority: optional -Maintainer: Alon Swartz +Maintainer: Jeremy Davis Build-Depends: debhelper (>= 10), dh-python, - python-all (>= 2.6.6-3~), + python3-all (>= 3.6~), Standards-Version: 4.0.0 Package: tklamq Architecture: any Depends: ${misc:Depends}, - ${python:Depends}, - python-amqplib, - python-kombu, - python-crypto, - python-simplejson, + ${python3:Depends}, + python3-amqplib, + python3-kombu, + python3-pycryptodome, Description: TurnKey Linux Advanced Message Queue (AMQ) client diff --git a/debian/copyright b/debian/copyright index c6ee89a..8f72cad 100644 --- a/debian/copyright +++ b/debian/copyright @@ -2,11 +2,12 @@ Author: Alon Swartz License: - Copyright (C) 2010 Alon Swartz + Copyright (C) 2010-2021 Alon Swartz + Copyright (C) 2022 TurnKey GNU/Linux This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or + the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -19,4 +20,4 @@ License: Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA On Debian and Ubuntu systems, the complete text of the GNU General Public -License can be found in /usr/share/common-licenses/GPL file. +License can be found in /usr/share/common-licenses/GPL-3 file. diff --git a/debian/rules b/debian/rules index 500f8be..88627e4 100755 --- a/debian/rules +++ b/debian/rules @@ -1,9 +1,6 @@ -#! /usr/bin/make -f +#!/usr/bin/make -f -include /usr/share/dpkg/pkg-info.mk +export DEB_BUILD_MAINT_OPTIONS = hardening=+all %: - dh $@ --with python2 --buildsystem=makefile - -override_dh_auto_install: - dh_auto_install -- prefix=debian/$(DEB_SOURCE)/usr + dh $@ --buildsystem=pybuild diff --git a/debian/tklamq.install b/debian/tklamq.install new file mode 100644 index 0000000..bf74900 --- /dev/null +++ b/debian/tklamq.install @@ -0,0 +1,3 @@ +tklamq-consume /usr/bin/ +tklamq-declare /usr/bin/ +tklamq-publish /usr/bin/ diff --git a/setup.py b/setup.py index 399c887..1633993 100644 --- a/setup.py +++ b/setup.py @@ -1,69 +1,13 @@ -# Copyright (c) 2010 Alon Swartz - all rights reserved - -import re -import os.path -import commands +#!/usr/bin/python3 from distutils.core import setup -class ExecError(Exception): - pass - -def _getoutput(command): - status, output = commands.getstatusoutput(command) - if status != 0: - raise ExecError() - return output - -def get_version(): - if not os.path.exists("debian/changelog"): - version = _getoutput("autoversion HEAD") - return version - else: - output = _getoutput("dpkg-parsechangelog") - version = [ line.split(" ")[1] - for line in output.split("\n") - if line.startswith("Version:") ][0] - return version - -def parse_control(control): - """parse control fields -> dict""" - d = {} - for line in control.split("\n"): - if not line or line[0] == " ": - continue - line = line.strip() - i = line.index(':') - key = line[:i] - val = line[i + 2:] - d[key] = val - - return d - -def parse_email(email): - m = re.match(r'(.*)\s*<(.*)>', email.strip()) - if m: - name, address = m.groups() - else: - name = "" - address = email - - return name.strip(), address.strip() - -def main(): - control_fields = parse_control(file("debian/control").read()) - maintainer = control_fields['Maintainer'] - maintainer_name, maintainer_email = parse_email(maintainer) - - setup(packages = ['tklamq'], - # non-essential meta-data - name=control_fields['Source'], - version=get_version(), - maintainer=maintainer_name, - maintainer_email=maintainer_email, - description=control_fields['Description']) - -if __name__=="__main__": - main() - - +setup( + name="tklamq", + version="0.10", + author="Jeremy Davis", + author_email="jeremy@turnkeylinux.org", + url="https://github.com/turnkeylinux/tklamq", + packages=["tklamq_lib"], + scripts=["tklamq-consume", "tklamq-declare", "tklamq-publish"] +) diff --git a/tklamq-consume b/tklamq-consume new file mode 100755 index 0000000..27f1c82 --- /dev/null +++ b/tklamq-consume @@ -0,0 +1,51 @@ +#!/usr/bin/python3 +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved + +import os +import sys +import argparse + +from tklamq_lib.amqp import __doc__ as env_doc +from tklamq_lib.amqp import connect, decode_message + + +def fatal(s): + print("error: " + str(s), file=sys.stderr) + sys.exit(1) + + +def decrypt_callback(message_data, message): + encrypted = message_data['encrypted'] + secret = os.getenv('TKLAMQ_SECRET', None) + + if encrypted and not secret: + fatal('TKLAMQ_SECRET not specified, cannot decrypt cipher text') + + sender, content, timestamp = decode_message(message_data, secret) + print(content) + + message.ack() + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + prog="tklamq-consume", + description="Consumes messages from a queue", + epilog=(f"{env_doc}" + " TKLAMQ_SECRET" + " decryption key (required if encrypted)") + ) + parser.add_argument( + "queue", + help="queue to consume messages from", + ) + args = parser.parse_args() + + conn = connect() + conn.consume(args.queue, callback=decrypt_callback) + + +if __name__ == "__main__": + main() diff --git a/tklamq-declare b/tklamq-declare new file mode 100755 index 0000000..e6f0292 --- /dev/null +++ b/tklamq-declare @@ -0,0 +1,43 @@ +#!/usr/bin/python3 +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved + +import sys +import argparse + +from tklamq.amqp import __doc__ as env_doc +from tklamq.amqp import connect + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + prog="tklamq-declare", + description="Declares an exchange/binding/queue", + epilog=env_doc, + ) + parser.add_argument( + "exchange", + help="name of exchange", + ) + parser.add_argument( + "exchange-type", + choices=['direct', 'topic', 'fanout'], + help="exchange type (direct, topic, fanout)", + ) + parser.add_argument( + "binding", + help="queue binding (used by routing_key)", + ) + parser.add_argument( + "queue", + help="queue to bind to exchange using binding", + ) + args = parser.parse_args() + + conn = connect() + conn.declare(args.exchange, args.exchange_type, args.binding, args.queue) + + +if __name__ == "__main__": + main() diff --git a/tklamq-publish b/tklamq-publish new file mode 100755 index 0000000..683948d --- /dev/null +++ b/tklamq-publish @@ -0,0 +1,92 @@ +#!/usr/bin/python3 +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved + +import os +import sys +import argparse +import json + +from tklamq_lib.amqp import __doc__ as env_doc +from tklamq_lib.amqp import connect, encode_message + + +def fatal(s): + print("error: " + str(s), file=sys.stderr) + sys.exit(1) + + +def file_exists(path: str) -> str: + #if path != '-' or not os.path.exists(path): + if not os.path.exists(path): + raise TypeError(f"File not found ({path})") + return path + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + prog="tklamq-publish", + description="Publishes a message to exchange/routing_key", + epilog=env_doc, + ) + parser.add_argument( + "exchange", + help="name of exchange", + ) + parser.add_argument( + "routing_key", + help="interpretation of routing key depends on exchange type", + ) + parser.add_argument( + "--input", "-i", + type=file_exists, + #help="content to send (path/to/file or '-' for stdin)", + help="content to send (path/to/file)", + ) + parser.add_argument( + "--json", "-j", + action="store_true", + help="treat input as encoded json", + ) + parser.add_argument( + "--encrypt", "-e", + action="store_true", + help="encrypt message using secret TKLAMQ_SECRET", + ) + parser.add_argument( + "--sender", "-s", + help="message sender", + ) + parser.add_argument( + "--non-persistent", + dest="persistent", + action="store_false", # store persistance + help="only store message in memory (not to disk)", + ) + args = parser.parse_args() + + secret = None + content = '' + if args.encrypt: + secret = os.getenv('TKLAMQ_SECRET', None) + if not secret: + fatal('TKLAMQ_SECRET not specified, cannot encrypt') + + if args.input: + #if args.input == '-': + with open(args.input) as fob: + content = fob.read() + + if args.json: + content = json.loads(content) + + message = encode_message(args.sender, content, secret=secret) + + conn = connect() + conn.publish(args.exchange, args.routing_key, + message, persistent=args.persistent) + + +if __name__ == "__main__": + main() diff --git a/tklamq/__init__.py b/tklamq_lib/__init__.py similarity index 100% rename from tklamq/__init__.py rename to tklamq_lib/__init__.py diff --git a/tklamq/amqp.py b/tklamq_lib/amqp.py similarity index 88% rename from tklamq/amqp.py rename to tklamq_lib/amqp.py index 6399ae5..3e9c0ff 100644 --- a/tklamq/amqp.py +++ b/tklamq_lib/amqp.py @@ -1,8 +1,8 @@ -# Copyright (c) 2010 Alon Swartz - all rights reserved +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved """ -Environment variables: - +environment variables: BROKER_HOST default: localhost BROKER_PORT default: 5672 BROKER_USER default: guest @@ -12,30 +12,32 @@ import os import base64 -import simplejson as json +import json from datetime import datetime from kombu.connection import BrokerConnection from kombu.compat import Publisher, Consumer -from crypto import encrypt, decrypt +from .crypto import encrypt, decrypt + -class Error(Exception): +class TklAmqError(Exception): pass + class Connection: def __init__(self, hostname, port, vhost, userid, password): """connects to broker and provides convenience methods""" self.broker = BrokerConnection(hostname=hostname, port=port, userid=userid, password=password, virtual_host=vhost) - + def __del__(self): self.broker.close() def declare(self, exchange, exchange_type, binding="", queue=""): """declares the exchange, the queue and binds the queue to the exchange - + exchange - exchange name exchange_type - direct, topic, fanout binding - binding to queue (optional) @@ -43,7 +45,8 @@ def declare(self, exchange, exchange_type, binding="", queue=""): """ if (binding and not queue) or (queue and not binding): if queue and not exchange_type == "fanout": - raise Error("binding and queue are not mutually exclusive") + raise TklAmqError( + "binding and queue are not mutually exclusive") consumer = Consumer(connection=self.broker, exchange=exchange, exchange_type=exchange_type, @@ -53,9 +56,10 @@ def declare(self, exchange, exchange_type, binding="", queue=""): def consume(self, queue, limit=None, callback=None, auto_declare=False): """consume messages in queue - + queue - name of queue - limit - amount of messages to iterate through (default: no limit) + limit - amount of messages to iterate through + (default: no limit) callback - method to call when a new message is received must take two arguments: message_data, message @@ -79,12 +83,14 @@ def consume(self, queue, limit=None, callback=None, auto_declare=False): def publish(self, exchange, routing_key, message, auto_declare=False, persistent=True): """publish a message to exchange using routing_key - + exchange - name of exchange - routing_key - interpretation of routing key depends on exchange type + routing_key - interpretation of routing key depends on exchange + type message - message content to send auto_declare - automatically declare the exchange (default: false) - persistent - store message on disk as well as memory (default: True) + persistent - store message on disk as well as memory + (default: True) """ delivery_mode = 2 if not persistent: @@ -97,11 +103,13 @@ def publish(self, exchange, routing_key, message, publisher.send(message, delivery_mode=delivery_mode) publisher.close() + def _consume_callback(message_data, message): """default consume callback if not specified""" - print message_data + print(message_data) message.ack() + def connect(): """convenience method using environment variables""" BROKER_HOST = os.getenv('BROKER_HOST', 'localhost') @@ -143,6 +151,7 @@ def encode_message(sender, content, secret=None): return message + def decode_message(message_data, secret=None): """decode message envelope args @@ -156,11 +165,10 @@ def decode_message(message_data, secret=None): """ sender = str(message_data['sender']) content = base64.urlsafe_b64decode(str(message_data['content'])) - timestamp = datetime(*map(lambda f: int(f), message_data['timestamp-utc'])) + timestamp = datetime(*[int(f) for f in message_data['timestamp-utc']]) if message_data['encrypted']: content = decrypt(content, secret) content = json.loads(content) return sender, content, timestamp - diff --git a/tklamq/crypto.py b/tklamq_lib/crypto.py similarity index 83% rename from tklamq/crypto.py rename to tklamq_lib/crypto.py index bab6712..34aa090 100644 --- a/tklamq/crypto.py +++ b/tklamq_lib/crypto.py @@ -1,11 +1,14 @@ -# Copyright (c) 2010 Alon Swartz - all rights reserved +# Copyright (c) 2010-2021 Alon Swartz - all rights reserved +# Copyright (c) 2022 TurnKey GNU/Linux - all rights reserved -from Crypto.Cipher import AES +from Cryptodome.Cipher import AES from hashlib import sha1 -class CheckSumError(Exception): + +class TklAmqCheckSumError(Exception): pass + def _lazysecret(secret, blocksize=32, padding='}'): """pads secret if not legal AES block size (16, 24, 32)""" if not len(secret) in (16, 24, 32): @@ -13,6 +16,7 @@ def _lazysecret(secret, blocksize=32, padding='}'): return secret + def encrypt(plaintext, secret, lazy=True, checksum=True): """encrypt plaintext with secret plaintext - content to encrypt @@ -29,6 +33,7 @@ def encrypt(plaintext, secret, lazy=True, checksum=True): return encobj.encrypt(plaintext) + def decrypt(ciphertext, secret, lazy=True, checksum=True): """decrypt ciphertext with secret ciphertext - encrypted content to decrypt @@ -44,7 +49,6 @@ def decrypt(ciphertext, secret, lazy=True, checksum=True): if checksum: digest, plaintext = (plaintext[-20:], plaintext[:-20]) if not digest == sha1(plaintext).digest(): - raise CheckSumError("checksum mismatch") + raise TklAmqCheckSumError("checksum mismatch") return plaintext -