From d86cb2d5ae201cfcd04665c3918fb1e123fa08c5 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 7 Dec 2018 09:44:33 +0000 Subject: [PATCH 01/79] Add new coveralls dependency constraint --- constraints.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/constraints.txt b/constraints.txt index 6c3eab64..74809e04 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,4 +1,5 @@ # Constraints that apply to pip installed requirements -# coveralls dependency that needs an older version for Python 2.6 +# coveralls dependencies that need older versions for Python 2.6 pycparser<2.19 +idna<2.8 From f6199e0939bfacd72d152502ebc1232019aacc4e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 7 Dec 2018 09:29:36 +0000 Subject: [PATCH 02/79] Update .travis.yml Remove sudo: flase as Travis is deprecating and moving everyone to VM-based infrastructure: https://blog.travis-ci.com/2018-11-19-required-linux-infrastructure-migration --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index db6608ce..8633d739 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,6 @@ matrix: - python: "nightly" fast_finish: true -# Route build to container-based infrastructure -sudo: false - # Cache the dependencies installed by pip cache: pip # Avoid pip log from affecting cache From 7a1e2a90d41adcb203639214f169d07845108b9d Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 7 Dec 2018 17:07:17 +0000 Subject: [PATCH 03/79] Update requirements-test.txt Restrict coveralls to version that supports Python 2.6 --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index c9eb8ea3..f9d5b6b7 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,6 +4,6 @@ -c constraints.txt unittest2 -coveralls +coveralls<=1.2.0 mock codecov From 5a501ff87d72b0fbd04d2b3da8d722993ec8a522 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 8 May 2018 14:27:57 +0100 Subject: [PATCH 04/79] Add new option to say if a dirq is being used - defaults to dirq format for backwards compatibility --- bin/sender.py | 11 ++++++++++- conf/sender.cfg | 4 ++++ ssm/ssm2.py | 12 ++++++++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/sender.py b/bin/sender.py index 2f1313b8..addb9a28 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -120,9 +120,18 @@ def main(): raise Ssm2Exception('No destination queue is configured.') except ConfigParser.NoOptionError, e: raise Ssm2Exception(e) - + + # Determine what type of message store we are interacting with, + # i.e. a dirq QueueSimple object or a directory + try: + path_type = cp.get('messaging', 'path_type') + except ConfigParser.NoOptionError: + log.info('No path type defined, assuming dirq.') + path_type = 'dirq' + sender = Ssm2(brokers, cp.get('messaging','path'), + path_type=path_type, cert=cp.get('certificates','certificate'), key=cp.get('certificates','key'), dest=cp.get('messaging','destination'), diff --git a/conf/sender.cfg b/conf/sender.cfg index 73c0511f..813a3fb2 100644 --- a/conf/sender.cfg +++ b/conf/sender.cfg @@ -40,6 +40,10 @@ destination: # Outgoing messages will be read and removed from this directory. path: /var/spool/apel/outgoing +# If 'path_type: directory' is supplied, the supplied 'path' will be +# treated as if it is a directory rather than a dirq. As a result, +# 'path' cannot contain subdirectories. +#path_type: directory [logging] logfile: /var/log/apel/ssmsend.log diff --git a/ssm/ssm2.py b/ssm/ssm2.py index e451948a..76a5d04b 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -57,7 +57,8 @@ class Ssm2(stomp.ConnectionListener): def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, - enc_cert=None, verify_enc_cert=True, pidfile=None): + enc_cert=None, verify_enc_cert=True, pidfile=None, + path_type='dirq'): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. @@ -86,7 +87,14 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: - self._outq = QueueSimple(qpath) + # Determine what sort of outgoing structure to make + if path_type == 'dirq': + self._outq = QueueSimple(qpath) + elif path_type == 'directory': + self._outq = None + else: + raise Ssm2Exception('Unsupported path_type variable.') + elif listen is not None: inqpath = os.path.join(qpath, 'incoming') rejectqpath = os.path.join(qpath, 'reject') From cba8c3a719095ba088ab67af29f6164ac59baf4a Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 8 May 2018 14:47:43 +0100 Subject: [PATCH 05/79] Replicate the dirq interface for the methods used --- ssm/message_directory.py | 59 ++++++++++++++++++++++++++++++++++++++++ ssm/ssm2.py | 3 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 ssm/message_directory.py diff --git a/ssm/message_directory.py b/ssm/message_directory.py new file mode 100644 index 00000000..adfd9442 --- /dev/null +++ b/ssm/message_directory.py @@ -0,0 +1,59 @@ +""" +Copyright (C) 2018 STFC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +@author: Greg Corbett +""" + +class MessageDirectory(object): + """A structure for holding Accounting messages in a directory.""" + + def __init__(self, path): + """Create a new directory structure for holding Accounting messages.""" + raise NotImplementedError() + + def add(self, data): + """Add a new element to the queue and return its name.""" + raise NotImplementedError() + + def count(self): + """ + Return the number of elements in the queue. + + Regardless of their state. + """ + raise NotImplementedError() + + def get(self, name): + """Get an element data from a locked element.""" + raise NotImplementedError() + + def lock(self, name): + """Lock an element.""" + raise NotImplementedError() + + def purge(self): + """ + Purge the queue. + + - delete unused intermediate directories + - delete too old temporary directories + - unlock too old locked directories + + """ + raise NotImplementedError() + + def remove(self, name): + """Remove locked element from the queue.""" + raise NotImplementedError() diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 76a5d04b..6f751580 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -25,6 +25,7 @@ ssl = None from ssm import crypto +from ssm.message_directory import MessageDirectory from dirq.QueueSimple import QueueSimple from dirq.queue import Queue @@ -91,7 +92,7 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, if path_type == 'dirq': self._outq = QueueSimple(qpath) elif path_type == 'directory': - self._outq = None + self._outq = MessageDirectory(qpath) else: raise Ssm2Exception('Unsupported path_type variable.') From 912f4b23f073c26c2982427fc20cdc2fd23199d0 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:00:02 +0100 Subject: [PATCH 06/79] Implement the __init__ method - it just stores the path to the messages for later use --- ssm/message_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index adfd9442..4fbcafcc 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -21,7 +21,7 @@ class MessageDirectory(object): def __init__(self, path): """Create a new directory structure for holding Accounting messages.""" - raise NotImplementedError() + self.directory_path = path def add(self, data): """Add a new element to the queue and return its name.""" From 9e5543ad5502a0a07bd92bc9103eea949e1ed97a Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:00:24 +0100 Subject: [PATCH 07/79] Add a helper method to get the messages - a list of messages/files is not stored in the structure because it would be difficult to keep the directory and the structure in sync, especially when deleting files from the underlying directory, because you cant iterate over a stored list of files and delete elements in that list in a sensible/easy way. - mtime is used because (apparently) there is not way to find the original date of file creation due to a limitation of the underlying filesystem --- ssm/message_directory.py | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 4fbcafcc..e056ae86 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -16,6 +16,9 @@ @author: Greg Corbett """ +import os + + class MessageDirectory(object): """A structure for holding Accounting messages in a directory.""" @@ -57,3 +60,43 @@ def purge(self): def remove(self, name): """Remove locked element from the queue.""" raise NotImplementedError() + + def _get_messages(self, sort_by_mtime=False): + """ + Get the messages stored in this MessageDirectory. + + if sort_by_mtime is set to True, the returned list is guaranteed to be + in increasing order of modification time. + + mtime is used because (apparently) there is not way to find the + original date of file creation due to a limitation + of the underlying filesystem + """ + # Get a list of files under self.directory_path in an arbitrary order. + file_name_list = os.listdir(self.directory_path) + + if sort_by_mtime: + # Working space to hold the unsorted messages + # as file paths and mtimes. + unsorted_messages = [] + # Working space to hold the sorted messages as file names. + sorted_messages = [] + + # Work out the mtime of each file. + for file_name in file_name_list: + file_path = os.path.join(self.directory_path, file_name) + # Store the file path and the time the file was last modified. + unsorted_messages.append((file_name, + os.path.getmtime(file_path))) + + # Sort the file paths by mtime and then only store the file name. + for (file_name, _mtime) in sorted(unsorted_messages, + key=lambda tup: tup[1]): + # Store the sorted file paths in a class element. + sorted_messages.append(file_name) + + # Return the sorted list. + return sorted_messages + + # If we get here, just return the arbitrarily ordered list. + return file_name_list From 2fda8719e5a519ed1e6b2a16d99460fc7c8e458f Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:02:57 +0100 Subject: [PATCH 08/79] Implement count() using the helper method --- ssm/message_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index e056ae86..d7d8f898 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -36,7 +36,7 @@ def count(self): Regardless of their state. """ - raise NotImplementedError() + return len(self._get_messages()) def get(self, name): """Get an element data from a locked element.""" From b9be53495efdaad2576d88de6bbfdf459aa79938 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:11:13 +0100 Subject: [PATCH 09/79] Make this class iterable --- ssm/message_directory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index d7d8f898..db9f4f8a 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -100,3 +100,7 @@ def _get_messages(self, sort_by_mtime=False): # If we get here, just return the arbitrarily ordered list. return file_name_list + + def __iter__(self): + """Return an iterable of the files currently in the MessageDirectory.""" + return self._get_messages(sort_by_mtime=True).__iter__() From d47afe80c0e518631675435f560991b01380ebdb Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:15:12 +0100 Subject: [PATCH 10/79] Implement lock() to preserve dirq interface --- ssm/message_directory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index db9f4f8a..374d1b10 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -42,9 +42,9 @@ def get(self, name): """Get an element data from a locked element.""" raise NotImplementedError() - def lock(self, name): - """Lock an element.""" - raise NotImplementedError() + def lock(self, _name): + """Return True to simulate a successful lock. Does nothing else.""" + return True def purge(self): """ From ffd2fff93c2d13f33355e1364079fe8218a665ab Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:18:39 +0100 Subject: [PATCH 11/79] Implement get() method --- ssm/message_directory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 374d1b10..597f2f48 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -39,8 +39,10 @@ def count(self): return len(self._get_messages()) def get(self, name): - """Get an element data from a locked element.""" - raise NotImplementedError() + """Return the content of the named message.""" + with open("%s/%s" % (self.directory_path, name)) as message: + content = message.read() + return content def lock(self, _name): """Return True to simulate a successful lock. Does nothing else.""" From 52d368c0728dc1369ff4733f81abda2a39e92159 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:27:35 +0100 Subject: [PATCH 12/79] Implement remove() --- ssm/message_directory.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 597f2f48..b03b32ec 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -16,8 +16,12 @@ @author: Greg Corbett """ +import logging import os +# logging configuration +LOG = logging.getLogger(__name__) + class MessageDirectory(object): """A structure for holding Accounting messages in a directory.""" @@ -60,8 +64,11 @@ def purge(self): raise NotImplementedError() def remove(self, name): - """Remove locked element from the queue.""" - raise NotImplementedError() + """Remove the named message.""" + try: + os.unlink("%s/%s" % (self.directory_path, name)) + except OSError: + LOG.warning("Could not remove %s, it may get resent.", name) def _get_messages(self, sort_by_mtime=False): """ From 226458fc7905b4aa77d25684b066279df7e2faeb Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:38:25 +0100 Subject: [PATCH 13/79] Implement add() - this isn't strictly needed for replacing the dirq module as it's not called in ssm/ssm2.py - however, I find it useful to be able to programmatically create messages in the outq when testing. - and it might be useful for collector developers - return a string (not a UUID object) because thats what the dirq module does --- ssm/message_directory.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index b03b32ec..828d742d 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -18,6 +18,7 @@ import logging import os +import uuid # logging configuration LOG = logging.getLogger(__name__) @@ -31,8 +32,22 @@ def __init__(self, path): self.directory_path = path def add(self, data): - """Add a new element to the queue and return its name.""" - raise NotImplementedError() + """Add the passed data to a new file and return it's name.""" + # Use uuid4 to create a file name because uuid4 is + # sufficient to create an unique ID. + name = uuid.uuid4() + + try: + # Open the file and write the provided data into the file. + with open("%s/%s" % (self.directory_path, name), 'w') as message: + message.write(data) + except OSError: + LOG.warning("Could not create file %s/%s", + self.directory_path, name) + + # Return the name of the created file as a string, + # to keep the dirq like interface. + return "%s" % name def count(self): """ From 875b6750e99f6a5d5792290a47f63268ed9bbd5d Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Thu, 10 May 2018 14:50:59 +0100 Subject: [PATCH 14/79] Implement purge() - although it does nothing as MessageDirectory is a flat structure so there will be no: - unused intermediate directories - old temporary directories - old locked directories --- ssm/message_directory.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 828d742d..a44281ac 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -69,14 +69,11 @@ def lock(self, _name): def purge(self): """ - Purge the queue. - - - delete unused intermediate directories - - delete too old temporary directories - - unlock too old locked directories + Do nothing, as there are no old/intermediate directories to purge. + Only included to preserve dirq interface. """ - raise NotImplementedError() + LOG.debug("purge() called, but purge() does nothing.") def remove(self, name): """Remove the named message.""" From d3e90e4dca1cce2f27ccf9c24c1bb944f9d708b9 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 18 May 2018 14:48:54 +0100 Subject: [PATCH 15/79] Add a unit test file for MessageDirectory --- test/test_message_directory.py | 100 +++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test/test_message_directory.py diff --git a/test/test_message_directory.py b/test/test_message_directory.py new file mode 100644 index 00000000..947568dd --- /dev/null +++ b/test/test_message_directory.py @@ -0,0 +1,100 @@ +""" +Created on 18 May 2018. + +@author: Greg Corbett +""" + +import shutil +import tempfile +import unittest + +from ssm.message_directory import MessageDirectory + + +class TestMessageDirectory(unittest.TestCase): + """Class used for testing the MessageDirectory class.""" + + def setUp(self): + """Create a MessageDirectory class on top of a temporary directory.""" + self.tmp_dir = tempfile.mkdtemp(prefix='message_directory') + self.message_directory = MessageDirectory(self.tmp_dir) + # Assert no files exist in the underlying file system. + self.assertEqual(self.message_directory.count(), 0) + + def test_add_and_get(self): + """ + Test the add and get methods of the MessageDirectory class. + + This test adds a file to a MessageDirectory, checks it has been + written to the underlying directory and then checks the saved file + for content equality. + """ + test_content = "FOO" + # Add the test content to the MessageDirectory. + file_name = self.message_directory.add(test_content) + + # Assert there is exactly on message in the directory. + self.assertEqual(self.message_directory.count(), 1) + + # Fetch the saved content using the get method. + saved_content = self.message_directory.get(file_name) + + # Assert the saved content is equal to the original test content. + self.assertEqual(saved_content, test_content) + + def test_count(self): + """ + Test the count method of the MessageDirectory class. + + This test adds two files to a MessageDirectory and then checks + the output of the count() function is as expected. + """ + # Add some files to the MessageDirectory. + self.message_directory.add("FOO") + self.message_directory.add("BAR") + + # Check the count method returns the correct value. + self.assertEqual(self.message_directory.count(), 2) + + def test_lock(self): + """ + Test the lock method of the MessageDirectory class. + + This test checks the lock method returns true for any file. + """ + self.assertTrue(self.message_directory.lock("any file")) + + def test_purge(self): + """ + Test the purge method of the MessageDirectory class. + + This test only checks the purge method is callable without error, + as the purge method only logs that it has been called. + """ + self.message_directory.purge() + + def test_remove(self): + """ + Test the remove method of the MessageDirectory class. + + This test adds a file, removes the file and then checks + the number of files present. + """ + # Add some files to the MessageDirectory. + file_name = self.message_directory.add("FOO") + # Use the remove method to delete the recently added file. + self.message_directory.remove(file_name) + # Check the count method returns the expected value. + self.assertEqual(self.message_directory.count(), 0) + + def tearDown(self): + """Remove test directory and all contents.""" + try: + shutil.rmtree(self.tmp_dir) + except OSError, error: + print 'Error removing temporary directory %s' % self.tmp_dir + print error + + +if __name__ == "__main__": + unittest.main() From 985acf4a5b78f7e0e5f3041fb4adf625bb0f328d Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 18 May 2018 15:03:44 +0100 Subject: [PATCH 16/79] Test support for non-dirq sending can be used --- test/test_ssm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_ssm.py b/test/test_ssm.py index 1dcbbaca..79d1c2a3 100644 --- a/test/test_ssm.py +++ b/test/test_ssm.py @@ -10,6 +10,7 @@ import unittest from subprocess import call +from ssm.message_directory import MessageDirectory from ssm.ssm2 import Ssm2, Ssm2Exception @@ -120,6 +121,18 @@ def test_init_expired_server_cert(self): # verify_enc_cert is set to False as we don't want to risk raising an # exception by failing cert verification. + def test_ssm_init_non_dirq(self): + """Test a SSM can be initialised with support for non-dirq sending.""" + try: + ssm = Ssm2(self._brokers, self._msgdir, TEST_CERT_FILE, + self._key_path, dest=self._dest, listen=None, + path_type='directory') + except Ssm2Exception as error: + self.fail('An error occured trying to create an SSM using ' + 'the non-dirq functionality: %s.' % error) + + # Assert the outbound queue is of the expected type. + self.assertTrue(isinstance(ssm._outq, MessageDirectory)) TEST_CERT_FILE = '/tmp/test.crt' From d4f93b085fcfe013efafb1788ae7ff516d5f7acc Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 18 May 2018 15:47:03 +0100 Subject: [PATCH 17/79] Improve error handling around permissions --- ssm/message_directory.py | 75 +++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index a44281ac..93b96d36 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -41,9 +41,9 @@ def add(self, data): # Open the file and write the provided data into the file. with open("%s/%s" % (self.directory_path, name), 'w') as message: message.write(data) - except OSError: - LOG.warning("Could not create file %s/%s", - self.directory_path, name) + except (IOError, OSError) as error: + LOG.error("Could not create file %s/%s: %s", + self.directory_path, name, error) # Return the name of the created file as a string, # to keep the dirq like interface. @@ -79,8 +79,9 @@ def remove(self, name): """Remove the named message.""" try: os.unlink("%s/%s" % (self.directory_path, name)) - except OSError: + except (IOError, OSError) as error: LOG.warning("Could not remove %s, it may get resent.", name) + LOG.debug(error) def _get_messages(self, sort_by_mtime=False): """ @@ -93,34 +94,44 @@ def _get_messages(self, sort_by_mtime=False): original date of file creation due to a limitation of the underlying filesystem """ - # Get a list of files under self.directory_path in an arbitrary order. - file_name_list = os.listdir(self.directory_path) - - if sort_by_mtime: - # Working space to hold the unsorted messages - # as file paths and mtimes. - unsorted_messages = [] - # Working space to hold the sorted messages as file names. - sorted_messages = [] - - # Work out the mtime of each file. - for file_name in file_name_list: - file_path = os.path.join(self.directory_path, file_name) - # Store the file path and the time the file was last modified. - unsorted_messages.append((file_name, - os.path.getmtime(file_path))) - - # Sort the file paths by mtime and then only store the file name. - for (file_name, _mtime) in sorted(unsorted_messages, - key=lambda tup: tup[1]): - # Store the sorted file paths in a class element. - sorted_messages.append(file_name) - - # Return the sorted list. - return sorted_messages - - # If we get here, just return the arbitrarily ordered list. - return file_name_list + try: + # Get a list of files under self.directory_path + # in an arbitrary order. + file_name_list = os.listdir(self.directory_path) + + if sort_by_mtime: + # Working space to hold the unsorted messages + # as file paths and mtimes. + unsorted_messages = [] + # Working space to hold the sorted messages as file names. + sorted_messages = [] + + # Work out the mtime of each file. + for file_name in file_name_list: + file_path = os.path.join(self.directory_path, file_name) + # Store the file path and the time + # the file was last modified. + unsorted_messages.append((file_name, + os.path.getmtime(file_path))) + + # Sort the file paths by mtime and + # then only store the file name. + for (file_name, _mtime) in sorted(unsorted_messages, + key=lambda tup: tup[1]): + # Store the sorted file paths in a class element. + sorted_messages.append(file_name) + + # Return the sorted list. + return sorted_messages + + # If we get here, just return the arbitrarily ordered list. + return file_name_list + + except (IOError, OSError) as error: + LOG.error(error) + # Return an empty file list. + return [] + def __iter__(self): """Return an iterable of the files currently in the MessageDirectory.""" From 488d118ed86aa3070b23cdf8da2f55660134d497 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 22 May 2018 14:28:24 +0100 Subject: [PATCH 18/79] Test files are retrieved in the correct order --- test/test_message_directory.py | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/test_message_directory.py b/test/test_message_directory.py index 947568dd..412d2b3a 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -42,6 +42,44 @@ def test_add_and_get(self): # Assert the saved content is equal to the original test content. self.assertEqual(saved_content, test_content) + def test_orderd_file_retrieval(self): + """ + Test the messages are retrieved in the order they were last modified. + + This test adds files to the MessageDirectory and then iterates over + the MessageDirectory to retrieve the file names. If the for loop does + not return them in the order they were last modfied, this test fails. + + + """ + # In the event of a failure of underlying _get_messages sorting, it's + # possible the returned list of files could still be in the correct + # order by random chance. + # A 'large' list of test_content reduces this chance. + test_content_list = ["Lobster Thermidor", "Crevettes", "Mornay sauce", + "Truffle Pate", "Brandy", "Fried egg", "Spam"] + + # A list to hold file names by creation time. + file_names_by_creation_time = [] + + for test_content in test_content_list: + # Add the content to the MessageDirectory. + file_name = self.message_directory.add(test_content) + # Append the file name to the list of file names by create time. + file_names_by_creation_time.append(file_name) + + # A list to hold file names by modification time. + file_names_by_modification_time = [] + # Use a for loop (similar to how the SSM retrieves messages) + # to build up an ordered list of files. + for file_name in self.message_directory: + file_names_by_modification_time.append(file_name) + + # As the files are not modified once added to the MessageDirectory, + # the two lists of file names should be equal. + self.assertEqual(file_names_by_modification_time, + file_names_by_creation_time) + def test_count(self): """ Test the count method of the MessageDirectory class. From 24141037da880fab2cfaba660a23f2a1899ad614 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 30 May 2018 14:49:02 +0100 Subject: [PATCH 19/79] Loosen requirement for the python dirq module. - the spec file does not list is as a requirement, as only hard dependencies can and should be set as RedHat RPM dependencies. - requirements.txt now notes is as optional. - setup.py allows it to be installed as an 'extra' - imports and use of dirq in ssm2.py now have error handing around them to prevent exceptions rising. --- apel-ssm.spec | 2 +- requirements.txt | 1 + setup.py | 3 ++- ssm/ssm2.py | 22 ++++++++++++++++++++-- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index 7fc3e1c9..d784bba3 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -21,7 +21,7 @@ BuildArch: noarch BuildRequires: python-devel %endif -Requires: stomppy >= 3.1.1, python-daemon < 2.2.0, python-dirq, python-ldap +Requires: stomppy >= 3.1.1, python-daemon < 2.2.0, python-ldap Requires(pre): shadow-utils %define ssmconf %_sysconfdir/apel diff --git a/requirements.txt b/requirements.txt index a5977a79..d3838cdc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ stomp.py>=3.1.1 python-daemon<2.2.0 python-ldap +# Dependencies for optional dirq based sending dirq diff --git a/setup.py b/setup.py index 69c42aff..9302fc12 100644 --- a/setup.py +++ b/setup.py @@ -50,9 +50,10 @@ def main(): url='http://apel.github.io/', download_url='https://github.com/apel/ssm/releases', license='Apache License, Version 2.0', - install_requires=['stomp.py>=3.1.1', 'python-ldap', 'dirq'], + install_requires=['stomp.py>=3.1.1', 'python-ldap'], extras_require={ 'python-daemon': ['python-daemon<2.2.0'], + 'dirq': ['dirq'] }, packages=find_packages(exclude=['bin', 'test']), scripts=['bin/ssmreceive', 'bin/ssmsend'], diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 6f751580..c891f546 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -26,8 +26,14 @@ from ssm import crypto from ssm.message_directory import MessageDirectory -from dirq.QueueSimple import QueueSimple -from dirq.queue import Queue + +try: + from dirq.QueueSimple import QueueSimple + from dirq.queue import Queue +except ImportError: + # ImportError is raised later on if dirq is requested but not installed. + QueueSimple = None + Queue = None import stomp from stomp.exception import ConnectFailedException @@ -90,7 +96,12 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, if dest is not None and listen is None: # Determine what sort of outgoing structure to make if path_type == 'dirq': + if QueueSimple is None: + raise ImportError("dirq path_type requested but the dirq " + "module wasn't found.") + self._outq = QueueSimple(qpath) + elif path_type == 'directory': self._outq = MessageDirectory(qpath) else: @@ -99,6 +110,13 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, elif listen is not None: inqpath = os.path.join(qpath, 'incoming') rejectqpath = os.path.join(qpath, 'reject') + + # Receivers must use the dirq module, so make a quick sanity check + # that dirq is installed. + if Queue is None: + raise ImportError("Receiving SSMs must use dirq, but the dirq " + "module wasn't found.") + self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA) self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA) else: From bec79aa4c22bf131db906b442fdba53679c03e1d Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 15 Jun 2018 10:28:35 +0100 Subject: [PATCH 20/79] Update documentation relating to the dirq module --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 30b7dcdf..3109efb1 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ The python daemon library (N.B. only versions below 2.2.0 are currently supporte The python ldap library * `yum install python-ldap` -The python dirq library +Optionally, the python dirq library (N.B. this is only required if your messages +are stored in a dirq structure) * `yum install python-dirq` You need a certificate and key in PEM format accessible to the SSM. @@ -119,10 +120,11 @@ configuration will send messages to the test apel server. ## Adding Files -There are two ways to add files to be sent: +There are multiple manual and programmatic ways to add files to be sent: ### Manual +#### With the dirq module All file and directory names must use hex characters: `[0-9a-f]`. * Create a directory within /var/spool/apel/outgoing with a name @@ -130,8 +132,14 @@ All file and directory names must use hex characters: `[0-9a-f]`. * Put files in this directory with names of FOURTEEN hex e.g. `1234567890abcd` +#### Without the dirq module +Ensure `path_type: directory` is set in your `sender.cfg`. +Then add messages as files to `/var/spool/apel/outgoing`, +there are no restrictions on the file names used. + ### Programmatic +#### With the dirq module Use the python or perl dirq libraries: * python: http://pypi.python.org/pypi/dirq * perl: http://search.cpan.org/~lcons/Directory-Queue/ @@ -139,6 +147,11 @@ Use the python or perl dirq libraries: Create a QueueSimple object with path /var/spool/apel/outgoing/ and add your messages. +#### Without the dirq module +Use the `MessageDirectory` class provided in `ssm.message_directory`. + +Create a `MessageDirectory` object with path `/var/spool/apel/outgoing/` and +add your messages using the `add` method. ## Running the SSM From 999ffa7cc00d1d37562a1abcc44bfd83380cb5cd Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 11 Sep 2018 14:14:53 +0100 Subject: [PATCH 21/79] uncomment path_type, set to dirq and clarify use - uncomment as existing users won't get their config overwritten. - set to dirq so the current behavior is the ongoing default behavior - clarify use so that both options are described --- conf/sender.cfg | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/conf/sender.cfg b/conf/sender.cfg index 813a3fb2..e4372ab0 100644 --- a/conf/sender.cfg +++ b/conf/sender.cfg @@ -40,10 +40,13 @@ destination: # Outgoing messages will be read and removed from this directory. path: /var/spool/apel/outgoing -# If 'path_type: directory' is supplied, the supplied 'path' will be -# treated as if it is a directory rather than a dirq. As a result, -# 'path' cannot contain subdirectories. -#path_type: directory +# If 'path_type' is set to 'dirq' (or if 'path_type' is omitted), the supplied +# 'path' will be treated as a Python dirq (a directory based queue, which is a +# port of the Perl module Directory::Queue). +# If 'path_type' is set to 'directory', the supplied 'path' will be treated +# as if it is a directory rather than a dirq. +# As a result, 'path' cannot contain subdirectories. +path_type: dirq [logging] logfile: /var/log/apel/ssmsend.log From d9bee4802c8259193c5d23e597e9f26774c20ac6 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 11 Sep 2018 14:25:27 +0100 Subject: [PATCH 22/79] Clarify need for unique file names --- ssm/message_directory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 93b96d36..c5a01965 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -33,8 +33,9 @@ def __init__(self, path): def add(self, data): """Add the passed data to a new file and return it's name.""" - # Use uuid4 to create a file name because uuid4 is - # sufficient to create an unique ID. + # Create a unique file name so APEL admins can pair sent and recieved + # messages easily (as the file name appears in the sender and receiver + # logs as the message ID). name = uuid.uuid4() try: From 7a71517d347b3a716345d4112a2bc757c3903796 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 11 Sep 2018 14:28:02 +0100 Subject: [PATCH 23/79] De-capitalise LOG as it's not a constant --- ssm/message_directory.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index c5a01965..eb5a9967 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -21,7 +21,7 @@ import uuid # logging configuration -LOG = logging.getLogger(__name__) +log = logging.getLogger(__name__) class MessageDirectory(object): @@ -43,7 +43,7 @@ def add(self, data): with open("%s/%s" % (self.directory_path, name), 'w') as message: message.write(data) except (IOError, OSError) as error: - LOG.error("Could not create file %s/%s: %s", + log.error("Could not create file %s/%s: %s", self.directory_path, name, error) # Return the name of the created file as a string, @@ -74,15 +74,15 @@ def purge(self): Only included to preserve dirq interface. """ - LOG.debug("purge() called, but purge() does nothing.") + log.debug("purge() called, but purge() does nothing.") def remove(self, name): """Remove the named message.""" try: os.unlink("%s/%s" % (self.directory_path, name)) except (IOError, OSError) as error: - LOG.warning("Could not remove %s, it may get resent.", name) - LOG.debug(error) + log.warning("Could not remove %s, it may get resent.", name) + log.debug(error) def _get_messages(self, sort_by_mtime=False): """ @@ -129,7 +129,7 @@ def _get_messages(self, sort_by_mtime=False): return file_name_list except (IOError, OSError) as error: - LOG.error(error) + log.error(error) # Return an empty file list. return [] From 850c54b8037185c336bdd1f55d62d2144662e8e5 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 11 Sep 2018 14:30:54 +0100 Subject: [PATCH 24/79] Clarify log line applies only to non dirq sending - brackets removed to limit line length --- ssm/message_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index eb5a9967..d27e9eae 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -74,7 +74,7 @@ def purge(self): Only included to preserve dirq interface. """ - log.debug("purge() called, but purge() does nothing.") + log.debug("purge called, but purge does nothing for non-dirq sending.") def remove(self, name): """Remove the named message.""" From d5451cfbdf6b1a6a947639cb17377b3818a0c687 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 11 Sep 2018 15:33:19 +0100 Subject: [PATCH 25/79] Make some python <2.7 code python 3 compliant --- test/test_message_directory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_message_directory.py b/test/test_message_directory.py index 412d2b3a..62786ff4 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -129,9 +129,9 @@ def tearDown(self): """Remove test directory and all contents.""" try: shutil.rmtree(self.tmp_dir) - except OSError, error: - print 'Error removing temporary directory %s' % self.tmp_dir - print error + except OSError as error: + print('Error removing temporary directory %s' % self.tmp_dir) + print(error) if __name__ == "__main__": From a35a55adb3e9fb3d74afd2b14bfddf0edefc3675 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Tue, 11 Sep 2018 15:35:42 +0100 Subject: [PATCH 26/79] Fix some whitespace/long line errors --- ssm/message_directory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index d27e9eae..28bacfca 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -133,7 +133,6 @@ def _get_messages(self, sort_by_mtime=False): # Return an empty file list. return [] - def __iter__(self): - """Return an iterable of the files currently in the MessageDirectory.""" + """Return an iterable of files currently in the MessageDirectory.""" return self._get_messages(sort_by_mtime=True).__iter__() From 040c5d7f805abf25e5b72aa1b53093f843717f8a Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Mon, 15 Oct 2018 11:40:34 +0100 Subject: [PATCH 27/79] Move license info to comments, remove authorship --- ssm/message_directory.py | 31 ++++++++++++++----------------- test/test_message_directory.py | 19 ++++++++++++++----- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 28bacfca..44c45f94 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -1,20 +1,17 @@ -""" -Copyright (C) 2018 STFC. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: Greg Corbett -""" +# Copyright 2018 Science and Technology Facilities Council +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This module contains the MessageDirectory class.""" import logging import os diff --git a/test/test_message_directory.py b/test/test_message_directory.py index 62786ff4..c4d882a9 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -1,8 +1,17 @@ -""" -Created on 18 May 2018. - -@author: Greg Corbett -""" +# Copyright 2018 Science and Technology Facilities Council +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This module contains test cases for the MessageDirectory class.""" import shutil import tempfile From d5d82d61bbf5300a3cf1aa5145df8b7a85987331 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Mon, 15 Oct 2018 11:44:11 +0100 Subject: [PATCH 28/79] Let unlink exception raise up - as that is what the dirq module would do --- ssm/message_directory.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 44c45f94..3490e575 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -75,11 +75,8 @@ def purge(self): def remove(self, name): """Remove the named message.""" - try: - os.unlink("%s/%s" % (self.directory_path, name)) - except (IOError, OSError) as error: - log.warning("Could not remove %s, it may get resent.", name) - log.debug(error) + os.unlink("%s/%s" % (self.directory_path, name)) + def _get_messages(self, sort_by_mtime=False): """ From 914f2a7b31734583352d5b5399f8041fb5e3441e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 7 Dec 2018 15:53:59 +0000 Subject: [PATCH 29/79] Remove Error catching in add method - Remove catching of IOError and OSError in add method of MessageDirectory to make it more similar to the dirq module. - Add trailing comma to dictionary in setup.py to avoid future changes invoking change to that line. --- setup.py | 2 +- ssm/message_directory.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 9302fc12..9d5ea532 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def main(): install_requires=['stomp.py>=3.1.1', 'python-ldap'], extras_require={ 'python-daemon': ['python-daemon<2.2.0'], - 'dirq': ['dirq'] + 'dirq': ['dirq'], }, packages=find_packages(exclude=['bin', 'test']), scripts=['bin/ssmreceive', 'bin/ssmsend'], diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 3490e575..927c9f94 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -35,13 +35,9 @@ def add(self, data): # logs as the message ID). name = uuid.uuid4() - try: - # Open the file and write the provided data into the file. - with open("%s/%s" % (self.directory_path, name), 'w') as message: - message.write(data) - except (IOError, OSError) as error: - log.error("Could not create file %s/%s: %s", - self.directory_path, name, error) + # Open the file and write the provided data into the file. + with open("%s/%s" % (self.directory_path, name), 'w') as message: + message.write(data) # Return the name of the created file as a string, # to keep the dirq like interface. From 4c3ebe9473b91c8194666a4bc02ac0b89fb3d2da Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 7 Dec 2018 16:38:44 +0000 Subject: [PATCH 30/79] Improvements to tests and small tweaks - Remove assertion check from test setUp and add more count checks to other tests instead. - Add small wait when creating messages in ordering test so that their modification times can be distinguised. - Tweaks to comments and whitespace. --- bin/sender.py | 2 +- ssm/message_directory.py | 1 - test/test_message_directory.py | 18 ++++++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/bin/sender.py b/bin/sender.py index addb9a28..26d79a71 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -122,7 +122,7 @@ def main(): raise Ssm2Exception(e) # Determine what type of message store we are interacting with, - # i.e. a dirq QueueSimple object or a directory + # i.e. a dirq QueueSimple object or a plain MessageDirectory directory. try: path_type = cp.get('messaging', 'path_type') except ConfigParser.NoOptionError: diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 927c9f94..cf1db8a1 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -73,7 +73,6 @@ def remove(self, name): """Remove the named message.""" os.unlink("%s/%s" % (self.directory_path, name)) - def _get_messages(self, sort_by_mtime=False): """ Get the messages stored in this MessageDirectory. diff --git a/test/test_message_directory.py b/test/test_message_directory.py index c4d882a9..1f61a0f2 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -15,6 +15,7 @@ import shutil import tempfile +import time import unittest from ssm.message_directory import MessageDirectory @@ -27,8 +28,6 @@ def setUp(self): """Create a MessageDirectory class on top of a temporary directory.""" self.tmp_dir = tempfile.mkdtemp(prefix='message_directory') self.message_directory = MessageDirectory(self.tmp_dir) - # Assert no files exist in the underlying file system. - self.assertEqual(self.message_directory.count(), 0) def test_add_and_get(self): """ @@ -58,8 +57,6 @@ def test_orderd_file_retrieval(self): This test adds files to the MessageDirectory and then iterates over the MessageDirectory to retrieve the file names. If the for loop does not return them in the order they were last modfied, this test fails. - - """ # In the event of a failure of underlying _get_messages sorting, it's # possible the returned list of files could still be in the correct @@ -70,12 +67,15 @@ def test_orderd_file_retrieval(self): # A list to hold file names by creation time. file_names_by_creation_time = [] - for test_content in test_content_list: # Add the content to the MessageDirectory. file_name = self.message_directory.add(test_content) # Append the file name to the list of file names by create time. file_names_by_creation_time.append(file_name) + # Wait a small amount of time to allow differentiation of times. + time.sleep(0.02) + + self.assertEqual(self.message_directory.count(), 7) # A list to hold file names by modification time. file_names_by_modification_time = [] @@ -96,11 +96,10 @@ def test_count(self): This test adds two files to a MessageDirectory and then checks the output of the count() function is as expected. """ - # Add some files to the MessageDirectory. + self.assertEqual(self.message_directory.count(), 0) self.message_directory.add("FOO") + self.assertEqual(self.message_directory.count(), 1) self.message_directory.add("BAR") - - # Check the count method returns the correct value. self.assertEqual(self.message_directory.count(), 2) def test_lock(self): @@ -127,8 +126,11 @@ def test_remove(self): This test adds a file, removes the file and then checks the number of files present. """ + # Check the directory starts empty + self.assertEqual(self.message_directory.count(), 0) # Add some files to the MessageDirectory. file_name = self.message_directory.add("FOO") + self.assertEqual(self.message_directory.count(), 1) # Use the remove method to delete the recently added file. self.message_directory.remove(file_name) # Check the count method returns the expected value. From 078c40a7c69a41af017d5e9f81c3473beb31397b Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 7 Jan 2019 13:28:06 +0000 Subject: [PATCH 31/79] Remove trailing whitepsace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3109efb1..7b59125e 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ configuration will send messages to the test apel server. ## Adding Files -There are multiple manual and programmatic ways to add files to be sent: +There are multiple manual and programmatic ways to add files to be sent: ### Manual From d8fc57a365e8e11ac35f7b7ba77521ee9b9e7617 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 4 Mar 2019 14:48:39 +0000 Subject: [PATCH 32/79] Fix PEP8 issues, unused code and copyright notice --- test/test_brokers.py | 79 ++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/test/test_brokers.py b/test/test_brokers.py index 6780242e..9f8fac52 100644 --- a/test/test_brokers.py +++ b/test/test_brokers.py @@ -1,38 +1,26 @@ -''' - Copyright (C) 2012 STFC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - @author: Will Rogers -''' +# Copyright 2019 UK Research and Innovation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from ssm import brokers import unittest -class Test(unittest.TestCase): - - - def setUp(self): - pass - - - def tearDown(self): - pass +class Test(unittest.TestCase): def test_parse_stomp_url(self): - + wrong_url = 'this is not a correct url' try: brokers.parse_stomp_url(wrong_url) @@ -40,30 +28,29 @@ def test_parse_stomp_url(self): except (IndexError, ValueError): # Expected exception pass - + http_url = 'http://not.a.stomp.url:8080' - + try: brokers.parse_stomp_url(http_url) self.fail('Parsed a URL which was not STOMP.') except ValueError: pass - + stomp_url = 'stomp://stomp.cern.ch:6262' - + try: brokers.parse_stomp_url(stomp_url) - except: - self.fail('Could not parse a valid stomp URL: %s' % stomp_url) - + except Exception: + self.fail('Could not parse a valid stomp URL: %s' % stomp_url) stomp_ssl_url = 'stomp+ssl://stomp.cern.ch:61262' - + try: brokers.parse_stomp_url(stomp_ssl_url) - except: - self.fail('Could not parse a valid stomp+ssl URL: %s' % stomp_url) - + except Exception: + self.fail('Could not parse a valid stomp+ssl URL: %s' % stomp_url) + def test_fetch_brokers(self): ''' Requires an internet connection to get information from the BDII. @@ -71,21 +58,21 @@ def test_fetch_brokers(self): ''' bdii = 'ldap://lcg-bdii.cern.ch:2170' network = 'PROD' - + sbg = brokers.StompBrokerGetter(bdii) - + bs = sbg.get_broker_hosts_and_ports(brokers.STOMP_SERVICE, network) - + if len(bs) < 1: self.fail('No brokers found in the BDII.') - + host, port = bs[0] if not str(port).isdigit(): self.fail('Got a non-integer port from fetch_brokers()') - - if not '.' in host: + + if '.' not in host: self.fail("Didn't get a hostname from fetch_brokers()") + if __name__ == '__main__': - #import sys;sys.argv = ['', 'Test.testName'] - unittest.main() \ No newline at end of file + unittest.main() From 59bc8b32b74b11e27a97094e8244927f69030924 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 4 Mar 2019 17:21:43 +0000 Subject: [PATCH 33/79] Mock out external LDAP search calls in tests --- test/test_brokers.py | 52 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/test/test_brokers.py b/test/test_brokers.py index 9f8fac52..1e1aa325 100644 --- a/test/test_brokers.py +++ b/test/test_brokers.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ssm import brokers - import unittest +import mock + +from ssm import brokers + class Test(unittest.TestCase): @@ -56,12 +58,15 @@ def test_fetch_brokers(self): Requires an internet connection to get information from the BDII. Could fail if the BDII is down. This isn't very unit-test-like. ''' - bdii = 'ldap://lcg-bdii.cern.ch:2170' + bdii = 'ldap://no-bdii.utopia.ch:2170' network = 'PROD' sbg = brokers.StompBrokerGetter(bdii) - bs = sbg.get_broker_hosts_and_ports(brokers.STOMP_SERVICE, network) + # So that there are no external LDAP calls, mock out the LDAP seach. + with mock.patch('ldap.ldapobject.SimpleLDAPObject.search_s', + side_effect=self._mocked_search): + bs = sbg.get_broker_hosts_and_ports(brokers.STOMP_SERVICE, network) if len(bs) < 1: self.fail('No brokers found in the BDII.') @@ -73,6 +78,45 @@ def test_fetch_brokers(self): if '.' not in host: self.fail("Didn't get a hostname from fetch_brokers()") + def _mocked_search(*args, **kwargs): + """Return values to mocked search call based on input.""" + + if ( + '(&(objectClass=GlueService)(GlueServiceType=msg.broker.stomp))' + ) in args: + return [( + 'GlueServiceUniqueID=mq.cro-ngi.hr_msg.broker.stomp_3523291347' + ',Mds-Vo-name=egee.srce.hr,Mds-Vo-name=local,o=grid', + {'GlueServiceUniqueID': + ['mq.cro-ngi.hr_msg.broker.stomp_3523291347'], + 'GlueServiceEndpoint': ['stomp://mq.cro-ngi.hr:6163/']}), + ( + 'GlueServiceUniqueID=broker-prod1.argo.grnet.gr_msg.broker.sto' + 'mp_175215210,Mds-Vo-name=HG-06-EKT,Mds-Vo-name=local,o=grid', + {'GlueServiceUniqueID': + ['broker-prod1.argo.grnet.gr_msg.broker.stomp_175215210'], + 'GlueServiceEndpoint': + ['stomp://broker-prod1.argo.grnet.gr:6163/']} + )] + elif ( + '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=' + 'mq.cro-ngi.hr_msg.broker.stomp_3523291347))' + ) in args: + return [( + 'GlueServiceDataKey=cluster,GlueServiceUniqueID=mq.cro-ngi.hr_' + 'msg.broker.stomp_3523291347,Mds-Vo-name=egee.srce.hr,Mds-Vo-n' + 'ame=local,o=grid', {'GlueServiceDataValue': ['PROD']} + )] + elif ( + '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=' + 'broker-prod1.argo.grnet.gr_msg.broker.stomp_175215210))' + ) in args: + return [( + 'GlueServiceDataKey=cluster,GlueServiceUniqueID=broker-prod1.a' + 'rgo.grnet.gr_msg.broker.stomp_175215210,Mds-Vo-name=HG-06-EKT' + ',Mds-Vo-name=local,o=grid', {'GlueServiceDataValue': ['PROD']} + )] + if __name__ == '__main__': unittest.main() From 852606d0ec287daac01d40534343d65fdf296219 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 5 Mar 2019 09:27:32 +0000 Subject: [PATCH 34/79] Add two small broker tests and correct doctsring --- test/test_brokers.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_brokers.py b/test/test_brokers.py index 1e1aa325..28db2fae 100644 --- a/test/test_brokers.py +++ b/test/test_brokers.py @@ -39,6 +39,9 @@ def test_parse_stomp_url(self): except ValueError: pass + self.assertRaises(ValueError, brokers.parse_stomp_url, + 'stomp://invalid.port.number:abc') + stomp_url = 'stomp://stomp.cern.ch:6262' try: @@ -54,10 +57,7 @@ def test_parse_stomp_url(self): self.fail('Could not parse a valid stomp+ssl URL: %s' % stomp_url) def test_fetch_brokers(self): - ''' - Requires an internet connection to get information from the BDII. - Could fail if the BDII is down. This isn't very unit-test-like. - ''' + """Check the handling of responses from a mocked BDII.""" bdii = 'ldap://no-bdii.utopia.ch:2170' network = 'PROD' @@ -78,6 +78,15 @@ def test_fetch_brokers(self): if '.' not in host: self.fail("Didn't get a hostname from fetch_brokers()") + # Check that no brokers are returned from the TEST-NWOB network. + test_network = 'TEST-NWOB' + # So that there are no external LDAP calls, mock out the LDAP seach. + with mock.patch('ldap.ldapobject.SimpleLDAPObject.search_s', + side_effect=self._mocked_search): + test_bs = sbg.get_broker_hosts_and_ports(brokers.STOMP_SERVICE, + test_network) + self.assertEqual(len(test_bs), 0, "Test brokers found in error.") + def _mocked_search(*args, **kwargs): """Return values to mocked search call based on input.""" From 65971ae254b3a3b2cff73f26891edcc578f038f9 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 9 May 2018 14:04:24 +0100 Subject: [PATCH 35/79] Add support for AMS in underlying module - add a protocol variable to differentiate between STOMP and HTTPS methods. - add destination type variable to allow future versions to differentiate between sending/pulling via HTTPS to AMS, APEL REST interface, data from DataSet storage locations (eg OneData) - add methods to handle HTTPS/AMS sending and receiving --- ssm/ssm2.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 152 insertions(+), 8 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index c891f546..bea1f003 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -43,6 +43,8 @@ import time import logging +from argo_ams_library import ArgoMessagingService, AmsMessage, AmsException + # Set up logging log = logging.getLogger(__name__) @@ -64,8 +66,8 @@ class Ssm2(stomp.ConnectionListener): def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, - enc_cert=None, verify_enc_cert=True, pidfile=None, - path_type='dirq'): + enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq', + protocol="STOMP", dest_type='MQ-BROKER', project=None): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. @@ -92,6 +94,23 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, self._valid_dns = [] self._pidfile = pidfile + # Used to differentiate between STOMP and HTTPS methods + self._protocol = protocol + # Used to differentiate between AMS and other REST endpoints + self._dest_type = dest_type + + # Used when interacting with an Argo Messaging Service + self._project = project + + if self._protocol == "HTTPS" and self._dest_type == "AMS": + self._ams = ArgoMessagingService(endpoint=self._brokers[0], + token=self._pwd, + project=self._project) + + else: + # Set _ams rather than leave it unset + self._ams = None + # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: # Determine what sort of outgoing structure to make @@ -338,6 +357,82 @@ def _send_msg(self, message, msgid): # If it fails, use the v3 metod signiture self._conn.send(to_send, headers=headers) + def pull_msg_rest(self): + """Pull a message via HTTPS from self._dest.""" + if self._protocol != 'HTTPS': + # Then this method should not be called, + # raise an exception if it is. + raise Ssm2Exception('pull_msg_rest called, ' + 'but protocol not set to HTTPS. ' + 'Protocol: %s' % self._protocol) + + if self._dest_type == "AMS": + self._pull_msg_rest_ams() + else: + raise Ssm2Exception('Unsupported Destination Type.' + 'Type: %s' % self._dest_type) + + def _pull_msg_rest_ams(self): + """Pull 1 message from the AMS and acknowledge it.""" + # This method is setup so that you could pull down and + # acknowledge more than one method at a time, but + # currently there is no use case for it. + messages_to_pull = 1 + # ack id's will be stored in this list and then acknowledged + ackids = [] + + for msg_ack_id, msg in self._ams.pull_sub(self._listen, + messages_to_pull): + # Get the AMS message id + msgid = msg.get_msgid() + # Get the SSM dirq id + empaid = msg.get_attr().get('empaid') + # get the message body + body = msg.get_data() + + log.info('Received message. ID = %s', empaid) + extracted_msg, signer, err_msg = self._handle_msg(body) + + try: + # If the message is empty or the error message is not empty + # then reject the message. + if extracted_msg is None or err_msg is not None: + if signer is None: # crypto failed + signer = 'Not available.' + elif extracted_msg is not None: + # If there is a signer then it was rejected for not + # being in the DNs list, so we can use the + # extracted msg, which allows the msg to be + # reloaded if needed. + body = extracted_msg + + log.warn("Message rejected: %s", err_msg) + + name = self._rejectq.add({'body': body, + 'signer': signer, + 'empaid': empaid, + 'error': err_msg}) + log.info("Message saved to reject queue as %s", name) + + else: # message verified ok + name = self._inq.add({'body': extracted_msg, + 'signer': signer, + 'empaid': empaid}) + log.info("Message saved to incoming queue as %s", name) + + # If we get here, we have saved the message, so add the + # ack ID to the list of those to be acknowledged. + ackids.append(msg_ack_id) + + except OSError, error: + log.error('Failed to read or write file: %s', error) + + # pass list of extracted ackIds to AMS Service so that + # it can move the offset for the next subscription pull + # (basically acknowledging pulled messages) + if ackids: + self._ams.ack_sub(self._listen, ackids) + def send_ping(self): ''' If a STOMP connection is left open with no activity for an hour or @@ -361,6 +456,8 @@ def has_msgs(self): def send_all(self): ''' Send all the messages in the outgoing queue. + + Either via STOMP or HTTPS (to an Argo Message Broker). ''' log.info('Found %s messages.', self._outq.count()) for msgid in self._outq: @@ -369,14 +466,45 @@ def send_all(self): continue text = self._outq.get(msgid) - self._send_msg(text, msgid) - log.info('Waiting for broker to accept message.') - while self._last_msg is None: - if not self.connected: - raise Ssm2Exception('Lost connection.') + if self._protocol == "STOMP" and self._dest_type == "MQ-BROKER": + # Then this is an MQ-BROKER and we are going + # to send using the STOMP protocol + self._send_msg(text, msgid) + + log.info('Waiting for broker to accept message.') + while self._last_msg is None: + if not self.connected: + raise Ssm2Exception('Lost connection.') + + elif self._protocol == "HTTPS" and self._dest_type == "AMS": + # Then we are sending to an Argo Messaging Service instance. + if text is not None: + # First we sign the message + to_send = crypto.sign(text, self._cert, self._key) + # Possibly encrypt the message. + if self._enc_cert is not None: + to_send = crypto.encrypt(to_send, self._enc_cert) + + # Then we need to wrap text up as an AMS Message. + message = AmsMessage(data=to_send, + attributes={'empaid': msgid}).dict() + + # Attempt to the AMS Message. + try: + self._ams.publish(self._dest, message) + except AmsException as error: + raise error - time.sleep(0.1) + else: + # The SSM has been improperly configured + raise Ssm2Exception('Unknown configuration: %s and %s' % + self._protocol, + self._dest_type) + + time.sleep(0.1) + # log that the message was sent + log.info("Sent %s", msgid) self._last_msg = None self._outq.remove(msgid) @@ -424,6 +552,10 @@ def handle_connect(self): If more than one is in the list self._network_brokers, try to connect to each in turn until successful. ''' + if self._protocol == "HTTPS": + log.debug('handle_connect called for HTTPS SSM, doing nothing.') + return + for host, port in self._brokers: self._initialise_connection(host, port) try: @@ -443,6 +575,10 @@ def handle_disconnect(self): When disconnected, attempt to reconnect using the same method as used when starting up. ''' + if self._protocol == "HTTPS": + log.debug('handle_disconnect called for HTTPS SSM, doing nothing.') + return + self.connected = False # Shut down properly self.close_connection() @@ -470,6 +606,10 @@ def start_connection(self): If the timeout is reached without receiving confirmation of connection, raise an exception. ''' + if self._protocol == "HTTPS": + log.debug('start_connection called for HTTPS SSM, doing nothing.') + return + if self._conn is None: raise Ssm2Exception('Called start_connection() before a \ connection object was initialised.') @@ -502,6 +642,10 @@ def close_connection(self): in a separate thread, so it can outlive the main process if it is not ended. ''' + if self._protocol == "HTTPS": + log.debug('close_connection called for HTTPS SSM, doing nothing.') + return + try: self._conn.disconnect() except (stomp.exception.NotConnectedException, socket.error): From 21f255f3b11ee706f10bbce1f9021c5819260a8d Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 28 Apr 2017 11:58:49 +0100 Subject: [PATCH 36/79] Refactor sender to allow AMS and STOMP send --- bin/sender.py | 128 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 34 deletions(-) diff --git a/bin/sender.py b/bin/sender.py index 26d79a71..edc0daa0 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -62,40 +62,96 @@ def main(): sys.exit(1) log = logging.getLogger('ssmsend') - + + # Set defaults for MQ-BROKER only variables + use_ssl = None + # Set defaults for AMS only variables + token = None + project = None + log.info(LOG_BREAK) log.info('Starting sending SSM version %s.%s.%s.', *__version__) - # If we can't get a broker to connect to, we have to give up. + + # Determine the protocol and destination type of the SSM to configure. try: - bdii_url = cp.get('broker','bdii') - log.info('Retrieving broker details from %s ...', bdii_url) - bg = StompBrokerGetter(bdii_url) - use_ssl = cp.getboolean('broker', 'use_ssl') - if use_ssl: - service = STOMP_SSL_SERVICE - else: - service = STOMP_SERVICE - brokers = bg.get_broker_hosts_and_ports(service, cp.get('broker','network')) - log.info('Found %s brokers.', len(brokers)) - except ConfigParser.NoOptionError, e: + destination_type = cp.get('SSM Type', 'destination type') + protocol = cp.get('SSM Type', 'protocol') + + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + # if newer configuration settings 'protocol' and 'destination type' + # are not set, use 'STOMP' and 'MQ-BROKER' for + # backwards compatability. + log.debug('No options supplied for destination_type and/or protocol.') + destination_type = 'MQ-BROKER' + protocol = 'STOMP' + + log.info('Setting up SSM with Dest Type: %s, Protocol : %s' + % (destination_type, protocol)) + + if destination_type == 'MQ-BROKER': + # If we can't get a broker to connect to, we have to give up. + try: + bdii_url = cp.get('broker', 'bdii') + log.info('Retrieving broker details from %s ...', bdii_url) + bg = StompBrokerGetter(bdii_url) + use_ssl = cp.getboolean('broker', 'use_ssl') + if use_ssl: + service = STOMP_SSL_SERVICE + else: + service = STOMP_SERVICE + brokers = bg.get_broker_hosts_and_ports(service, cp.get('broker', + 'network')) + log.info('Found %s brokers.', len(brokers)) + except ConfigParser.NoOptionError, e: + try: + host = cp.get('broker', 'host') + port = cp.get('broker', 'port') + brokers = [(host, int(port))] + except ConfigParser.NoOptionError: + log.error('Options incorrectly supplied for either single ' + 'broker or broker network. ' + 'Please check configuration') + log.error('System will exit.') + log.info(LOG_BREAK) + print 'SSM failed to start. See log file for details.' + sys.exit(1) + except ldap.LDAPError, e: + log.error('Could not connect to LDAP server: %s', e) + log.error('System will exit.') + log.info(LOG_BREAK) + print 'SSM failed to start. See log file for details.' + sys.exit(1) + + elif destination_type == 'AMS': + # Then we are setting up an SSM to connect to a AMS. try: + # We only need a hostname, not a port host = cp.get('broker', 'host') - port = cp.get('broker', 'port') - brokers = [(host, int(port))] + # Use brokers variable so subsequent code is not dependant on + # the exact destination type. + brokers = [host] + except ConfigParser.NoOptionError: - log.error('Options incorrectly supplied for either single broker or \ - broker network. Please check configuration') + log.error('The host must be specified when connecting to AMS, ' + 'please check your configuration') log.error('System will exit.') log.info(LOG_BREAK) print 'SSM failed to start. See log file for details.' sys.exit(1) - except ldap.LDAPError, e: - log.error('Could not connect to LDAP server: %s', e) - log.error('System will exit.') - log.info(LOG_BREAK) - print 'SSM failed to start. See log file for details.' - sys.exit(1) - + + # Attempt to configure AMS specific variables. + try: + token = cp.get('messaging', 'token') + project = cp.get('messaging', 'project') + + except (ConfigParser.Error, ValueError, IOError), err: + # A token and project are needed to successfully send to an + # AMS instance, so log and then exit on an error. + log.error('Error configuring AMS values: %s', err) + log.error('SSM will exit.') + print 'SSM failed to start. See log file for details.' + sys.exit(1) + if len(brokers) == 0: log.error('No brokers available.') log.error('System will exit.') @@ -130,16 +186,20 @@ def main(): path_type = 'dirq' sender = Ssm2(brokers, - cp.get('messaging','path'), - path_type=path_type, - cert=cp.get('certificates','certificate'), - key=cp.get('certificates','key'), - dest=cp.get('messaging','destination'), - use_ssl=cp.getboolean('broker','use_ssl'), - capath=cp.get('certificates', 'capath'), - enc_cert=server_cert, - verify_enc_cert=verify_server_cert) - + cp.get('messaging', 'path'), + path_type=path_type, + cert=cp.get('certificates', 'certificate'), + key=cp.get('certificates', 'key'), + dest=cp.get('messaging', 'destination'), + use_ssl=use_ssl, + capath=cp.get('certificates', 'capath'), + enc_cert=server_cert, + verify_enc_cert=verify_server_cert, + dest_type=destination_type, + protocol=protocol, + project=project, + password=token) + if sender.has_msgs(): sender.handle_connect() sender.send_all() From 389453486fbd2c7aded4bae3b361efa1c5119f71 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 28 Apr 2017 11:58:57 +0100 Subject: [PATCH 37/79] Refactor receiver to allow AMS and STOMP receive --- bin/receiver.py | 128 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 95 insertions(+), 33 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index a96afe34..f6bc14e0 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -106,36 +106,91 @@ def main(): global log log = logging.getLogger('ssmreceive') - + + # Set defaults for MQ-BROKER only variables + use_ssl = None + # Set defaults for AMS only variables + project = None + token = None + log.info(LOG_BREAK) log.info('Starting receiving SSM version %s.%s.%s.', *__version__) - # If we can't get a broker to connect to, we have to give up. + # Determine the protocol and destination type of the SSM to configure. try: - bg = StompBrokerGetter(cp.get('broker','bdii')) - use_ssl = cp.getboolean('broker', 'use_ssl') - if use_ssl: - service = STOMP_SSL_SERVICE - else: - service = STOMP_SERVICE - brokers = bg.get_broker_hosts_and_ports(service, cp.get('broker','network')) - except ConfigParser.NoOptionError, e: + destination_type = cp.get('SSM Type', 'destination type') + protocol = cp.get('SSM Type', 'protocol') + + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + # if newer configuration settings 'protocol' and 'destination type' + # are not set, use 'STOMP' and 'MQ-BROKER' for + # backwards compatability. + log.debug('No options supplied for destination_type and/or protocol.') + destination_type = 'MQ-BROKER' + protocol = 'STOMP' + + log.info('Setting up SSM with Dest Type: %s, Protocol : %s' + % (destination_type, protocol)) + + if destination_type == 'MQ-BROKER': + # If we can't get a broker to connect to, we have to give up. + try: + bg = StompBrokerGetter(cp.get('broker', 'bdii')) + use_ssl = cp.getboolean('broker', 'use_ssl') + if use_ssl: + service = STOMP_SSL_SERVICE + else: + service = STOMP_SERVICE + brokers = bg.get_broker_hosts_and_ports(service, cp.get('broker', + 'network')) + except ConfigParser.NoOptionError, e: + try: + host = cp.get('broker', 'host') + port = cp.get('broker', 'port') + brokers = [(host, int(port))] + except ConfigParser.NoOptionError: + log.error('Options incorrectly supplied for either single ' + 'broker or broker network. ' + 'Please check configuration') + log.error('System will exit.') + log.info(LOG_BREAK) + sys.exit(1) + except ldap.SERVER_DOWN, e: + log.error('Could not connect to LDAP server: %s', e) + log.error('System will exit.') + log.info(LOG_BREAK) + sys.exit(1) + + elif destination_type == 'AMS': + # Then we are setting up an SSM to connect to a AMS. try: + # We only need a hostname, not a port host = cp.get('broker', 'host') - port = cp.get('broker', 'port') - brokers = [(host, int(port))] + # Use brokers variable so subsequent code is not dependant on + # the exact destination type. + brokers = [host] + except ConfigParser.NoOptionError: - log.error('Options incorrectly supplied for either single broker \ - or broker network. Please check configuration') + log.error('The host must be specified when connecting to AMS, ' + 'please check your configuration') log.error('System will exit.') log.info(LOG_BREAK) + print 'SSM failed to start. See log file for details.' sys.exit(1) - except ldap.SERVER_DOWN, e: - log.error('Could not connect to LDAP server: %s', e) - log.error('System will exit.') - log.info(LOG_BREAK) - sys.exit(1) - + + # Attempt to configure AMS specific variables. + try: + token = cp.get('messaging', 'token') + project = cp.get('messaging', 'project') + + except (ConfigParser.Error, ValueError, IOError), err: + # A token and project are needed to successfully send to an + # AMS instance, so log and then exit on an error. + log.error('Error configuring AMS values: %s', err) + log.error('SSM will exit.') + print 'SSM failed to start. See log file for details.' + sys.exit(1) + if len(brokers) == 0: log.error('No brokers available.') log.error('System will exit.') @@ -155,10 +210,14 @@ def main(): cert=cp.get('certificates','certificate'), key=cp.get('certificates','key'), listen=cp.get('messaging','destination'), - use_ssl=cp.getboolean('broker','use_ssl'), + use_ssl=use_ssl, capath=cp.get('certificates', 'capath'), check_crls=cp.getboolean('certificates', 'check_crls'), - pidfile=pidfile) + pidfile=pidfile, + project=project, + password=token, + dest_type=destination_type, + protocol=protocol) log.info('Fetching valid DNs.') dns = get_dns(options.dn_file) @@ -181,23 +240,26 @@ def main(): while True: time.sleep(1) + if protocol == 'HTTPS': + ssm.pull_msg_rest() if i % REFRESH_DNS == 0: log.info('Refreshing valid DNs and then sending ping.') dns = get_dns(options.dn_file) ssm.set_dns(dns) - try: - ssm.send_ping() - except NotConnectedException: - log.warn('Connection lost.') - ssm.shutdown() - dc.close() - log.info("Waiting for 10 minutes before restarting...") - time.sleep(10 * 60) - log.info('Restarting SSM.') - dc.open() - ssm.startup() + if protocol == 'STOMP': + try: + ssm.send_ping() + except NotConnectedException: + log.warn('Connection lost.') + ssm.shutdown() + dc.close() + log.info("Waiting for 10 minutes before restarting...") + time.sleep(10 * 60) + log.info('Restarting SSM.') + dc.open() + ssm.startup() i += 1 From 7670fdaabe18c6911d692cecf3d9102e104af7f6 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 17 May 2017 15:31:22 +0100 Subject: [PATCH 38/79] Add support for AMS to example config files --- conf/receiver.cfg | 9 +++++++++ conf/sender.cfg | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/conf/receiver.cfg b/conf/receiver.cfg index cb36d28a..8429d1a6 100644 --- a/conf/receiver.cfg +++ b/conf/receiver.cfg @@ -1,3 +1,10 @@ +# SSM protocol/destination type options +[SSM Type] +# Either 'MQ-BROKER' for EGI Message Brokers or 'AMS' for Argo Messaging Service +destination type: MQ-BROKER +# Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service +protocol: STOMP + [broker] # The SSM will query a BDII to find brokers available. These details are for the @@ -19,6 +26,8 @@ capath: /etc/grid-security/certificates check_crls: false [messaging] +# Project to which SSM will pull messages from. Uncomment and populate for AMS sending +# project: # Destination to which SSM will listen. destination: /queue/ssm2test diff --git a/conf/sender.cfg b/conf/sender.cfg index e4372ab0..f3ba8f3e 100644 --- a/conf/sender.cfg +++ b/conf/sender.cfg @@ -1,3 +1,11 @@ +################################################################################ +# SSM protocol/destination type options. +[SSM Type] +# Either 'MQ-BROKER' for EGI Message Brokers or 'AMS' for Argo Messaging Service +destination type: MQ-BROKER +# Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service +protocol: STOMP + ################################################################################ # Required: broker configuration options # @@ -34,6 +42,8 @@ capath: /etc/grid-security/certificates # Messaging configuration. # [messaging] +# Project to which SSM will send messages. Uncomment and populate for AMS sending +# project: # Queue to which SSM will send messages destination: From 12f675ee38b921a0d4b1f723ccf9bad80f5abf09 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 9 May 2018 14:57:33 +0100 Subject: [PATCH 39/79] Add AMS library to requirements.txt - I did consider keeping it constrained to 0.4.2, the latest version I tested against as I don't believe I added any meaningful test of this new functionality, but the developers say the 0.X series is stable. --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index d3838cdc..02bfe1b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,7 @@ python-daemon<2.2.0 python-ldap # Dependencies for optional dirq based sending dirq + +# Dependencies for experimental AMS functionality + +argo-ams-library From 86c1b0afdab4bb9f8ade343fae44cc5d69f1b356 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 1 Jun 2018 14:10:44 +0100 Subject: [PATCH 40/79] Handle AMS disconnects the same way as for STOMP --- bin/receiver.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index f6bc14e0..584f9b0f 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -24,6 +24,7 @@ from ssm import __version__, set_up_logging, LOG_BREAK from stomp.exception import NotConnectedException +from argo_ams_library import AmsConnectionException as NotConnectedException import time import logging.config @@ -238,28 +239,31 @@ def main(): i = 0 # The message listening loop. while True: + try: + time.sleep(1) + if protocol == 'HTTPS': + # We need to pull down messages as part of + # this loop when using HTTPS. + ssm.pull_msg_rest() - time.sleep(1) - if protocol == 'HTTPS': - ssm.pull_msg_rest() - - if i % REFRESH_DNS == 0: - log.info('Refreshing valid DNs and then sending ping.') - dns = get_dns(options.dn_file) - ssm.set_dns(dns) + if i % REFRESH_DNS == 0: + log.info('Refreshing valid DNs and then sending ping.') + dns = get_dns(options.dn_file) + ssm.set_dns(dns) - if protocol == 'STOMP': - try: + if protocol == 'STOMP': ssm.send_ping() - except NotConnectedException: - log.warn('Connection lost.') - ssm.shutdown() - dc.close() - log.info("Waiting for 10 minutes before restarting...") - time.sleep(10 * 60) - log.info('Restarting SSM.') - dc.open() - ssm.startup() + + except NotConnectedException as error: + log.warn('Connection lost.') + log.debug(error) + ssm.shutdown() + dc.close() + log.info("Waiting for 10 minutes before restarting...") + time.sleep(10 * 60) + log.info('Restarting SSM.') + dc.open() + ssm.startup() i += 1 From 8231ff5bf6c413dc797b1974071a12170f0f39b3 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Fri, 1 Jun 2018 14:37:11 +0100 Subject: [PATCH 41/79] Log the ARGO message ID on send and receive. - if using the AMS. --- ssm/ssm2.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index bea1f003..9bf72e2b 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -390,7 +390,8 @@ def _pull_msg_rest_ams(self): # get the message body body = msg.get_data() - log.info('Received message. ID = %s', empaid) + log.info('Received message. ID = %s, Argo ID = %s', empaid, msgid) + extracted_msg, signer, err_msg = self._handle_msg(body) try: @@ -477,6 +478,8 @@ def send_all(self): if not self.connected: raise Ssm2Exception('Lost connection.') + log_string = "Sent %s" % msgid + elif self._protocol == "HTTPS" and self._dest_type == "AMS": # Then we are sending to an Argo Messaging Service instance. if text is not None: @@ -492,10 +495,13 @@ def send_all(self): # Attempt to the AMS Message. try: - self._ams.publish(self._dest, message) + argo_response = self._ams.publish(self._dest, message) except AmsException as error: raise error + argo_id = argo_response['messageIds'][0] + log_string = "Sent %s, Argo ID: %s" % (msgid, argo_id) + else: # The SSM has been improperly configured raise Ssm2Exception('Unknown configuration: %s and %s' % @@ -504,7 +510,7 @@ def send_all(self): time.sleep(0.1) # log that the message was sent - log.info("Sent %s", msgid) + log.info(log_string) self._last_msg = None self._outq.remove(msgid) From 12eb521e9601b5415be75c23b5cbb8501e396bec Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 19 Sep 2018 10:12:38 +0100 Subject: [PATCH 42/79] Make token optional when sending via AMS - as the user can now use a cert key pair instead of a token - set the default token to "" rather than None as if the default token gets passed to AMS, it will treat None as a token and "" as the absence of a token. --- bin/receiver.py | 2 +- bin/sender.py | 14 ++++++++++---- ssm/ssm2.py | 2 ++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index 584f9b0f..cb85cafb 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -112,7 +112,7 @@ def main(): use_ssl = None # Set defaults for AMS only variables project = None - token = None + token = "" log.info(LOG_BREAK) log.info('Starting receiving SSM version %s.%s.%s.', *__version__) diff --git a/bin/sender.py b/bin/sender.py index edc0daa0..a59c1bc3 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -66,7 +66,7 @@ def main(): # Set defaults for MQ-BROKER only variables use_ssl = None # Set defaults for AMS only variables - token = None + token = "" project = None log.info(LOG_BREAK) @@ -139,19 +139,25 @@ def main(): print 'SSM failed to start. See log file for details.' sys.exit(1) - # Attempt to configure AMS specific variables. + # Attempt to configure AMS project variable. try: - token = cp.get('messaging', 'token') project = cp.get('messaging', 'project') except (ConfigParser.Error, ValueError, IOError), err: - # A token and project are needed to successfully send to an + # A project is needed to successfully send to an # AMS instance, so log and then exit on an error. log.error('Error configuring AMS values: %s', err) log.error('SSM will exit.') print 'SSM failed to start. See log file for details.' sys.exit(1) + try: + token = cp.get('messaging', 'token') + except (ConfigParser.Error, ValueError, IOError), err: + # A token is not necessarily needed, if the cert and key can be + # used by the underlying auth system to get a suitable token. + log.info('No AMS token provided, using cert/key pair instead.') + if len(brokers) == 0: log.error('No brokers available.') log.error('System will exit.') diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 9bf72e2b..1b58c584 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -105,6 +105,8 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, if self._protocol == "HTTPS" and self._dest_type == "AMS": self._ams = ArgoMessagingService(endpoint=self._brokers[0], token=self._pwd, + cert=self._cert, + key=self._key, project=self._project) else: From fa6fd393f153d1b55bec596d32c1a3ffab5acfba Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 9 May 2018 16:27:51 +0100 Subject: [PATCH 43/79] Update README.md and add migration guide --- README.md | 18 +++++++++++--- migrating_to_ams_broker.md | 51 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 migrating_to_ams_broker.md diff --git a/README.md b/README.md index 7b59125e..5a8c0628 100644 --- a/README.md +++ b/README.md @@ -155,15 +155,21 @@ add your messages using the `add` method. ## Running the SSM -### Sender +### Sender (sending via the EGI message brokers) * Run 'ssmsend' * SSM will pick up any messages and send them to the configured queue on the configured broker - + +### Sender (sending via the ARGO Messaging Service (AMS)) + * Edit your sender configuration, usually under `/etc/apel/sender.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#sender) with some minor differences: + * There is no need to add the `[SSM Type]` section as it already exists. instead change the `destination type` and `protocol` to `AMS` and `HTTPS` respectively. + * Uncomment `project` and set it to the appropriate project. + * Then run 'ssmsend', SSM will pick up any messages and send them via the ARGO Messaging Service. + ### Sender (container) * Download the example [configuration file](conf/sender.cfg) - * Edit the downloaded sender.cfg file to configure the queue and broker + * Edit the downloaded `sender.cfg` file as above for sending either via the [EGI message brokers](README.md#sender-sending-via-the-egi-message-brokers) or the [ARGO Messaging Service](https://github.com/gregcorbett/ssm/blob/cert_enabled_ams_support/README.md#sender-sending-via-the-argo-messaging-service-ams). * Run the following docker command to send ``` docker run \ @@ -204,6 +210,12 @@ add your messages using the `add` method. * SSM will receive any messages on the specified queue and write them to the filesystem * To stop, run ```'kill `cat /var/run/apel/ssm.pid`'``` + +### Receiver (receiving via the ARGO Messaging Service (AMS)) + * Edit your receiver configuration, usually under `/etc/apel/receiver.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#receiver) with some minor differences: + * There is no need to add the `[SSM Type]` section as it already exists. instead change the `destination type` and `protocol` to `AMS` and `HTTPS` respectively. + * Uncomment `project` and set it to the appropriate project. + * Then run your receiver ([as a service](README.md#receiver-service), [as a container](README.md#receiver-container) or [manually](README.md#receiver-manual)) as above. ## Removing the RPM diff --git a/migrating_to_ams_broker.md b/migrating_to_ams_broker.md new file mode 100644 index 00000000..b4c3ba37 --- /dev/null +++ b/migrating_to_ams_broker.md @@ -0,0 +1,51 @@ +# Migrating from using EGI ActiveMQ Message Brokers to using EGI ARGO Messaging Service + +Migration requires upgrading to SSM-X.X.X and adding new values to your configuration. + +## Sender + +The sender configuration is usually found under `/etc/apel/sender.cfg`. Follow the following steps to migrate. + +1. Comment out `bdii` and `network` +2. Uncomment `host` and set it to `msg-devel.argo.grnet.gr` +3. Add the following as a new section in your configuration. +``` +# SSM protocol/destination type options +[SSM Type] +# Either 'MQ-BROKER' for EGI Message Brokers or 'AMS' for Argo Messaging Service +destination type: AMS +# Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service +protocol: HTTPS +``` +4. Add the following to the `[messaging]` section of your configuration +``` +# Project to which SSM will pull messages from. Uncomment and populate for AMS sending +project: EGI-ACCOUNTING +``` +5. To send to the APEL central server, change `destination` to one of the following depending on your type of accounting: + * `gLite-APEL` for Grid Accounting + * `eu.egi.cloud.accounting` for Cloud Accounting + * `eu.egi.storage.accounting` for Storage Accounting + +The next time `ssmsend` runs it should be using the AMS. You can check this by looking in the logs a successful run, which will look like: +``` +2018-09-19 14:18:06,423 - ssmsend - INFO - ======================================== +2018-09-19 14:18:06,424 - ssmsend - INFO - Starting sending SSM version 2.2.1. +2018-09-19 14:18:06,424 - ssmsend - INFO - Setting up SSM with Dest Type: AMS, Protocol : HTTPS +2018-09-19 14:18:06,424 - ssmsend - INFO - No AMS token provided, using cert/key pair instead. +2018-09-19 14:18:06,424 - ssmsend - INFO - No server certificate supplied. Will not encrypt messages. +2018-09-19 14:18:07,061 - ssm.ssm2 - INFO - Found 1 messages. +2018-09-19 14:18:07,860 - ssm.ssm2 - INFO - Sent 5ba24c88/5ba24c8f0f129d, Argo ID: 18 +2018-09-19 14:18:07,861 - ssm.ssm2 - INFO - Tidying message directory. +2018-09-19 14:18:07,862 - ssmsend - INFO - SSM run has finished. +2018-09-19 14:18:07,862 - ssmsend - INFO - SSM has shut down. +2018-09-19 14:18:07,862 - ssmsend - INFO - ======================================== +``` + +## Receiver +1. Follow the steps 1 to 4 from above editing your receiver configuration, usually found under `/etc/apel/receiver.cfg` +2. Change `destination` to be the subscription you are using to pull messages down. +3. Add your token to the `[messaging]` section of your configuration. +``` +token: your_token_here +``` From d8886ff777fae9363f4a0d2c561bde952a915066 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 3 Oct 2018 10:26:40 +0100 Subject: [PATCH 44/79] Prevent an empaid-less message blocking the queue - A message without an empaid could be received if it wasn't sent via the SSM, we need to pull down that message else it to prevent it blocking the message queue. --- ssm/ssm2.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 1b58c584..9821a807 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -388,7 +388,14 @@ def _pull_msg_rest_ams(self): # Get the AMS message id msgid = msg.get_msgid() # Get the SSM dirq id - empaid = msg.get_attr().get('empaid') + try: + empaid = msg.get_attr().get('empaid') + except AttributeError: + # A message without an empaid could be received if it wasn't + # sent via the SSM, we need to pull down that message + # to prevent it blocking the message queue. + log.debug("Message %s has no empaid.", msgid) + empaid = "N/A" # get the message body body = msg.get_data() From 2c849495f490ac894197d0316caaa1173e1bad39 Mon Sep 17 00:00:00 2001 From: Greg Corbett Date: Wed, 3 Oct 2018 12:10:51 +0100 Subject: [PATCH 45/79] Fix some Markdown lint --- README.md | 6 ++++-- migrating_to_ams_broker.md | 14 ++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5a8c0628..97cc0e24 100644 --- a/README.md +++ b/README.md @@ -155,13 +155,14 @@ add your messages using the `add` method. ## Running the SSM -### Sender (sending via the EGI message brokers) +### Sender (sending via the EGI message brokers) * Run 'ssmsend' * SSM will pick up any messages and send them to the configured queue on the configured broker -### Sender (sending via the ARGO Messaging Service (AMS)) +### Sender (sending via the ARGO Messaging Service (AMS)) + * Edit your sender configuration, usually under `/etc/apel/sender.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#sender) with some minor differences: * There is no need to add the `[SSM Type]` section as it already exists. instead change the `destination type` and `protocol` to `AMS` and `HTTPS` respectively. * Uncomment `project` and set it to the appropriate project. @@ -212,6 +213,7 @@ add your messages using the `add` method. * To stop, run ```'kill `cat /var/run/apel/ssm.pid`'``` ### Receiver (receiving via the ARGO Messaging Service (AMS)) + * Edit your receiver configuration, usually under `/etc/apel/receiver.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#receiver) with some minor differences: * There is no need to add the `[SSM Type]` section as it already exists. instead change the `destination type` and `protocol` to `AMS` and `HTTPS` respectively. * Uncomment `project` and set it to the appropriate project. diff --git a/migrating_to_ams_broker.md b/migrating_to_ams_broker.md index b4c3ba37..646b1301 100644 --- a/migrating_to_ams_broker.md +++ b/migrating_to_ams_broker.md @@ -7,8 +7,8 @@ Migration requires upgrading to SSM-X.X.X and adding new values to your configur The sender configuration is usually found under `/etc/apel/sender.cfg`. Follow the following steps to migrate. 1. Comment out `bdii` and `network` -2. Uncomment `host` and set it to `msg-devel.argo.grnet.gr` -3. Add the following as a new section in your configuration. +1. Uncomment `host` and set it to `msg-devel.argo.grnet.gr` +1. Add the following as a new section in your configuration. ``` # SSM protocol/destination type options [SSM Type] @@ -17,17 +17,18 @@ destination type: AMS # Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service protocol: HTTPS ``` -4. Add the following to the `[messaging]` section of your configuration +1. Add the following to the `[messaging]` section of your configuration ``` # Project to which SSM will pull messages from. Uncomment and populate for AMS sending project: EGI-ACCOUNTING ``` -5. To send to the APEL central server, change `destination` to one of the following depending on your type of accounting: +1. To send to the APEL central server, change `destination` to one of the following depending on your type of accounting: * `gLite-APEL` for Grid Accounting * `eu.egi.cloud.accounting` for Cloud Accounting * `eu.egi.storage.accounting` for Storage Accounting The next time `ssmsend` runs it should be using the AMS. You can check this by looking in the logs a successful run, which will look like: + ``` 2018-09-19 14:18:06,423 - ssmsend - INFO - ======================================== 2018-09-19 14:18:06,424 - ssmsend - INFO - Starting sending SSM version 2.2.1. @@ -43,9 +44,10 @@ The next time `ssmsend` runs it should be using the AMS. You can check this by l ``` ## Receiver + 1. Follow the steps 1 to 4 from above editing your receiver configuration, usually found under `/etc/apel/receiver.cfg` -2. Change `destination` to be the subscription you are using to pull messages down. -3. Add your token to the `[messaging]` section of your configuration. +1. Change `destination` to be the subscription you are using to pull messages down. +1. Add your token to the `[messaging]` section of your configuration. ``` token: your_token_here ``` From c9eee9ce4398146dddcdf7984f065f53be8b57a3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Dec 2018 12:04:31 +0000 Subject: [PATCH 46/79] Simplify AMS options and tidy up config files - Collapse options to switch between STOMP message brokers and AMS to a single config option to simplify things for the user. - Rename AMS project config option to 'ams_project' to make purpose clear. This also means that it doens't need to go into a completely separate AMS section. - Remove unnecessary comments from receiver config. - Update host & port for broker in receiver config (not done for sender to avoid misconfiguration at a site cuasing problems centrally). --- conf/receiver.cfg | 13 +++++-------- conf/sender.cfg | 27 ++++++--------------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/conf/receiver.cfg b/conf/receiver.cfg index 8429d1a6..0d43326b 100644 --- a/conf/receiver.cfg +++ b/conf/receiver.cfg @@ -1,8 +1,5 @@ -# SSM protocol/destination type options -[SSM Type] -# Either 'MQ-BROKER' for EGI Message Brokers or 'AMS' for Argo Messaging Service -destination type: MQ-BROKER -# Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service +[receiver] +# Either 'STOMP' for STOMP message brokers or 'AMS' for Argo Messaging Service protocol: STOMP [broker] @@ -15,7 +12,7 @@ network: TEST-NWOB #host: test-msg01.afroditi.hellasgrid.gr #port: 6163 -# broker authentication. If use-ssl is set, the certificates configured +# broker authentication. If use_ssl is set, the certificates configured # in the mandatory [certificates] section will be used. use_ssl: true @@ -26,8 +23,8 @@ capath: /etc/grid-security/certificates check_crls: false [messaging] -# Project to which SSM will pull messages from. Uncomment and populate for AMS sending -# project: +# If using AMS this is the project that SSM will connect to. Ignored for STOMP. +ams_project: # Destination to which SSM will listen. destination: /queue/ssm2test diff --git a/conf/sender.cfg b/conf/sender.cfg index f3ba8f3e..62d85da8 100644 --- a/conf/sender.cfg +++ b/conf/sender.cfg @@ -1,15 +1,7 @@ -################################################################################ -# SSM protocol/destination type options. -[SSM Type] -# Either 'MQ-BROKER' for EGI Message Brokers or 'AMS' for Argo Messaging Service -destination type: MQ-BROKER -# Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service +[sender] +# Either 'STOMP' for STOMP message brokers or 'AMS' for Argo Messaging Service protocol: STOMP -################################################################################ -# Required: broker configuration options -# - [broker] # The SSM will query a BDII to find brokers available. These details are for the @@ -17,17 +9,13 @@ protocol: STOMP bdii: ldap://lcg-bdii.cern.ch:2170 network: PROD # OR (these details will only be used if the broker network settings aren't used) -#host: test-msg01.afroditi.hellasgrid.gr -#port: 6163 +#host: broker-prod1.argo.grnet.gr +#port: 6162 # broker authentication. If use_ssl is set, the certificates configured # in the mandatory [certificates] section will be used. use_ssl: true - -################################################################################ -# Required: Certificate configuration - [certificates] certificate: /etc/grid-security/hostcert.pem key: /etc/grid-security/hostkey.pem @@ -38,12 +26,9 @@ capath: /etc/grid-security/certificates # the final server that's receiving your messages; not your own, nor the broker. #server_cert: /etc/grid-security/servercert.pem -################################################################################ -# Messaging configuration. -# [messaging] -# Project to which SSM will send messages. Uncomment and populate for AMS sending -# project: +# If using AMS this is the project that SSM will connect to. Ignored for STOMP. +ams_project: # Queue to which SSM will send messages destination: From 4e827bed51938834b829d5b2f75eb8a847728a8c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Dec 2018 15:46:40 +0000 Subject: [PATCH 47/79] Collapse destination_type into protocol - Collapse destination_type into protocol to simplify code. MQ-BROKER converts to STOMP, and HTTPS converts to AMS. - Change name of section that variables are fetched form the config to match new names. - Remove some unnecessary checks and reorder arguments. --- bin/receiver.py | 29 ++++++++++++---------------- bin/sender.py | 21 ++++++++------------ ssm/ssm2.py | 51 ++++++++++++++++++------------------------------- 3 files changed, 39 insertions(+), 62 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index cb85cafb..2e1c3900 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -108,32 +108,28 @@ def main(): global log log = logging.getLogger('ssmreceive') - # Set defaults for MQ-BROKER only variables + # Set defaults for STOMP only variables use_ssl = None # Set defaults for AMS only variables - project = None token = "" + project = None log.info(LOG_BREAK) log.info('Starting receiving SSM version %s.%s.%s.', *__version__) # Determine the protocol and destination type of the SSM to configure. try: - destination_type = cp.get('SSM Type', 'destination type') - protocol = cp.get('SSM Type', 'protocol') + protocol = cp.get('receiver', 'protocol') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - # if newer configuration settings 'protocol' and 'destination type' - # are not set, use 'STOMP' and 'MQ-BROKER' for - # backwards compatability. - log.debug('No options supplied for destination_type and/or protocol.') - destination_type = 'MQ-BROKER' + # If the newer configuration setting 'protocol' is not set, use 'STOMP' + # for backwards compatability. + log.debug("No option set for 'protocol'. Defaulting to 'STOMP'.") protocol = 'STOMP' - log.info('Setting up SSM with Dest Type: %s, Protocol : %s' - % (destination_type, protocol)) + log.info('Setting up SSM with protocol: %s', protocol) - if destination_type == 'MQ-BROKER': + if protocol == 'STOMP': # If we can't get a broker to connect to, we have to give up. try: bg = StompBrokerGetter(cp.get('broker', 'bdii')) @@ -162,7 +158,7 @@ def main(): log.info(LOG_BREAK) sys.exit(1) - elif destination_type == 'AMS': + elif protocol == 'AMS': # Then we are setting up an SSM to connect to a AMS. try: # We only need a hostname, not a port @@ -215,11 +211,10 @@ def main(): capath=cp.get('certificates', 'capath'), check_crls=cp.getboolean('certificates', 'check_crls'), pidfile=pidfile, + protocol=protocol, project=project, - password=token, - dest_type=destination_type, - protocol=protocol) - + password=token) + log.info('Fetching valid DNs.') dns = get_dns(options.dn_file) ssm.set_dns(dns) diff --git a/bin/sender.py b/bin/sender.py index a59c1bc3..8249fd3a 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -63,7 +63,7 @@ def main(): log = logging.getLogger('ssmsend') - # Set defaults for MQ-BROKER only variables + # Set defaults for STOMP only variables use_ssl = None # Set defaults for AMS only variables token = "" @@ -74,21 +74,17 @@ def main(): # Determine the protocol and destination type of the SSM to configure. try: - destination_type = cp.get('SSM Type', 'destination type') - protocol = cp.get('SSM Type', 'protocol') + protocol = cp.get('sender', 'protocol') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - # if newer configuration settings 'protocol' and 'destination type' - # are not set, use 'STOMP' and 'MQ-BROKER' for - # backwards compatability. - log.debug('No options supplied for destination_type and/or protocol.') - destination_type = 'MQ-BROKER' + # If the newer configuration setting 'protocol' is not set, use 'STOMP' + # for backwards compatability. + log.debug("No option set for 'protocol'. Defaulting to 'STOMP'.") protocol = 'STOMP' - log.info('Setting up SSM with Dest Type: %s, Protocol : %s' - % (destination_type, protocol)) + log.info('Setting up SSM with protocol: %s', protocol) - if destination_type == 'MQ-BROKER': + if protocol == 'STOMP': # If we can't get a broker to connect to, we have to give up. try: bdii_url = cp.get('broker', 'bdii') @@ -122,7 +118,7 @@ def main(): print 'SSM failed to start. See log file for details.' sys.exit(1) - elif destination_type == 'AMS': + elif protocol == 'AMS': # Then we are setting up an SSM to connect to a AMS. try: # We only need a hostname, not a port @@ -201,7 +197,6 @@ def main(): capath=cp.get('certificates', 'capath'), enc_cert=server_cert, verify_enc_cert=verify_server_cert, - dest_type=destination_type, protocol=protocol, project=project, password=token) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 9821a807..fbff1e57 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -67,7 +67,7 @@ class Ssm2(stomp.ConnectionListener): def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq', - protocol="STOMP", dest_type='MQ-BROKER', project=None): + protocol="STOMP", project=None): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. @@ -94,25 +94,19 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, self._valid_dns = [] self._pidfile = pidfile - # Used to differentiate between STOMP and HTTPS methods + # Used to differentiate between STOMP and AMS methods self._protocol = protocol - # Used to differentiate between AMS and other REST endpoints - self._dest_type = dest_type # Used when interacting with an Argo Messaging Service self._project = project - if self._protocol == "HTTPS" and self._dest_type == "AMS": + if self._protocol == 'AMS': self._ams = ArgoMessagingService(endpoint=self._brokers[0], token=self._pwd, cert=self._cert, key=self._key, project=self._project) - else: - # Set _ams rather than leave it unset - self._ams = None - # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: # Determine what sort of outgoing structure to make @@ -361,18 +355,13 @@ def _send_msg(self, message, msgid): def pull_msg_rest(self): """Pull a message via HTTPS from self._dest.""" - if self._protocol != 'HTTPS': + if self._protocol != 'AMS': # Then this method should not be called, # raise an exception if it is. raise Ssm2Exception('pull_msg_rest called, ' - 'but protocol not set to HTTPS. ' + 'but protocol not set to AMS. ' 'Protocol: %s' % self._protocol) - - if self._dest_type == "AMS": - self._pull_msg_rest_ams() - else: - raise Ssm2Exception('Unsupported Destination Type.' - 'Type: %s' % self._dest_type) + self._pull_msg_rest_ams() def _pull_msg_rest_ams(self): """Pull 1 message from the AMS and acknowledge it.""" @@ -477,9 +466,8 @@ def send_all(self): text = self._outq.get(msgid) - if self._protocol == "STOMP" and self._dest_type == "MQ-BROKER": - # Then this is an MQ-BROKER and we are going - # to send using the STOMP protocol + if self._protocol == 'STOMP': + # Then we are sending to a STOMP message broker. self._send_msg(text, msgid) log.info('Waiting for broker to accept message.') @@ -489,7 +477,7 @@ def send_all(self): log_string = "Sent %s" % msgid - elif self._protocol == "HTTPS" and self._dest_type == "AMS": + elif self._protocol == 'AMS': # Then we are sending to an Argo Messaging Service instance. if text is not None: # First we sign the message @@ -513,9 +501,8 @@ def send_all(self): else: # The SSM has been improperly configured - raise Ssm2Exception('Unknown configuration: %s and %s' % - self._protocol, - self._dest_type) + raise Ssm2Exception('Unknown messaging protocol: %s' % + self._protocol) time.sleep(0.1) # log that the message was sent @@ -567,8 +554,8 @@ def handle_connect(self): If more than one is in the list self._network_brokers, try to connect to each in turn until successful. ''' - if self._protocol == "HTTPS": - log.debug('handle_connect called for HTTPS SSM, doing nothing.') + if self._protocol == 'AMS': + log.debug('handle_connect called for AMS, doing nothing.') return for host, port in self._brokers: @@ -590,8 +577,8 @@ def handle_disconnect(self): When disconnected, attempt to reconnect using the same method as used when starting up. ''' - if self._protocol == "HTTPS": - log.debug('handle_disconnect called for HTTPS SSM, doing nothing.') + if self._protocol == 'AMS': + log.debug('handle_disconnect called for AMS, doing nothing.') return self.connected = False @@ -621,8 +608,8 @@ def start_connection(self): If the timeout is reached without receiving confirmation of connection, raise an exception. ''' - if self._protocol == "HTTPS": - log.debug('start_connection called for HTTPS SSM, doing nothing.') + if self._protocol == 'AMS': + log.debug('start_connection called for AMS, doing nothing.') return if self._conn is None: @@ -657,8 +644,8 @@ def close_connection(self): in a separate thread, so it can outlive the main process if it is not ended. ''' - if self._protocol == "HTTPS": - log.debug('close_connection called for HTTPS SSM, doing nothing.') + if self._protocol == 'AMS': + log.debug('close_connection called for AMS, doing nothing.') return try: From 23c654510bbe152556591c6df0aacc428683ee0f Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Dec 2018 15:50:37 +0000 Subject: [PATCH 48/79] Remove unncessary try...except Remove unncessary try...except that raises the exception without modifying it. --- ssm/ssm2.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index fbff1e57..56f481e1 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -490,11 +490,7 @@ def send_all(self): message = AmsMessage(data=to_send, attributes={'empaid': msgid}).dict() - # Attempt to the AMS Message. - try: - argo_response = self._ams.publish(self._dest, message) - except AmsException as error: - raise error + argo_response = self._ams.publish(self._dest, message) argo_id = argo_response['messageIds'][0] log_string = "Sent %s, Argo ID: %s" % (msgid, argo_id) From 115e2fdadb2a98650a31e5e11dba7470c25e37db Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Dec 2018 16:06:44 +0000 Subject: [PATCH 49/79] Replace magic protocol strings with constants Replace magic strings used for protocols with constants defined in the Ssm2 class as this is good practice. --- bin/receiver.py | 12 ++++++------ bin/sender.py | 8 ++++---- ssm/ssm2.py | 26 +++++++++++++++----------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index 2e1c3900..c9fb0d5f 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -124,12 +124,12 @@ def main(): except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): # If the newer configuration setting 'protocol' is not set, use 'STOMP' # for backwards compatability. - log.debug("No option set for 'protocol'. Defaulting to 'STOMP'.") - protocol = 'STOMP' + protocol = Ssm2.STOMP_MESSAGING + log.debug("No option set for 'protocol'. Defaulting to %s.", protocol) log.info('Setting up SSM with protocol: %s', protocol) - if protocol == 'STOMP': + if protocol == Ssm2.STOMP_MESSAGING: # If we can't get a broker to connect to, we have to give up. try: bg = StompBrokerGetter(cp.get('broker', 'bdii')) @@ -158,7 +158,7 @@ def main(): log.info(LOG_BREAK) sys.exit(1) - elif protocol == 'AMS': + elif protocol == Ssm2.AMS_MESSAGING: # Then we are setting up an SSM to connect to a AMS. try: # We only need a hostname, not a port @@ -236,7 +236,7 @@ def main(): while True: try: time.sleep(1) - if protocol == 'HTTPS': + if protocol == Ssm2.AMS_MESSAGING: # We need to pull down messages as part of # this loop when using HTTPS. ssm.pull_msg_rest() @@ -246,7 +246,7 @@ def main(): dns = get_dns(options.dn_file) ssm.set_dns(dns) - if protocol == 'STOMP': + if protocol == Ssm2.STOMP_MESSAGING: ssm.send_ping() except NotConnectedException as error: diff --git a/bin/sender.py b/bin/sender.py index 8249fd3a..5ef82999 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -79,12 +79,12 @@ def main(): except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): # If the newer configuration setting 'protocol' is not set, use 'STOMP' # for backwards compatability. - log.debug("No option set for 'protocol'. Defaulting to 'STOMP'.") - protocol = 'STOMP' + protocol = Ssm2.STOMP_MESSAGING + log.debug("No option set for 'protocol'. Defaulting to %s.", protocol) log.info('Setting up SSM with protocol: %s', protocol) - if protocol == 'STOMP': + if protocol == Ssm2.STOMP_MESSAGING: # If we can't get a broker to connect to, we have to give up. try: bdii_url = cp.get('broker', 'bdii') @@ -118,7 +118,7 @@ def main(): print 'SSM failed to start. See log file for details.' sys.exit(1) - elif protocol == 'AMS': + elif protocol == Ssm2.AMS_MESSAGING: # Then we are setting up an SSM to connect to a AMS. try: # We only need a hostname, not a port diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 56f481e1..edcdd536 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -64,10 +64,14 @@ class Ssm2(stomp.ConnectionListener): REJECT_SCHEMA = {'body': 'string', 'signer':'string?', 'empaid':'string?', 'error':'string'} CONNECTION_TIMEOUT = 10 - def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, - capath=None, check_crls=False, use_ssl=False, username=None, password=None, + # Messaging protocols + STOMP_MESSAGING = 'STOMP' + AMS_MESSAGING = 'AMS' + + def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, + capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq', - protocol="STOMP", project=None): + protocol=STOMP_MESSAGING, project=None): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. @@ -100,7 +104,7 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, # Used when interacting with an Argo Messaging Service self._project = project - if self._protocol == 'AMS': + if self._protocol == Ssm2.AMS_MESSAGING: self._ams = ArgoMessagingService(endpoint=self._brokers[0], token=self._pwd, cert=self._cert, @@ -355,7 +359,7 @@ def _send_msg(self, message, msgid): def pull_msg_rest(self): """Pull a message via HTTPS from self._dest.""" - if self._protocol != 'AMS': + if self._protocol != Ssm2.AMS_MESSAGING: # Then this method should not be called, # raise an exception if it is. raise Ssm2Exception('pull_msg_rest called, ' @@ -466,7 +470,7 @@ def send_all(self): text = self._outq.get(msgid) - if self._protocol == 'STOMP': + if self._protocol == Ssm2.STOMP_MESSAGING: # Then we are sending to a STOMP message broker. self._send_msg(text, msgid) @@ -477,7 +481,7 @@ def send_all(self): log_string = "Sent %s" % msgid - elif self._protocol == 'AMS': + elif self._protocol == Ssm2.AMS_MESSAGING: # Then we are sending to an Argo Messaging Service instance. if text is not None: # First we sign the message @@ -550,7 +554,7 @@ def handle_connect(self): If more than one is in the list self._network_brokers, try to connect to each in turn until successful. ''' - if self._protocol == 'AMS': + if self._protocol == Ssm2.AMS_MESSAGING: log.debug('handle_connect called for AMS, doing nothing.') return @@ -573,7 +577,7 @@ def handle_disconnect(self): When disconnected, attempt to reconnect using the same method as used when starting up. ''' - if self._protocol == 'AMS': + if self._protocol == Ssm2.AMS_MESSAGING: log.debug('handle_disconnect called for AMS, doing nothing.') return @@ -604,7 +608,7 @@ def start_connection(self): If the timeout is reached without receiving confirmation of connection, raise an exception. ''' - if self._protocol == 'AMS': + if self._protocol == Ssm2.AMS_MESSAGING: log.debug('start_connection called for AMS, doing nothing.') return @@ -640,7 +644,7 @@ def close_connection(self): in a separate thread, so it can outlive the main process if it is not ended. ''' - if self._protocol == 'AMS': + if self._protocol == Ssm2.AMS_MESSAGING: log.debug('close_connection called for AMS, doing nothing.') return From 662d9ebd317ce801e29b519307f940dea3b24e43 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Dec 2018 17:41:07 +0000 Subject: [PATCH 50/79] Merge _pull_msg_rest_ams method into pull_msg_rest Now that the pull_msg_rest method has been simplified, the split with the ..._ams method is a bit unnecessary. --- ssm/ssm2.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index edcdd536..517e2443 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -358,19 +358,16 @@ def _send_msg(self, message, msgid): self._conn.send(to_send, headers=headers) def pull_msg_rest(self): - """Pull a message via HTTPS from self._dest.""" + """Pull 1 message from the AMS and acknowledge it.""" if self._protocol != Ssm2.AMS_MESSAGING: # Then this method should not be called, # raise an exception if it is. raise Ssm2Exception('pull_msg_rest called, ' 'but protocol not set to AMS. ' 'Protocol: %s' % self._protocol) - self._pull_msg_rest_ams() - def _pull_msg_rest_ams(self): - """Pull 1 message from the AMS and acknowledge it.""" # This method is setup so that you could pull down and - # acknowledge more than one method at a time, but + # acknowledge more than one message at a time, but # currently there is no use case for it. messages_to_pull = 1 # ack id's will be stored in this list and then acknowledged From c10f55f573e64652dbc87ea849fcfaa8e4870cbd Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 11 Dec 2018 18:25:15 +0000 Subject: [PATCH 51/79] Add separate token argument to Ssm2 Add separate token argument to Ssm2 to remove overloading of password argument as password and username seem redundant and could be removed at a later stage. --- bin/receiver.py | 4 ++-- bin/sender.py | 4 ++-- ssm/ssm2.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index c9fb0d5f..a25ec000 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -111,7 +111,7 @@ def main(): # Set defaults for STOMP only variables use_ssl = None # Set defaults for AMS only variables - token = "" + token = '' project = None log.info(LOG_BREAK) @@ -213,7 +213,7 @@ def main(): pidfile=pidfile, protocol=protocol, project=project, - password=token) + token=token) log.info('Fetching valid DNs.') dns = get_dns(options.dn_file) diff --git a/bin/sender.py b/bin/sender.py index 5ef82999..0566bf25 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -66,7 +66,7 @@ def main(): # Set defaults for STOMP only variables use_ssl = None # Set defaults for AMS only variables - token = "" + token = '' project = None log.info(LOG_BREAK) @@ -199,7 +199,7 @@ def main(): verify_enc_cert=verify_server_cert, protocol=protocol, project=project, - password=token) + token=token) if sender.has_msgs(): sender.handle_connect() diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 517e2443..c4e6e89c 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -71,7 +71,7 @@ class Ssm2(stomp.ConnectionListener): def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq', - protocol=STOMP_MESSAGING, project=None): + protocol=STOMP_MESSAGING, project=None, token=''): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. @@ -103,10 +103,11 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, # Used when interacting with an Argo Messaging Service self._project = project + self._token = token if self._protocol == Ssm2.AMS_MESSAGING: self._ams = ArgoMessagingService(endpoint=self._brokers[0], - token=self._pwd, + token=self._token, cert=self._cert, key=self._key, project=self._project) From 261da249ca7788f9927599037336f0c0667ab79c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 1 Mar 2019 16:12:59 +0000 Subject: [PATCH 52/79] Add AMS exception to errors caught Rather than aliasing the AMS exception to the STOMP one, which would possibly break things if for some reason you have AMS installed but weren't using it with SSM, add it to the list of exceptions caught. --- bin/receiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index a25ec000..4142d3d1 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -24,7 +24,7 @@ from ssm import __version__, set_up_logging, LOG_BREAK from stomp.exception import NotConnectedException -from argo_ams_library import AmsConnectionException as NotConnectedException +from argo_ams_library import AmsConnectionException import time import logging.config @@ -249,7 +249,7 @@ def main(): if protocol == Ssm2.STOMP_MESSAGING: ssm.send_ping() - except NotConnectedException as error: + except (NotConnectedException, AmsConnectionException) as error: log.warn('Connection lost.') log.debug(error) ssm.shutdown() From a81fd3f0478df9141c521bf16698ddc798df5dea Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 1 Mar 2019 16:19:18 +0000 Subject: [PATCH 53/79] Remove unused AMS import --- ssm/ssm2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index c4e6e89c..06472e46 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -43,11 +43,12 @@ import time import logging -from argo_ams_library import ArgoMessagingService, AmsMessage, AmsException +from argo_ams_library import ArgoMessagingService, AmsMessage # Set up logging log = logging.getLogger(__name__) + class Ssm2Exception(Exception): ''' Exception for use by SSM2. From 021160f991e73220ab758e3ac4dd768c92c0ae14 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 1 Mar 2019 17:15:40 +0000 Subject: [PATCH 54/79] Remove setting of AMS and STOMP defaults manually The variables are either required in the config and are protected by a try...except when fetching the config, or are better set nearer to where they're needed. --- bin/receiver.py | 10 ++-------- bin/sender.py | 10 +++------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index 4142d3d1..a9a6ecaa 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -104,16 +104,10 @@ def main(): print 'Error configuring logging: %s' % str(err) print 'SSM will exit.' sys.exit(1) - + global log log = logging.getLogger('ssmreceive') - # Set defaults for STOMP only variables - use_ssl = None - # Set defaults for AMS only variables - token = '' - project = None - log.info(LOG_BREAK) log.info('Starting receiving SSM version %s.%s.%s.', *__version__) @@ -181,7 +175,7 @@ def main(): project = cp.get('messaging', 'project') except (ConfigParser.Error, ValueError, IOError), err: - # A token and project are needed to successfully send to an + # A token and project are needed to successfully receive from an # AMS instance, so log and then exit on an error. log.error('Error configuring AMS values: %s', err) log.error('SSM will exit.') diff --git a/bin/sender.py b/bin/sender.py index 0566bf25..c182143b 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -60,14 +60,8 @@ def main(): print 'Error configuring logging: %s' % str(err) print 'The system will exit.' sys.exit(1) - - log = logging.getLogger('ssmsend') - # Set defaults for STOMP only variables - use_ssl = None - # Set defaults for AMS only variables - token = '' - project = None + log = logging.getLogger('ssmsend') log.info(LOG_BREAK) log.info('Starting sending SSM version %s.%s.%s.', *__version__) @@ -153,6 +147,8 @@ def main(): # A token is not necessarily needed, if the cert and key can be # used by the underlying auth system to get a suitable token. log.info('No AMS token provided, using cert/key pair instead.') + # Empty string used by AMS to define absence of token. + token = '' if len(brokers) == 0: log.error('No brokers available.') From ef6e459cbc3a52aaff06843867823b3dcd259743 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Fri, 1 Mar 2019 17:31:20 +0000 Subject: [PATCH 55/79] Rename pull_msg_rest to pull_msg_ams Rename pull_msg_rest to pull_msg_ams to match other refactoring --- bin/receiver.py | 4 ++-- ssm/ssm2.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index a9a6ecaa..3f2e0198 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -232,8 +232,8 @@ def main(): time.sleep(1) if protocol == Ssm2.AMS_MESSAGING: # We need to pull down messages as part of - # this loop when using HTTPS. - ssm.pull_msg_rest() + # this loop when using AMS. + ssm.pull_msg_ams() if i % REFRESH_DNS == 0: log.info('Refreshing valid DNs and then sending ping.') diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 06472e46..534df304 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -359,12 +359,12 @@ def _send_msg(self, message, msgid): # If it fails, use the v3 metod signiture self._conn.send(to_send, headers=headers) - def pull_msg_rest(self): + def pull_msg_ams(self): """Pull 1 message from the AMS and acknowledge it.""" if self._protocol != Ssm2.AMS_MESSAGING: # Then this method should not be called, # raise an exception if it is. - raise Ssm2Exception('pull_msg_rest called, ' + raise Ssm2Exception('pull_msg_ams called, ' 'but protocol not set to AMS. ' 'Protocol: %s' % self._protocol) From b7cf4508c8e25fea3cfee153473b6ef9ed9138b4 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 7 Mar 2019 13:43:31 +0000 Subject: [PATCH 56/79] Improve comments and remove host & port examples - Remove referece to destination type in comment about getting protocol as the former is no longer used. - Clarify that host and port are an alterative to bdii and network unless using STOMP in which case it is mandatory. - Remove example hosts and ports as they're different depending on if using AMS or STOMP. - Change network in receiver.cfg to PROD as TEST-NWOB is no more. --- bin/receiver.py | 2 +- conf/receiver.cfg | 11 ++++++----- conf/sender.cfg | 7 ++++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index 3f2e0198..63ab01fd 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -111,7 +111,7 @@ def main(): log.info(LOG_BREAK) log.info('Starting receiving SSM version %s.%s.%s.', *__version__) - # Determine the protocol and destination type of the SSM to configure. + # Determine the protocol for the SSM to use. try: protocol = cp.get('receiver', 'protocol') diff --git a/conf/receiver.cfg b/conf/receiver.cfg index 0d43326b..bca39065 100644 --- a/conf/receiver.cfg +++ b/conf/receiver.cfg @@ -5,12 +5,13 @@ protocol: STOMP [broker] # The SSM will query a BDII to find brokers available. These details are for the -# EGI test broker network +# EGI production broker network bdii: ldap://lcg-bdii.cern.ch:2170 -network: TEST-NWOB -# OR (these details will only be used if the broker network settings aren't used) -#host: test-msg01.afroditi.hellasgrid.gr -#port: 6163 +network: PROD +# Alternatively, 'host' and 'port' can be set manually (with 'bdii' and +# 'network' commented out). This option MUST be used for AMS. +#host: +#port: # broker authentication. If use_ssl is set, the certificates configured # in the mandatory [certificates] section will be used. diff --git a/conf/sender.cfg b/conf/sender.cfg index 62d85da8..4e7698c4 100644 --- a/conf/sender.cfg +++ b/conf/sender.cfg @@ -8,9 +8,10 @@ protocol: STOMP # EGI production broker network bdii: ldap://lcg-bdii.cern.ch:2170 network: PROD -# OR (these details will only be used if the broker network settings aren't used) -#host: broker-prod1.argo.grnet.gr -#port: 6162 +# Alternatively, 'host' and 'port' may be set manually (with 'bdii' and +# 'network' commented out). This option must be used for AMS. +#host: +#port: # broker authentication. If use_ssl is set, the certificates configured # in the mandatory [certificates] section will be used. From f63d29ad2517f1564e85fbeb457fb77c0e9f28c0 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 7 Mar 2019 14:12:59 +0000 Subject: [PATCH 57/79] Update docs to match refactoring Update docs to match refactoring and fix a few small issues. --- README.md | 12 ++++++------ migrating_to_ams_broker.md | 31 ++++++++++++++----------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 97cc0e24..e9d51802 100644 --- a/README.md +++ b/README.md @@ -164,13 +164,13 @@ add your messages using the `add` method. ### Sender (sending via the ARGO Messaging Service (AMS)) * Edit your sender configuration, usually under `/etc/apel/sender.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#sender) with some minor differences: - * There is no need to add the `[SSM Type]` section as it already exists. instead change the `destination type` and `protocol` to `AMS` and `HTTPS` respectively. - * Uncomment `project` and set it to the appropriate project. - * Then run 'ssmsend', SSM will pick up any messages and send them via the ARGO Messaging Service. + * There is no need to add the `[sender]` section as it already exists. Instead change the `protocol` to `AMS`. + * Set `ams_project` to the appropriate project. + * Then run 'ssmsend'. SSM will pick up any messages and send them via the ARGO Messaging Service. ### Sender (container) * Download the example [configuration file](conf/sender.cfg) - * Edit the downloaded `sender.cfg` file as above for sending either via the [EGI message brokers](README.md#sender-sending-via-the-egi-message-brokers) or the [ARGO Messaging Service](https://github.com/gregcorbett/ssm/blob/cert_enabled_ams_support/README.md#sender-sending-via-the-argo-messaging-service-ams). + * Edit the downloaded `sender.cfg` file as above for sending either via the [EGI message brokers](README.md#sender-sending-via-the-egi-message-brokers) or the [ARGO Messaging Service](README.md#sender-sending-via-the-argo-messaging-service-ams). * Run the following docker command to send ``` docker run \ @@ -215,8 +215,8 @@ add your messages using the `add` method. ### Receiver (receiving via the ARGO Messaging Service (AMS)) * Edit your receiver configuration, usually under `/etc/apel/receiver.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#receiver) with some minor differences: - * There is no need to add the `[SSM Type]` section as it already exists. instead change the `destination type` and `protocol` to `AMS` and `HTTPS` respectively. - * Uncomment `project` and set it to the appropriate project. + * There is no need to add the `[receiver]` section as it already exists. Instead change the `protocol` to `AMS`. + * Set `ams_project` to the appropriate project. * Then run your receiver ([as a service](README.md#receiver-service), [as a container](README.md#receiver-container) or [manually](README.md#receiver-manual)) as above. ## Removing the RPM diff --git a/migrating_to_ams_broker.md b/migrating_to_ams_broker.md index 646b1301..4fa06521 100644 --- a/migrating_to_ams_broker.md +++ b/migrating_to_ams_broker.md @@ -4,30 +4,27 @@ Migration requires upgrading to SSM-X.X.X and adding new values to your configur ## Sender -The sender configuration is usually found under `/etc/apel/sender.cfg`. Follow the following steps to migrate. +The sender configuration is usually found under `/etc/apel/sender.cfg`. Follow the steps below to migrate. -1. Comment out `bdii` and `network` -1. Uncomment `host` and set it to `msg-devel.argo.grnet.gr` -1. Add the following as a new section in your configuration. +1. Comment out `bdii` and `network`. +1. Uncomment `host` and set it to `msg-devel.argo.grnet.gr`. +1. Add the following as a new section at the top of your configuration: ``` -# SSM protocol/destination type options -[SSM Type] -# Either 'MQ-BROKER' for EGI Message Brokers or 'AMS' for Argo Messaging Service -destination type: AMS -# Either 'STOMP' for EGI Message Brokers or 'HTTPS' for Argo Messaging Service -protocol: HTTPS +[sender] +# Either 'STOMP' for STOMP message brokers or 'AMS' for Argo Messaging Service +protocol: STOMP ``` -1. Add the following to the `[messaging]` section of your configuration +1. Add the following to the `[messaging]` section of your configuration: ``` -# Project to which SSM will pull messages from. Uncomment and populate for AMS sending -project: EGI-ACCOUNTING +# If using AMS this is the project that SSM will connect to. Ignored for STOMP. +ams_project: EGI-ACCOUNTING ``` -1. To send to the APEL central server, change `destination` to one of the following depending on your type of accounting: +1. To send to the central APEL Accounting server, change `destination` to one of the following depending on your type of accounting: * `gLite-APEL` for Grid Accounting * `eu.egi.cloud.accounting` for Cloud Accounting * `eu.egi.storage.accounting` for Storage Accounting -The next time `ssmsend` runs it should be using the AMS. You can check this by looking in the logs a successful run, which will look like: +The next time `ssmsend` runs it should be using the AMS. You can check this by looking in the logs for a successful run, which should look like this: ``` 2018-09-19 14:18:06,423 - ssmsend - INFO - ======================================== @@ -45,9 +42,9 @@ The next time `ssmsend` runs it should be using the AMS. You can check this by l ## Receiver -1. Follow the steps 1 to 4 from above editing your receiver configuration, usually found under `/etc/apel/receiver.cfg` +1. Follow the steps 1 to 4 as per the [Sender documentation](#Sender) but editing your receiver configuration instead, usually found under `/etc/apel/receiver.cfg`, naming the sction `[receiver]` rather than `[sender]`. 1. Change `destination` to be the subscription you are using to pull messages down. -1. Add your token to the `[messaging]` section of your configuration. +1. Add your token to the `[messaging]` section of your configuration: ``` token: your_token_here ``` From 7a61c0410326f6386811130c4b3d69943be1037b Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 7 Mar 2019 15:35:06 +0000 Subject: [PATCH 58/79] Rename migrating to ams file and update refs Rename migrating_to_ams_broker.md documentation file to remove _broker as we are informed that AMS is not a broker. Update references to this file. --- README.md | 4 ++-- migrating_to_ams_broker.md => migrating_to_ams.md | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename migrating_to_ams_broker.md => migrating_to_ams.md (100%) diff --git a/README.md b/README.md index e9d51802..ed5ae980 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ add your messages using the `add` method. ### Sender (sending via the ARGO Messaging Service (AMS)) - * Edit your sender configuration, usually under `/etc/apel/sender.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#sender) with some minor differences: + * Edit your sender configuration, usually under `/etc/apel/sender.cfg`, as per the [migration instructions](migrating_to_ams.md#sender) with some minor differences: * There is no need to add the `[sender]` section as it already exists. Instead change the `protocol` to `AMS`. * Set `ams_project` to the appropriate project. * Then run 'ssmsend'. SSM will pick up any messages and send them via the ARGO Messaging Service. @@ -214,7 +214,7 @@ add your messages using the `add` method. ### Receiver (receiving via the ARGO Messaging Service (AMS)) - * Edit your receiver configuration, usually under `/etc/apel/receiver.cfg`, as per the [migration instructions](migrating_to_ams_broker.md#receiver) with some minor differences: + * Edit your receiver configuration, usually under `/etc/apel/receiver.cfg`, as per the [migration instructions](migrating_to_ams.md#receiver) with some minor differences: * There is no need to add the `[receiver]` section as it already exists. Instead change the `protocol` to `AMS`. * Set `ams_project` to the appropriate project. * Then run your receiver ([as a service](README.md#receiver-service), [as a container](README.md#receiver-container) or [manually](README.md#receiver-manual)) as above. diff --git a/migrating_to_ams_broker.md b/migrating_to_ams.md similarity index 100% rename from migrating_to_ams_broker.md rename to migrating_to_ams.md From cb6970278674ad1d717f5e118a4383c99a6e3a7d Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 18 Mar 2019 15:42:34 +0000 Subject: [PATCH 59/79] Correct config reference and docs about AMS --- bin/receiver.py | 2 +- bin/sender.py | 2 +- migrating_to_ams.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/receiver.py b/bin/receiver.py index 63ab01fd..a8396b90 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -172,7 +172,7 @@ def main(): # Attempt to configure AMS specific variables. try: token = cp.get('messaging', 'token') - project = cp.get('messaging', 'project') + project = cp.get('messaging', 'ams_project') except (ConfigParser.Error, ValueError, IOError), err: # A token and project are needed to successfully receive from an diff --git a/bin/sender.py b/bin/sender.py index c182143b..6480ea56 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -131,7 +131,7 @@ def main(): # Attempt to configure AMS project variable. try: - project = cp.get('messaging', 'project') + project = cp.get('messaging', 'ams_project') except (ConfigParser.Error, ValueError, IOError), err: # A project is needed to successfully send to an diff --git a/migrating_to_ams.md b/migrating_to_ams.md index 4fa06521..5330e283 100644 --- a/migrating_to_ams.md +++ b/migrating_to_ams.md @@ -12,7 +12,7 @@ The sender configuration is usually found under `/etc/apel/sender.cfg`. Follow t ``` [sender] # Either 'STOMP' for STOMP message brokers or 'AMS' for Argo Messaging Service -protocol: STOMP +protocol: AMS ``` 1. Add the following to the `[messaging]` section of your configuration: ``` From 89ca99d24fd8ecbf9a90ee4fa57c6104237073f4 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 19 Mar 2019 13:58:21 +0000 Subject: [PATCH 60/79] Add setting of use_ssl to a value in AMS setup --- bin/receiver.py | 4 ++++ bin/sender.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/bin/receiver.py b/bin/receiver.py index a8396b90..d04c6044 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -154,6 +154,10 @@ def main(): elif protocol == Ssm2.AMS_MESSAGING: # Then we are setting up an SSM to connect to a AMS. + + # 'use_ssl' isn't checked when using AMS (SSL is always used), but it + # is needed for the call to the Ssm2 constructor below. + use_ssl = None try: # We only need a hostname, not a port host = cp.get('broker', 'host') diff --git a/bin/sender.py b/bin/sender.py index 6480ea56..5f2bcd43 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -114,6 +114,10 @@ def main(): elif protocol == Ssm2.AMS_MESSAGING: # Then we are setting up an SSM to connect to a AMS. + + # 'use_ssl' isn't checked when using AMS (SSL is always used), but it + # is needed for the call to the Ssm2 constructor below. + use_ssl = None try: # We only need a hostname, not a port host = cp.get('broker', 'host') From 5e7b10cb2ca65a0ca28d30b0c02f714bf71f7fc3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 5 Mar 2019 15:45:27 +0000 Subject: [PATCH 61/79] Improve STOMP on_error handling - Remove raising of exception in on_error as it's not the right place to do that and just leaves a mess in the log. - Make the error message clearer if the certificate is not authorised. - Increase log level for these messages to ERROR to match severity. --- ssm/ssm2.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 534df304..90b12b92 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -253,12 +253,15 @@ def on_message(self, headers, body): except (IOError, OSError) as e: log.error('Failed to read or write file: %s', e) - def on_error(self, unused_headers, body): + def on_error(self, headers, body): ''' Called by stomppy when an error frame is received. ''' - log.warn('Error message received: %s', body) - raise Ssm2Exception() + if 'No user for client certificate: ' in headers['message']: + log.error('The following certificate is not authorised: %s', + headers['message'].split(':')[1]) + else: + log.error('Error message received: %s', body) def on_connected(self, unused_headers, unused_body): ''' From 30f146463cf361a60d30c0b78b4bb5646d457184 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 5 Mar 2019 16:45:19 +0000 Subject: [PATCH 62/79] Add log INFO message that records stomp.py version --- ssm/ssm2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 90b12b92..1d9334c5 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -539,6 +539,8 @@ def _initialise_connection(self, host, port): log.warning("SSL connection not requested, your messages may be " "intercepted.") + log.info("Using stomp.py version: %s", stomp.__version__) + # _conn will use the default SSL version specified by stomp.py self._conn = stomp.Connection([(host, port)], use_ssl=self._use_ssl, From 66503754b1d7ac27b8910f80ec49da6a771095ea Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 6 Mar 2019 13:02:35 +0000 Subject: [PATCH 63/79] Disable wait on STOMP connection Disable wait on STOMP connection so that errors during connection do not freeze SSM. --- ssm/ssm2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 1d9334c5..12ffe558 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -621,8 +621,7 @@ def start_connection(self): connection object was initialised.') self._conn.start() - self._conn.connect(wait = True) - + self._conn.connect(wait=False) if self._dest is not None: log.info('Will send messages to: %s', self._dest) From 30004e5ca7d4152b085d8ba49c37d123a495b73e Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 6 Mar 2019 13:13:48 +0000 Subject: [PATCH 64/79] Rearrange start_connection to check conn earlier Rearrange start_connection to check if connected just after connecting so that subscription is not attempted without a connection. --- ssm/ssm2.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 12ffe558..3f144c86 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -622,15 +622,6 @@ def start_connection(self): self._conn.start() self._conn.connect(wait=False) - if self._dest is not None: - log.info('Will send messages to: %s', self._dest) - - if self._listen is not None: - # Use a static ID for the subscription ID because we only ever have - # one subscription within a connection and ID is only considered - # to differentiate subscriptions within a connection. - self._conn.subscribe(destination=self._listen, id=1, ack='auto') - log.info('Subscribing to: %s', self._listen) i = 0 while not self.connected: @@ -641,6 +632,16 @@ def start_connection(self): raise Ssm2Exception(err) i += 1 + if self._dest is not None: + log.info('Will send messages to: %s', self._dest) + + if self._listen is not None: + # Use a static ID for the subscription ID because we only ever have + # one subscription within a connection and ID is only considered + # to differentiate subscriptions within a connection. + self._conn.subscribe(destination=self._listen, id=1, ack='auto') + log.info('Subscribing to: %s', self._listen) + def close_connection(self): ''' Close the connection. This is important because it runs From b6ddb60709098953a01bf4d7b7263d962dbb4de3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 6 Mar 2019 14:04:52 +0000 Subject: [PATCH 65/79] Tweak stomp version logging Tweak stomp version message in log to format it nicely, and move to handle_connect so that it's only printed for each run of SSM rather than once for each broker that's attempted to connect to. --- ssm/ssm2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 3f144c86..c9b6d344 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -539,8 +539,6 @@ def _initialise_connection(self, host, port): log.warning("SSL connection not requested, your messages may be " "intercepted.") - log.info("Using stomp.py version: %s", stomp.__version__) - # _conn will use the default SSL version specified by stomp.py self._conn = stomp.Connection([(host, port)], use_ssl=self._use_ssl, @@ -562,6 +560,8 @@ def handle_connect(self): log.debug('handle_connect called for AMS, doing nothing.') return + log.info("Using stomp.py version %s.%s.%s.", *stomp.__version__) + for host, port in self._brokers: self._initialise_connection(host, port) try: From 01d873bb13c410328d30d5e22ea4dae4838dae52 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 19 Mar 2019 10:48:03 +0000 Subject: [PATCH 66/79] Add argo library to SPEC and setup and tidy --- apel-ssm.spec | 2 +- setup.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index d784bba3..a7ab6a1b 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -21,7 +21,7 @@ BuildArch: noarch BuildRequires: python-devel %endif -Requires: stomppy >= 3.1.1, python-daemon < 2.2.0, python-ldap +Requires: stomppy >= 3.1.1, python-daemon < 2.2.0, python-ldap, argo-ams-library Requires(pre): shadow-utils %define ssmconf %_sysconfdir/apel diff --git a/setup.py b/setup.py index 9d5ea532..15ab89c8 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ Requires setuptools. """ -from os import remove, path, makedirs +from os import remove from shutil import copyfile import sys @@ -50,7 +50,9 @@ def main(): url='http://apel.github.io/', download_url='https://github.com/apel/ssm/releases', license='Apache License, Version 2.0', - install_requires=['stomp.py>=3.1.1', 'python-ldap'], + install_requires=[ + 'stomp.py>=3.1.1', 'python-ldap', 'argo-ams-library', + ], extras_require={ 'python-daemon': ['python-daemon<2.2.0'], 'dirq': ['dirq'], @@ -92,5 +94,6 @@ def main(): remove('conf/apel-ssm') remove('apel-ssm') + if __name__ == "__main__": main() From f27aeba1dedbcf0818b9d8225af004615a3adcc3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 19 Mar 2019 15:21:53 +0000 Subject: [PATCH 67/79] Update version numbers for rc1 --- apel-ssm.spec | 4 ++-- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- ssm/__init__.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index a7ab6a1b..37fc3cb3 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -4,8 +4,8 @@ %endif Name: apel-ssm -Version: 2.3.0 -%define releasenumber 2 +Version: 2.4.0 +%define releasenumber 0.1.rc1 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index e4d0d006..48ad4fd6 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=2.3.0-1 +TAG=2.4.0-0.1.rc1 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index 005a0918..0b40db46 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=2.3.0-1 +VERSION=2.4.0-0.1.rc1 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS diff --git a/ssm/__init__.py b/ssm/__init__.py index c03583bd..bc90c920 100644 --- a/ssm/__init__.py +++ b/ssm/__init__.py @@ -19,7 +19,7 @@ import logging import sys -__version__ = (2, 3, 0) +__version__ = (2, 4, 0) LOG_BREAK = '========================================' From 789e5ba369707d18efd7e7ad209d8c8fd7604f94 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 19 Mar 2019 16:43:46 +0000 Subject: [PATCH 68/79] Re-add setting of AMS defaults if using STOMP Re-add setting of AMS defaults if using STOMP as they're required when the Ssm2 constructor is called --- bin/receiver.py | 4 ++++ bin/sender.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/bin/receiver.py b/bin/receiver.py index d04c6044..42ed11fc 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -124,6 +124,10 @@ def main(): log.info('Setting up SSM with protocol: %s', protocol) if protocol == Ssm2.STOMP_MESSAGING: + # Set defaults for AMS variables that Ssm2 constructor requires below. + project = None + token = '' + # If we can't get a broker to connect to, we have to give up. try: bg = StompBrokerGetter(cp.get('broker', 'bdii')) diff --git a/bin/sender.py b/bin/sender.py index 5f2bcd43..35c32123 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -79,6 +79,10 @@ def main(): log.info('Setting up SSM with protocol: %s', protocol) if protocol == Ssm2.STOMP_MESSAGING: + # Set defaults for AMS variables that Ssm2 constructor requires below. + project = None + token = '' + # If we can't get a broker to connect to, we have to give up. try: bdii_url = cp.get('broker', 'bdii') From 3f98cd0b08ec89c962c416a5440fc0968d235ccc Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 19 Mar 2019 16:47:04 +0000 Subject: [PATCH 69/79] Update version numbers for rc2 --- apel-ssm.spec | 2 +- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index 37fc3cb3..b5f419e6 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -5,7 +5,7 @@ Name: apel-ssm Version: 2.4.0 -%define releasenumber 0.1.rc1 +%define releasenumber 0.2.rc2 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 48ad4fd6..507ca179 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=2.4.0-0.1.rc1 +TAG=2.4.0-0.2.rc2 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index 0b40db46..b10d0c62 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=2.4.0-0.1.rc1 +VERSION=2.4.0-0.2.rc2 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS From 496731999f86825661b76ad6a68dd65bc0076e82 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Wed, 20 Mar 2019 17:07:17 +0000 Subject: [PATCH 70/79] Update version numbers in migration doc --- migrating_to_ams.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrating_to_ams.md b/migrating_to_ams.md index 5330e283..aa7bd8db 100644 --- a/migrating_to_ams.md +++ b/migrating_to_ams.md @@ -1,6 +1,6 @@ # Migrating from using EGI ActiveMQ Message Brokers to using EGI ARGO Messaging Service -Migration requires upgrading to SSM-X.X.X and adding new values to your configuration. +Migration requires upgrading SSM to at least version 2.4.0 and adding new values to your configuration. ## Sender @@ -28,7 +28,7 @@ The next time `ssmsend` runs it should be using the AMS. You can check this by l ``` 2018-09-19 14:18:06,423 - ssmsend - INFO - ======================================== -2018-09-19 14:18:06,424 - ssmsend - INFO - Starting sending SSM version 2.2.1. +2018-09-19 14:18:06,424 - ssmsend - INFO - Starting sending SSM version 2.4.0. 2018-09-19 14:18:06,424 - ssmsend - INFO - Setting up SSM with Dest Type: AMS, Protocol : HTTPS 2018-09-19 14:18:06,424 - ssmsend - INFO - No AMS token provided, using cert/key pair instead. 2018-09-19 14:18:06,424 - ssmsend - INFO - No server certificate supplied. Will not encrypt messages. From c9020bcf36e76c1d5edaaec18d93eabd52eac9a4 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Mar 2019 11:43:43 +0000 Subject: [PATCH 71/79] Add info on AMS to intro and RPM parts of README --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ed5ae980..fcdba2e0 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ [![Maintainability](https://api.codeclimate.com/v1/badges/34aa04f3583afce2ceb2/maintainability)](https://codeclimate.com/github/apel/ssm/maintainability) Secure Stomp Messenger (SSM) is designed to simply send messages -using the STOMP protocol. Messages are signed and may be encrypted -during transit. Persistent queues should be used to guarantee -delivery. +using the STOMP protocol or via the ARGO Messaging Service (AMS). +Messages are signed and may be encrypted during transit. +Persistent queues should be used to guarantee delivery. SSM is written in python. Packages are available for SL5 and SL6. @@ -26,6 +26,8 @@ http://fedoraproject.org/wiki/EPEL The python stomp library (N.B. versions 3.1.1 and above are currently supported) * `yum install stomppy` +The Python AMS library. See here for details on obtaining an RPM: https://github.com/ARGOeu/argo-ams-library/ + The python daemon library (N.B. only versions below 2.2.0 are currently supported) * `yum install python-daemon` From 540702643d7f2efcbe66419b24bebf29e32e4c1f Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Mar 2019 11:55:09 +0000 Subject: [PATCH 72/79] Correct capitalisation and add ref to STOMP --- README.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index fcdba2e0..557b423b 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,19 @@ -# Secure Stomp Messenger +# Secure STOMP Messenger [![Build Status](https://travis-ci.org/apel/ssm.svg?branch=dev)](https://travis-ci.org/apel/ssm) [![Coverage Status](https://coveralls.io/repos/github/apel/ssm/badge.svg?branch=dev)](https://coveralls.io/github/apel/ssm?branch=dev) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/cc3e808664ee41638938aa5c660a88ae)](https://www.codacy.com/app/apel/ssm) [![Maintainability](https://api.codeclimate.com/v1/badges/34aa04f3583afce2ceb2/maintainability)](https://codeclimate.com/github/apel/ssm/maintainability) -Secure Stomp Messenger (SSM) is designed to simply send messages -using the STOMP protocol or via the ARGO Messaging Service (AMS). +Secure STOMP Messenger (SSM) is designed to simply send messages +using the [STOMP protocol](http://stomp.github.io/) or via the ARGO Messaging Service (AMS). Messages are signed and may be encrypted during transit. Persistent queues should be used to guarantee delivery. -SSM is written in python. Packages are available for SL5 and SL6. +SSM is written in Python. Packages are available for RHEL 6 and 7, and + Ubuntu Trusty. -For more about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL/SSM). +For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL/SSM). ## Installing the RPM @@ -23,18 +24,18 @@ The EPEL repository must be enabled. This can be done by installing the RPM for your version of SL, which is available on this page: http://fedoraproject.org/wiki/EPEL -The python stomp library (N.B. versions 3.1.1 and above are currently supported) +The Python STOMP library (N.B. versions 3.1.1 and above are currently supported) * `yum install stomppy` The Python AMS library. See here for details on obtaining an RPM: https://github.com/ARGOeu/argo-ams-library/ -The python daemon library (N.B. only versions below 2.2.0 are currently supported) +The Python daemon library (N.B. only versions below 2.2.0 are currently supported) * `yum install python-daemon` -The python ldap library +The Python ldap library * `yum install python-ldap` -Optionally, the python dirq library (N.B. this is only required if your messages +Optionally, the Python dirq library (N.B. this is only required if your messages are stored in a dirq structure) * `yum install python-dirq` @@ -71,7 +72,7 @@ successfully. The RPM carries out a number of steps to run the SSM in a specific way. -1. It installs the core files in the python libraries directory +1. It installs the core files in the Python libraries directory 2. It installs scripts in /usr/bin 3. It installs configuration files in /etc/apel 4. It creates the messages directory /var/spool/apel/ @@ -88,7 +89,7 @@ Install APEL SSM: Install any missing system packages needed for the SSM: * `apt-get -f install` -Install any missing python requirements that don't have system packages: +Install any missing Python requirements that don't have system packages: * `pip install "stomp.py>=3.1.1" dirq` If you wish to run the SSM as a receiver, you will also need to install the python-daemon system package: @@ -98,7 +99,7 @@ If you wish to run the SSM as a receiver, you will also need to install the pyth The DEB carries out a number of steps to run the SSM in a specific way. -1. It installs the core files in the python libraries directory +1. It installs the core files in the Python libraries directory 2. It installs scripts in /usr/bin 3. It installs configuration files in /etc/apel 4. It creates the messages directory /var/spool/apel/ @@ -117,7 +118,7 @@ Ensure that the apel user running the SSM has access to the following: * `chown apel:apel /var/run/apel` The configuration files are in /etc/apel/. The default -configuration will send messages to the test apel server. +configuration will send messages to the test APEL server. ## Adding Files @@ -142,9 +143,9 @@ there are no restrictions on the file names used. ### Programmatic #### With the dirq module -Use the python or perl dirq libraries: - * python: http://pypi.python.org/pypi/dirq - * perl: http://search.cpan.org/~lcons/Directory-Queue/ +Use the Python or Perl dirq libraries: + * Python: http://pypi.python.org/pypi/dirq + * Perl: http://search.cpan.org/~lcons/Directory-Queue/ Create a QueueSimple object with path /var/spool/apel/outgoing/ and add your messages. From c389e5319ec078a54a8ef7f9f15983a0bb5a14a3 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Mar 2019 12:54:24 +0000 Subject: [PATCH 73/79] Update version numbers for rc3 --- apel-ssm.spec | 2 +- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index b5f419e6..d8f89c06 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -5,7 +5,7 @@ Name: apel-ssm Version: 2.4.0 -%define releasenumber 0.2.rc2 +%define releasenumber 0.3.rc3 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 507ca179..74bfe686 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=2.4.0-0.2.rc2 +TAG=2.4.0-0.3.rc3 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index b10d0c62..b84a2394 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=2.4.0-0.2.rc2 +VERSION=2.4.0-0.3.rc3 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS From 1a9f10c674a5623948b80935c91bc4accd782f2a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 21 Mar 2019 16:31:42 +0000 Subject: [PATCH 74/79] Update changelogs --- CHANGELOG | 8 ++++++++ apel-ssm.spec | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 1bd5e6f9..0f03c09c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ Changelog for ssm ================= +* Fri Mar 22 2019 Adrian Coveney - 2.4.0-1 + - Added support for sending and receiving messages using the ARGO Messaging + Service (AMS). + - Added option to send messages from a directory without needing to conform to + the file naming convention that the dirq module requires. + - Fixed SSM hanging if certificate is not authorised with the broker. Now it + will try other brokers if available and then correctly shut down. + * Wed Nov 28 2018 Adrian Coveney - 2.3.0-2 - Updated build and test files only. diff --git a/apel-ssm.spec b/apel-ssm.spec index d8f89c06..96e6fd86 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -100,6 +100,14 @@ rm -rf $RPM_BUILD_ROOT %doc %_defaultdocdir/%{name} %changelog +* Fri Mar 22 2019 Adrian Coveney - 2.4.0-1 + - Added support for sending and receiving messages using the ARGO Messaging + Service (AMS). + - Added option to send messages from a directory without needing to conform to + the file naming convention that the dirq module requires. + - Fixed SSM hanging if certificate is not authorised with the broker. Now it + will try other brokers if available and then correctly shut down. + * Wed Nov 28 2018 Adrian Coveney - 2.3.0-2 - Updated build and test files only. From 6d3f00a38deb33f39d977c8a432dd047ad44326c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 21 May 2019 14:25:23 +0100 Subject: [PATCH 75/79] OpenSSL checkend fix (#97) - Add missing seconds argument after '-checkend' as it should be followed by a number that defines how many seconds ahead openssl x509 should check for cert expiry, otherwise later versions of OpenSSL raise syntax errors (treating next arg as the number). Set it to a day as certs should really be replaced before that considering the amount of time it can take a message to traverse the messaging system. - Update error messages to match verify_cert_date as verify_cert_date has been updated to check that the cert won't expire within the next day, the error messages should explain this if there's a verification failure - Increase the temp cert validity to 2 days so that it's comfortably longer than the period that verify_cert_date checks for (1 day). - Update expected error for expired cert test to match updated function. --- ssm/crypto.py | 9 +++++++-- ssm/ssm2.py | 9 +++++---- test/test_ssm.py | 10 ++++++---- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/ssm/crypto.py b/ssm/crypto.py index 7fe6f176..f3b88ac0 100644 --- a/ssm/crypto.py +++ b/ssm/crypto.py @@ -214,11 +214,16 @@ def decrypt(encrypted_text, certpath, keypath): def verify_cert_date(certpath): - """Return True if certifcate is 'in date', otherwise return False.""" + """Check that certificate hasn't expired and won't expire within 24 hours. + + Return True if certifcate is 'in date', otherwise return False. + """ if certpath is None: raise CryptoException('Invalid None argument to verify_cert_date().') - args = ['openssl', 'x509', '-checkend', '-noout', '-in', certpath] + # Check if the certificate expires within the next 86400 seconds and exit + # non-zero if yes, it will expire, or zero if not. + args = ['openssl', 'x509', '-checkend', '86400', '-noout', '-in', certpath] p1 = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index c9b6d344..28e9762b 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -148,7 +148,8 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, # Check that the certificate has not expired. if not crypto.verify_cert_date(self._cert): - raise Ssm2Exception('Certificate %s has expired.' % self._cert) + raise Ssm2Exception('Certificate %s has expired or will expire ' + 'within a day.' % self._cert) # check the server certificate provided if enc_cert is not None: @@ -158,9 +159,9 @@ def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, # Check that the encyption certificate has not expired. if not crypto.verify_cert_date(enc_cert): raise Ssm2Exception( - 'Encryption certificate %s has expired. Please obtain the ' - 'new one from the final server receiving your messages.' % - enc_cert + 'Encryption certificate %s has expired or will expire ' + 'within a day. Please obtain the new one from the final ' + 'server receiving your messages.' % enc_cert ) if verify_enc_cert: if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls): diff --git a/test/test_ssm.py b/test/test_ssm.py index 79d1c2a3..896adba6 100644 --- a/test/test_ssm.py +++ b/test/test_ssm.py @@ -44,7 +44,8 @@ def setUp(self): # The subject has been hardcoded so that the generated # certificate subject matches the subject of the hardcoded, # expired, certificate at the bottom of this file. - call(['openssl', 'req', '-x509', '-nodes', '-days', '1', '-new', + # 2 days used so that verify_cert_date doesn't think it expires soon. + call(['openssl', 'req', '-x509', '-nodes', '-days', '2', '-new', '-key', self._key_path, '-out', TEST_CERT_FILE, '-subj', '/C=UK/O=STFC/OU=SC/CN=Test Cert']) @@ -66,7 +67,7 @@ def tearDown(self): except OSError, e: print 'Error removing temporary directory %s' % self._tmp_dir print e - + def test_on_message(self): ''' This is quite a complicated method, so it would take a long time @@ -95,8 +96,8 @@ def test_on_message(self): def test_init_expired_cert(self): """Test right exception is thrown creating an SSM with expired cert.""" - expected_error = ('Certificate %s has expired.' - % self._expired_cert_path) + expected_error = ('Certificate %s has expired or will expire ' + 'within a day.' % self._expired_cert_path) try: # Indirectly test crypto.verify_cert_date Ssm2(self._brokers, self._msgdir, self._expired_cert_path, @@ -134,6 +135,7 @@ def test_ssm_init_non_dirq(self): # Assert the outbound queue is of the expected type. self.assertTrue(isinstance(ssm._outq, MessageDirectory)) + TEST_CERT_FILE = '/tmp/test.crt' # As we want the expired certifcate to match the key used, we can't From 0b0bf8893079c2032db5fa05e6b3b303e861a0fa Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 21 May 2019 14:32:38 +0100 Subject: [PATCH 76/79] Update version numbers for rc4 --- apel-ssm.spec | 2 +- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index 96e6fd86..8a799c68 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -5,7 +5,7 @@ Name: apel-ssm Version: 2.4.0 -%define releasenumber 0.3.rc3 +%define releasenumber 0.4.rc4 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 74bfe686..1099a088 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=2.4.0-0.3.rc3 +TAG=2.4.0-0.4.rc4 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index b84a2394..500d7a7a 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=2.4.0-0.3.rc3 +VERSION=2.4.0-0.4.rc4 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS From 006f5268b1058aa001b9f6613ddf8988cbb52a1c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 13 Jun 2019 16:13:37 +0100 Subject: [PATCH 77/79] Update .travis.yml --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8633d739..d6f1f475 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,9 @@ matrix: - python: "nightly" fast_finish: true +# Pin Ubuntu to Trusty for the moment for Python 2.6 support +dist: trusty + # Cache the dependencies installed by pip cache: pip # Avoid pip log from affecting cache From a8ea4a0d36eee7600e6ce45aadc1d32cfd73b142 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 1 Aug 2019 09:52:38 +0100 Subject: [PATCH 78/79] Update changelog and version number for release --- CHANGELOG | 3 ++- apel-ssm.spec | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0f03c09c..05176934 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,12 +1,13 @@ Changelog for ssm ================= -* Fri Mar 22 2019 Adrian Coveney - 2.4.0-1 +* Thu Aug 01 2019 Adrian Coveney - 2.4.0-1 - Added support for sending and receiving messages using the ARGO Messaging Service (AMS). - Added option to send messages from a directory without needing to conform to the file naming convention that the dirq module requires. - Fixed SSM hanging if certificate is not authorised with the broker. Now it will try other brokers if available and then correctly shut down. + - Fixed an OpenSSL 1.1 syntax error by including missing argument to checkend. * Wed Nov 28 2018 Adrian Coveney - 2.3.0-2 - Updated build and test files only. diff --git a/apel-ssm.spec b/apel-ssm.spec index 8a799c68..a452b581 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -5,7 +5,7 @@ Name: apel-ssm Version: 2.4.0 -%define releasenumber 0.4.rc4 +%define releasenumber 1 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger @@ -100,13 +100,14 @@ rm -rf $RPM_BUILD_ROOT %doc %_defaultdocdir/%{name} %changelog -* Fri Mar 22 2019 Adrian Coveney - 2.4.0-1 +* Thu Aug 01 2019 Adrian Coveney - 2.4.0-1 - Added support for sending and receiving messages using the ARGO Messaging Service (AMS). - Added option to send messages from a directory without needing to conform to the file naming convention that the dirq module requires. - Fixed SSM hanging if certificate is not authorised with the broker. Now it will try other brokers if available and then correctly shut down. + - Fixed an OpenSSL 1.1 syntax error by including missing argument to checkend. * Wed Nov 28 2018 Adrian Coveney - 2.3.0-2 - Updated build and test files only. From 313b51e93533b9de472f88d68e5f62f5ddec515d Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Thu, 1 Aug 2019 09:56:34 +0100 Subject: [PATCH 79/79] Update version numbers in build scripts --- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 1099a088..4eb15210 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=2.4.0-0.4.rc4 +TAG=2.4.0-1 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index 500d7a7a..a892b88c 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=2.4.0-0.4.rc4 +VERSION=2.4.0-1 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS