From 564a20621d141f8e0c00b29254375e57ab5daff6 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 10:28:52 +0530 Subject: [PATCH 01/33] PYM-2: - The issue listed was due to wrong messages being passed by the user. Upon passing the right messages codes, the parser works as expected on all counts. - There is also changes that make the parser compatible with python2 and python3 - Verifier that the tool works on both python3 and python2 for all MODBUS message codes on TCP, RTU, and BIN --- examples/contrib/message-parser.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/examples/contrib/message-parser.py b/examples/contrib/message-parser.py index b5c653bf3..9e9c12d09 100755 --- a/examples/contrib/message-parser.py +++ b/examples/contrib/message-parser.py @@ -11,14 +11,17 @@ * rtu * binary ''' -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# # import needed libraries #---------------------------------------------------------------------------# from __future__ import print_function +import six import sys import collections import textwrap from optparse import OptionParser +import codecs as c + from pymodbus.utilities import computeCRC, computeLRC from pymodbus.factory import ClientDecoder, ServerDecoder from pymodbus.transaction import ModbusSocketFramer @@ -33,9 +36,9 @@ modbus_log = logging.getLogger("pymodbus") -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# # build a quick wrapper around the framers -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# class Decoder(object): def __init__(self, framer, encode=False): @@ -52,7 +55,10 @@ def decode(self, message): :param message: The messge to decode ''' - value = message if self.encode else message.encode('hex') + if six.PY3: + value = message if self.encode else c.encode(message, 'hex_codec') + else: + value = message if self.encode else message.encode('hex') print("="*80) print("Decoding Message %s" % value) print("="*80) @@ -64,7 +70,7 @@ def decode(self, message): print("%s" % decoder.decoder.__class__.__name__) print("-"*80) try: - decoder.addToFrame(message.encode()) + decoder.addToFrame(message) if decoder.checkFrame(): decoder.advanceFrame() decoder.processIncomingPacket(message, self.report) @@ -86,7 +92,7 @@ def report(self, message): :param message: The message to print ''' print("%-15s = %s" % ('name', message.__class__.__name__)) - for k,v in message.__dict__.iteritems(): + for (k, v) in message.__dict__.items(): if isinstance(v, dict): print("%-15s =" % k) for kk,vv in v.items(): @@ -102,9 +108,9 @@ def report(self, message): print("%-15s = %s" % ('documentation', message.__doc__)) -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# # and decode our message -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# def get_options(): ''' A helper method to parse the command line options @@ -151,7 +157,10 @@ def get_messages(option): ''' if option.message: if not option.ascii: - option.message = option.message.decode('hex') + if not six.PY3: + option.message = option.message.decode('hex') + else: + option.message = c.decode(option.message.encode(), 'hex_codec') yield option.message elif option.file: with open(option.file, "r") as handle: From da155a879d7ce7f75738d007a814b5ea6e6e5113 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 14:02:02 +0530 Subject: [PATCH 02/33] PYM-2: Checking the incoming framer. If te Framer is a Binary Framer, we take the unit address as the second incoming bite as opposed to the first bite. This is was done while fixing the message parsers for binary messages --- pymodbus/server/sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index f4beeea91..e52822260 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -102,7 +102,10 @@ def handle(self): if data: if _logger.isEnabledFor(logging.DEBUG): _logger.debug(" ".join([hex(byte2int(x)) for x in data])) - unit_address = byte2int(data[0]) + if not isinstance(self.framer, ModbusBinaryFramer): + unit_address = byte2int(data[0]) + else: + unit_address = byte2int(data[1]) if unit_address in self.server.context: self.framer.processIncomingPacket(data, self.execute) except Exception as msg: From a7fbe8c6fbb25c96d46cf70cfc0b6c1565159164 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 14:06:18 +0530 Subject: [PATCH 03/33] PYM-2: Changed the modbus binary header size from 2 to 1. According to the docs: Modbus Binary Frame Controller:: [ Start ][Address ][ Function ][ Data ][ CRC ][ End ] 1b 1b 1b Nb 2b 1b --- pymodbus/transaction.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymodbus/transaction.py b/pymodbus/transaction.py index 1efd3c17f..cc44be438 100644 --- a/pymodbus/transaction.py +++ b/pymodbus/transaction.py @@ -461,7 +461,7 @@ def processIncomingPacket(self, data, callback): def _process(self, callback, error=False): """ - Process incoming packets irrespective error condition + Process incoming packets irrespective error condition """ data = self.getRawFrame() if error else self.getFrame() result = self.decoder.decode(data) @@ -487,7 +487,7 @@ def resetFrame(self): def getRawFrame(self): """ - Returns the complete buffer + Returns the complete buffer """ return self.__buffer @@ -922,7 +922,7 @@ def __init__(self, decoder): ''' self.__buffer = b'' self.__header = {'crc':0x0000, 'len':0, 'uid':0x00} - self.__hsize = 0x02 + self.__hsize = 0x01 self.__start = b'\x7b' # { self.__end = b'\x7d' # } self.__repeat = [b'}'[0], b'{'[0]] # python3 hack From 63d3024efa2c934599e3782d61e2c384f8025f70 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 14:58:54 +0530 Subject: [PATCH 04/33] PYM-3: Script is now compatible with both python2 and python3 --- examples/contrib/message-generator.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/contrib/message-generator.py b/examples/contrib/message-generator.py index b9a1e8f0a..c6a5de68d 100755 --- a/examples/contrib/message-generator.py +++ b/examples/contrib/message-generator.py @@ -12,6 +12,8 @@ * binary - `./generate-messages.py -f binary -m tx -b` ''' from optparse import OptionParser +import codecs as c +import six #--------------------------------------------------------------------------# # import all the available framers #--------------------------------------------------------------------------# @@ -51,17 +53,17 @@ WriteSingleRegisterRequest, WriteSingleCoilRequest, ReadWriteMultipleRegistersRequest, - + ReadExceptionStatusRequest, GetCommEventCounterRequest, GetCommEventLogRequest, ReportSlaveIdRequest, - + ReadFileRecordRequest, WriteFileRecordRequest, MaskWriteRegisterRequest, ReadFifoQueueRequest, - + ReadDeviceInformationRequest, ReturnQueryDataRequest, @@ -97,7 +99,7 @@ WriteSingleRegisterResponse, WriteSingleCoilResponse, ReadWriteMultipleRegistersResponse, - + ReadExceptionStatusResponse, GetCommEventCounterResponse, GetCommEventLogResponse, @@ -149,13 +151,13 @@ 'write_registers' : [0x01] * 8, 'transaction' : 0x01, 'protocol' : 0x00, - 'unit' : 0x01, + 'unit' : 0xff, } -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# # generate all the requested messages -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# def generate_messages(framer, options): ''' A helper method to parse the command line options @@ -168,13 +170,16 @@ def generate_messages(framer, options): print ("%-44s = " % message.__class__.__name__) packet = framer.buildPacket(message) if not options.ascii: - packet = packet.encode('hex') + '\n' - print (packet) # because ascii ends with a \r\n + if not six.PY3: + packet = packet.encode('hex') + else: + packet = c.encode(packet, 'hex_codec').decode('utf-8') + print ("{}\n".format(packet)) # because ascii ends with a \r\n -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# # initialize our program settings -#---------------------------------------------------------------------------# +#---------------------------------------------------------------------------# def get_options(): ''' A helper method to parse the command line options From df9e74e7b72c7f463239d9c00627dbad24809747 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 15:40:41 +0530 Subject: [PATCH 05/33] WIP --- examples/contrib/message-parser.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/contrib/message-parser.py b/examples/contrib/message-parser.py index 9e9c12d09..5a08bdbad 100755 --- a/examples/contrib/message-parser.py +++ b/examples/contrib/message-parser.py @@ -28,6 +28,8 @@ from pymodbus.transaction import ModbusBinaryFramer from pymodbus.transaction import ModbusAsciiFramer from pymodbus.transaction import ModbusRtuFramer +from pymodbus.compat import byte2int, int2byte + #--------------------------------------------------------------------------# # Logging @@ -142,6 +144,10 @@ def get_options(): help="The file containing messages to parse", dest="file", default=None) + parser.add_option("-t", "--transaction", + help="If the incoming message is in hexadecimal format", + action="store_true", dest="transaction", default=False) + (opt, arg) = parser.parse_args() if not opt.message and len(arg) > 0: @@ -156,6 +162,12 @@ def get_messages(option): :returns: The message iterator to parse ''' if option.message: + if option.transaction: + # option.message = "".join(int2byte(int(x, 16)) for x in option.message.split()) + # option.message = "".join(x.replace("0x", "") for x in option.message.split()) + seg = option.message.split() + import pdb; pdb.set_trace() + if not option.ascii: if not six.PY3: option.message = option.message.decode('hex') From dbabc9ee122837ff29eabea63232429b2affb426 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 16:00:23 +0530 Subject: [PATCH 06/33] PYM-2: Added a new switch: -t --transaction This switch is meant to be used when we wish to parse messages directly from the logs of Modbus. The format of a message as shown in the logs is like bellow: 0x7b 0x1 0x5 0x0 0x0 0xff 0x0 0x8c 0x3a 0x7d We can pass this as the message to the parser along with the -t witch to convert it into a compatible message to be parsed. EG: (modbus3) [~/pymodbus/examples/contrib]$ ./message-parser.py -b -t -p binary -m "0x7b 0x1 0x5 0x0 0x0 0xff 0x0 0x8c 0x3a 0x7d" ================================================================================ Decoding Message b'7b01050000ff008c3a7d' ================================================================================ ServerDecoder -------------------------------------------------------------------------------- name = WriteSingleCoilRequest transaction_id = 0x0 protocol_id = 0x0 unit_id = . [1] skip_encode = 0x0 check = 0x0 address = 0x0 value = 0x1 documentation = This function code is used to write a single output to either ON or OFF in a remote device. The requested ON/OFF state is specified by a constant in the request data field. A value of FF 00 hex requests the output to be ON. A value of 00 00 requests it to be OFF. All other values are illegal and will not affect the output. The Request PDU specifies the address of the coil to be forced. Coils are addressed starting at zero. Therefore coil numbered 1 is addressed as 0. The requested ON/OFF state is specified by a constant in the Coil Value field. A value of 0XFF00 requests the coil to be ON. A value of 0X0000 requests the coil to be off. All other values are illegal and will not affect the coil. ClientDecoder -------------------------------------------------------------------------------- name = WriteSingleCoilResponse transaction_id = 0x0 protocol_id = 0x0 unit_id = . [1] skip_encode = 0x0 check = 0x0 address = 0x0 value = 0x1 documentation = The normal response is an echo of the request, returned after the coil state has been written. --- examples/contrib/message-parser.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/contrib/message-parser.py b/examples/contrib/message-parser.py index 5a08bdbad..6e1576556 100755 --- a/examples/contrib/message-parser.py +++ b/examples/contrib/message-parser.py @@ -163,10 +163,12 @@ def get_messages(option): ''' if option.message: if option.transaction: - # option.message = "".join(int2byte(int(x, 16)) for x in option.message.split()) - # option.message = "".join(x.replace("0x", "") for x in option.message.split()) - seg = option.message.split() - import pdb; pdb.set_trace() + msg = "" + for segment in option.message.split(): + segment = segment.replace("0x", "") + segment = "0" + segment if len(segment) == 1 else segment + msg = msg + segment + option.message = msg if not option.ascii: if not six.PY3: From 673de546e683918e7d0ceb19862e80edffc69d55 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 19:51:00 +0530 Subject: [PATCH 07/33] PYM-2: Removing additional dependancy and making use of existing porting tools --- examples/contrib/message-parser.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/contrib/message-parser.py b/examples/contrib/message-parser.py index 6e1576556..be8fc8b42 100755 --- a/examples/contrib/message-parser.py +++ b/examples/contrib/message-parser.py @@ -15,7 +15,6 @@ # import needed libraries #---------------------------------------------------------------------------# from __future__ import print_function -import six import sys import collections import textwrap @@ -28,7 +27,7 @@ from pymodbus.transaction import ModbusBinaryFramer from pymodbus.transaction import ModbusAsciiFramer from pymodbus.transaction import ModbusRtuFramer -from pymodbus.compat import byte2int, int2byte +from pymodbus.compat import byte2int, int2byte, IS_PYTHON3 #--------------------------------------------------------------------------# @@ -57,7 +56,7 @@ def decode(self, message): :param message: The messge to decode ''' - if six.PY3: + if IS_PYTHON3: value = message if self.encode else c.encode(message, 'hex_codec') else: value = message if self.encode else message.encode('hex') @@ -171,7 +170,7 @@ def get_messages(option): option.message = msg if not option.ascii: - if not six.PY3: + if not IS_PYTHON3: option.message = option.message.decode('hex') else: option.message = c.decode(option.message.encode(), 'hex_codec') From 8dc6dde281bb964bb443a99d1a4008e632df8f00 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 20 Nov 2017 19:52:42 +0530 Subject: [PATCH 08/33] PYM-3: Removing additional dependancy and making use of existing porting tools --- examples/contrib/message-generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/contrib/message-generator.py b/examples/contrib/message-generator.py index c6a5de68d..51146434b 100755 --- a/examples/contrib/message-generator.py +++ b/examples/contrib/message-generator.py @@ -13,7 +13,6 @@ ''' from optparse import OptionParser import codecs as c -import six #--------------------------------------------------------------------------# # import all the available framers #--------------------------------------------------------------------------# @@ -32,6 +31,7 @@ from pymodbus.mei_message import * from pymodbus.register_read_message import * from pymodbus.register_write_message import * +from pymodbus.compat import IS_PYTHON3 #--------------------------------------------------------------------------# # initialize logging @@ -170,7 +170,7 @@ def generate_messages(framer, options): print ("%-44s = " % message.__class__.__name__) packet = framer.buildPacket(message) if not options.ascii: - if not six.PY3: + if not IS_PYTHON3: packet = packet.encode('hex') else: packet = c.encode(packet, 'hex_codec').decode('utf-8') From 9c0344885994db555c4a9d9c343e24d8f06d1e02 Mon Sep 17 00:00:00 2001 From: sanjay Date: Tue, 21 Nov 2017 03:30:55 +0000 Subject: [PATCH 09/33] Initial Bitbucket Pipelines configuration --- bitbucket-pipelines.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 bitbucket-pipelines.yml diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 000000000..3d6bebcd4 --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,16 @@ +# This is a sample build configuration for Python. +# Check our guides at https://confluence.atlassian.com/x/x4UWN for more examples. +# Only use spaces to indent your .yml configuration. +# ----- +# You can specify a custom docker image from Docker Hub as your build environment. +image: python:3.5.1 + +pipelines: + default: + - step: + caches: + - pip + script: # Modify the commands below to build your repository. + - pip install --requirement=requirements-checks.txt + - pip install --requirement=requirements-tests.txt + - make test \ No newline at end of file From e05fd3f153bf84d53ed13f59367abb3b86e83acc Mon Sep 17 00:00:00 2001 From: sanjay Date: Tue, 21 Nov 2017 03:34:15 +0000 Subject: [PATCH 10/33] bitbucket-pipelines.yml edited online with Bitbucket --- bitbucket-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 3d6bebcd4..5661c8fc6 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -11,6 +11,7 @@ pipelines: caches: - pip script: # Modify the commands below to build your repository. + - scripts/travis.sh - pip install --requirement=requirements-checks.txt - pip install --requirement=requirements-tests.txt - make test \ No newline at end of file From 723109db31ac9f4a5ddbeb63abf5c65c37845fb9 Mon Sep 17 00:00:00 2001 From: sanjay Date: Tue, 21 Nov 2017 03:43:04 +0000 Subject: [PATCH 11/33] bitbucket-pipelines.yml edited online with Bitbucket --- bitbucket-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 5661c8fc6..e6359056d 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -11,6 +11,7 @@ pipelines: caches: - pip script: # Modify the commands below to build your repository. + - pip install virtualenv - scripts/travis.sh - pip install --requirement=requirements-checks.txt - pip install --requirement=requirements-tests.txt From ec936f773de0d809205cdf090ca8a3ee582f224a Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 09:20:08 +0530 Subject: [PATCH 12/33] PYM-2: Updated the transaction tests for BinaryFramerTransaction. The header for Binary trasaction is of size 1. This was recrtified earlier commits of this branch. The test ensure these changes --- test/test_transaction.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_transaction.py b/test/test_transaction.py index 7a90ed165..8c5e18f47 100644 --- a/test/test_transaction.py +++ b/test/test_transaction.py @@ -32,9 +32,9 @@ def tearDown(self): del self._rtu del self._ascii - #---------------------------------------------------------------------------# + #---------------------------------------------------------------------------# # Dictionary based transaction manager - #---------------------------------------------------------------------------# + #---------------------------------------------------------------------------# def testDictTransactionManagerTID(self): ''' Test the dict transaction manager TID ''' for tid in range(1, self._manager.getNextTID() + 10): @@ -65,9 +65,9 @@ class Request: pass self._manager.delTransaction(handle.transaction_id) self.assertEqual(None, self._manager.getTransaction(handle.transaction_id)) - #---------------------------------------------------------------------------# + #---------------------------------------------------------------------------# # Queue based transaction manager - #---------------------------------------------------------------------------# + #---------------------------------------------------------------------------# def testFifoTransactionManagerTID(self): ''' Test the fifo transaction manager TID ''' for tid in range(1, self._queue_manager.getNextTID() + 10): @@ -98,7 +98,7 @@ class Request: pass self._queue_manager.delTransaction(handle.transaction_id) self.assertEqual(None, self._queue_manager.getTransaction(handle.transaction_id)) - #---------------------------------------------------------------------------# + #---------------------------------------------------------------------------# # TCP tests #---------------------------------------------------------------------------# def testTCPFramerTransactionReady(self): @@ -361,7 +361,7 @@ def testBinaryFramerTransactionReady(self): def testBinaryFramerTransactionFull(self): ''' Test a full binary frame transaction ''' msg = b'\x7b\x01\x03\x00\x00\x00\x05\x85\xC9\x7d' - pack = msg[3:-3] + pack = msg[2:-3] self._binary.addToFrame(msg) self.assertTrue(self._binary.checkFrame()) result = self._binary.getFrame() @@ -372,7 +372,7 @@ def testBinaryFramerTransactionHalf(self): ''' Test a half completed binary frame transaction ''' msg1 = b'\x7b\x01\x03\x00' msg2 = b'\x00\x00\x05\x85\xC9\x7d' - pack = msg1[3:] + msg2[:-3] + pack = msg1[2:] + msg2[:-3] self._binary.addToFrame(msg1) self.assertFalse(self._binary.checkFrame()) result = self._binary.getFrame() From e2c7b1df0669e755d69b6234f2d0abd970b640f1 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 10:25:16 +0530 Subject: [PATCH 13/33] PYM-6: Minor Cleanup task Removing the argument handler in TCP Syncronous server. This argument is not used any where. --- pymodbus/server/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index e52822260..425c2bf90 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -257,7 +257,7 @@ class ModbusTcpServer(socketserver.ThreadingTCPServer): server context instance. ''' - def __init__(self, context, framer=None, identity=None, address=None, handler=None, **kwargs): + def __init__(self, context, framer=None, identity=None, address=None, **kwargs): ''' Overloaded initializer for the socket server If the identify structure is not passed in, the ModbusControlBlock From 85a628d63cdb41d37728c6b15b87033fea894255 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 10:47:56 +0530 Subject: [PATCH 14/33] PYM-6: ModbusUdpServer and ModbusTcpServer will now accept any legal Modbus request handler. The request handler being passed will have to be of instance ModbusBaseRequestHandler. The default request handler is ModbusDisconnectedRequestHandler. I.e., is no handler is passed, or if the handler is not of type ModbusBaseRequestHandler, ModbusDisconnectedRequestHandler will be made use of. --- pymodbus/server/sync.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index 425c2bf90..d67d0d183 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -257,7 +257,7 @@ class ModbusTcpServer(socketserver.ThreadingTCPServer): server context instance. ''' - def __init__(self, context, framer=None, identity=None, address=None, **kwargs): + def __init__(self, context, framer=None, identity=None, address=None, handler=None, **kwargs): ''' Overloaded initializer for the socket server If the identify structure is not passed in, the ModbusControlBlock @@ -276,13 +276,14 @@ def __init__(self, context, framer=None, identity=None, address=None, **kwargs): self.context = context or ModbusServerContext() self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) + self.handler = handler if isinstance(handler, ModbusBaseRequestHandler) else ModbusConnectedRequestHandler self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) socketserver.ThreadingTCPServer.__init__(self, - self.address, ModbusConnectedRequestHandler) + self.address, self.handler) def process_request(self, request, client): ''' Callback for connecting a new client thread @@ -339,13 +340,14 @@ def __init__(self, context, framer=None, identity=None, address=None, handler=No self.context = context or ModbusServerContext() self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) + self.handler = handler if isinstance(handler, ModbusBaseRequestHandler) else ModbusDisconnectedRequestHandler self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) socketserver.ThreadingUDPServer.__init__(self, - self.address, ModbusDisconnectedRequestHandler) + self.address, self.handler) def process_request(self, request, client): ''' Callback for connecting a new client thread From c8090a4e251e93953f6988224ad6fe68fe294fe1 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 11:00:41 +0530 Subject: [PATCH 15/33] PYM-6: Removing uneccessary check if handler is of type ModbusBaseRequestHandler --- pymodbus/server/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index d67d0d183..78ec598bc 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -276,7 +276,7 @@ def __init__(self, context, framer=None, identity=None, address=None, handler=No self.context = context or ModbusServerContext() self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) - self.handler = handler if isinstance(handler, ModbusBaseRequestHandler) else ModbusConnectedRequestHandler + self.handler = handler or ModbusConnectedRequestHandler self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) if isinstance(identity, ModbusDeviceIdentification): @@ -340,7 +340,7 @@ def __init__(self, context, framer=None, identity=None, address=None, handler=No self.context = context or ModbusServerContext() self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) - self.handler = handler if isinstance(handler, ModbusBaseRequestHandler) else ModbusDisconnectedRequestHandler + self.handler = handler or ModbusDisconnectedRequestHandler self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) if isinstance(identity, ModbusDeviceIdentification): From 1dead0df423d19fb4eb7160b0763ca546ada2fd5 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 15:50:11 +0530 Subject: [PATCH 16/33] PYM-8: Example that read from a database as a datastore --- examples/common/dbstore-update-server.py | 82 +++++++++++++++++++ .../database-store}/database-datastore.py | 0 .../database-store}/redis-datastore.py | 0 3 files changed, 82 insertions(+) create mode 100644 examples/common/dbstore-update-server.py rename {examples/contrib => pymodbus/datastore/database-store}/database-datastore.py (100%) rename {examples/contrib => pymodbus/datastore/database-store}/redis-datastore.py (100%) diff --git a/examples/common/dbstore-update-server.py b/examples/common/dbstore-update-server.py new file mode 100644 index 000000000..517ad0c68 --- /dev/null +++ b/examples/common/dbstore-update-server.py @@ -0,0 +1,82 @@ +''' +Pymodbus Server With Updating Thread +-------------------------------------------------------------------------- +This is an example of having a background thread updating the +context while the server is operating. This can also be done with +a python thread:: + from threading import Thread + thread = Thread(target=updating_writer, args=(context,)) + thread.start() +''' +#---------------------------------------------------------------------------# +# import the modbus libraries we need +#---------------------------------------------------------------------------# +from pymodbus.server.async import StartTcpServer +from pymodbus.device import ModbusDeviceIdentification +from pymodbus.datastore import ModbusSequentialDataBlock +from pymodbus.datastore import ModbusServerContext +from pymodbus.datastore.database import +from pymodbus.datastore.database import DatabaseSlaveContext +from pymodbus.transaction import ModbusRtuFramer, ModbusAsciiFramer + +#---------------------------------------------------------------------------# +# import the twisted libraries we need +#---------------------------------------------------------------------------# +from twisted.internet.task import LoopingCall + +#---------------------------------------------------------------------------# +# configure the service logging +#---------------------------------------------------------------------------# +import logging +logging.basicConfig() +log = logging.getLogger() +log.setLevel(logging.DEBUG) + +#---------------------------------------------------------------------------# +# define your callback process +#---------------------------------------------------------------------------# +def updating_writer(a): + ''' A worker process that runs every so often and + updates live values of the context. It should be noted + that there is a race condition for the update. + :param arguments: The input arguments to the call + ''' + log.debug("updating the context") + context = a[0] + readfunction = 0x03 # read holding registers + writefunction = 0x10 + slave_id = 0x01 # slave address + address = 16 # adress : 400017 + + + values = context[slave_id].getValues(readfunction, address, count=3) + log.debug("new values: " + str(values)) + + +#---------------------------------------------------------------------------# +# initialize your data store +#---------------------------------------------------------------------------# +block = ModbusSequentialDataBlock(0x00, [0]*0xff) +store = DatabaseSlaveContext(block) + +context = ModbusServerContext(slaves={1: store}, single=False) + + +#---------------------------------------------------------------------------# +# initialize the server information +#---------------------------------------------------------------------------# +identity = ModbusDeviceIdentification() +identity.VendorName = 'pymodbus' +identity.ProductCode = 'PM' +identity.VendorUrl = 'http://github.com/bashwork/pymodbus/' +identity.ProductName = 'pymodbus Server' +identity.ModelName = 'pymodbus Server' +identity.MajorMinorRevision = '1.0' + +#---------------------------------------------------------------------------# +# run the server you want +#---------------------------------------------------------------------------# +time = 5 # 5 seconds delay +loop = LoopingCall(f=updating_writer, a=(context,)) +loop.start(time, now=False) # initially delay by time +StartTcpServer(context, identity=identity, address=("", 5020)) diff --git a/examples/contrib/database-datastore.py b/pymodbus/datastore/database-store/database-datastore.py similarity index 100% rename from examples/contrib/database-datastore.py rename to pymodbus/datastore/database-store/database-datastore.py diff --git a/examples/contrib/redis-datastore.py b/pymodbus/datastore/database-store/redis-datastore.py similarity index 100% rename from examples/contrib/redis-datastore.py rename to pymodbus/datastore/database-store/redis-datastore.py From 455457103995525b17054ba0e379c7f60659007e Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 15:51:44 +0530 Subject: [PATCH 17/33] PYM-8: Added two new datastores that can be used. - SQLite3 - Reddis --- pymodbus/datastore/database/__init__.py | 7 + .../datastore/database/database_datastore.py | 173 +++++++++++++ .../datastore/database/redis_datastore.py | 243 ++++++++++++++++++ 3 files changed, 423 insertions(+) create mode 100644 pymodbus/datastore/database/__init__.py create mode 100644 pymodbus/datastore/database/database_datastore.py create mode 100644 pymodbus/datastore/database/redis_datastore.py diff --git a/pymodbus/datastore/database/__init__.py b/pymodbus/datastore/database/__init__.py new file mode 100644 index 000000000..e8be9c255 --- /dev/null +++ b/pymodbus/datastore/database/__init__.py @@ -0,0 +1,7 @@ +from database_datastore import DatabaseSlaveContext +from redis_datastore import RedisSlaveContext + +#---------------------------------------------------------------------------# +# Exported symbols +#---------------------------------------------------------------------------# +__all__ = ["DatabaseSlaveContext", "RedisSlaveContext"] diff --git a/pymodbus/datastore/database/database_datastore.py b/pymodbus/datastore/database/database_datastore.py new file mode 100644 index 000000000..d3fbed07a --- /dev/null +++ b/pymodbus/datastore/database/database_datastore.py @@ -0,0 +1,173 @@ +import sqlalchemy +import sqlalchemy.types as sqltypes +from sqlalchemy.sql import and_ +from sqlalchemy.schema import UniqueConstraint +from sqlalchemy.sql.expression import bindparam + +from pymodbus.exceptions import NotImplementedException +from pymodbus.interfaces import IModbusSlaveContext + +#---------------------------------------------------------------------------# +# Logging +#---------------------------------------------------------------------------# +import logging; +_logger = logging.getLogger(__name__) + + +#---------------------------------------------------------------------------# +# Context +#---------------------------------------------------------------------------# +class DatabaseSlaveContext(IModbusSlaveContext): + ''' + This creates a modbus data model with each data access + stored in its own personal block + ''' + + def __init__(self, *args, **kwargs): + ''' Initializes the datastores + + :param kwargs: Each element is a ModbusDataBlock + ''' + self.table = kwargs.get('table', 'pymodbus') + self.database = kwargs.get('database', 'sqlite:///pymodbus.db') + self._db_create(self.table, self.database) + + def __str__(self): + ''' Returns a string representation of the context + + :returns: A string representation of the context + ''' + return "Modbus Slave Context" + + def reset(self): + ''' Resets all the datastores to their default values ''' + self._metadata.drop_all() + self._db_create(self.table, self.database) + + def validate(self, fx, address, count=1): + ''' Validates the request to make sure it is in range + + :param fx: The function we are working with + :param address: The starting address + :param count: The number of values to test + :returns: True if the request in within range, False otherwise + ''' + address = address + 1 # section 4.4 of specification + _logger.debug("validate[%d] %d:%d" % (fx, address, count)) + return self._validate(self.decode(fx), address, count) + + def getValues(self, fx, address, count=1): + ''' Validates the request to make sure it is in range + + :param fx: The function we are working with + :param address: The starting address + :param count: The number of values to retrieve + :returns: The requested values from a:a+c + ''' + address = address + 1 # section 4.4 of specification + _logger.debug("get-values[%d] %d:%d" % (fx, address, count)) + return self._get(self.decode(fx), address, count) + + def setValues(self, fx, address, values): + ''' Sets the datastore with the supplied values + + :param fx: The function we are working with + :param address: The starting address + :param values: The new values to be set + ''' + address = address + 1 # section 4.4 of specification + _logger.debug("set-values[%d] %d:%d" % (fx, address, len(values))) + self._set(self.decode(fx), address, values) + + #--------------------------------------------------------------------------# + # Sqlite Helper Methods + #--------------------------------------------------------------------------# + def _db_create(self, table, database): + ''' A helper method to initialize the database and handles + + :param table: The table name to create + :param database: The database uri to use + ''' + self._engine = sqlalchemy.create_engine(database, echo=False) + self._metadata = sqlalchemy.MetaData(self._engine) + self._table = sqlalchemy.Table(table, self._metadata, + sqlalchemy.Column('type', sqltypes.String(1)), + sqlalchemy.Column('index', sqltypes.Integer), + sqlalchemy.Column('value', sqltypes.Integer), + UniqueConstraint('type', 'index', name='key')) + self._table.create(checkfirst=True) + self._connection = self._engine.connect() + + def _get(self, type, offset, count): + ''' + + :param type: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + :returns: The resulting values + ''' + query = self._table.select(and_( + self._table.c.type == type, + self._table.c.index >= offset, + self._table.c.index <= offset + count) + ) + query = query.order_by(self._table.c.index.asc()) + result = self._connection.execute(query).fetchall() + return [row.value for row in result] + + def _build_set(self, type, offset, values, p=''): + ''' A helper method to generate the sql update context + + :param type: The key prefix to use + :param offset: The address offset to start at + :param values: The values to set + ''' + result = [] + for index, value in enumerate(values): + result.append({ + p + 'type' : type, + p + 'index' : offset + index, + 'value' : value + }) + return result + + def _set(self, type, offset, values): + ''' + + :param key: The type prefix to use + :param offset: The address offset to start at + :param values: The values to set + ''' + context = self._build_set(type, offset, values) + query = self._table.insert() + result = self._connection.execute(query, context) + return result.rowcount == len(values) + + def _update(self, type, offset, values): + ''' + + :param type: The type prefix to use + :param offset: The address offset to start at + :param values: The values to set + ''' + context = self._build_set(type, offset, values, p='x_') + query = self._table.update().values(name='value') + query = query.where(and_( + self._table.c.type == bindparam('x_type'), + self._table.c.index == bindparam('x_index'))) + result = self._connection.execute(query, context) + return result.rowcount == len(values) + + def _validate(self, type, offset, count): + ''' + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + :returns: The result of the validation + ''' + query = self._table.select(and_( + self._table.c.type == type, + self._table.c.index >= offset, + self._table.c.index <= offset + count)) + result = self._connection.execute(query) + return result.rowcount == count diff --git a/pymodbus/datastore/database/redis_datastore.py b/pymodbus/datastore/database/redis_datastore.py new file mode 100644 index 000000000..ef44c6544 --- /dev/null +++ b/pymodbus/datastore/database/redis_datastore.py @@ -0,0 +1,243 @@ +import redis +from pymodbus.interfaces import IModbusSlaveContext +from pymodbus.utilities import pack_bitstring, unpack_bitstring + +#---------------------------------------------------------------------------# +# Logging +#---------------------------------------------------------------------------# +import logging; +_logger = logging.getLogger(__name__) + + +#---------------------------------------------------------------------------# +# Context +#---------------------------------------------------------------------------# +class RedisSlaveContext(IModbusSlaveContext): + ''' + This is a modbus slave context using redis as a backing + store. + ''' + + def __init__(self, **kwargs): + ''' Initializes the datastores + + :param host: The host to connect to + :param port: The port to connect to + :param prefix: A prefix for the keys + ''' + host = kwargs.get('host', 'localhost') + port = kwargs.get('port', 6379) + self.prefix = kwargs.get('prefix', 'pymodbus') + self.client = kwargs.get('client', redis.Redis(host=host, port=port)) + self.__build_mapping() + + def __str__(self): + ''' Returns a string representation of the context + + :returns: A string representation of the context + ''' + return "Redis Slave Context %s" % self.client + + def reset(self): + ''' Resets all the datastores to their default values ''' + self.client.flushall() + + def validate(self, fx, address, count=1): + ''' Validates the request to make sure it is in range + + :param fx: The function we are working with + :param address: The starting address + :param count: The number of values to test + :returns: True if the request in within range, False otherwise + ''' + address = address + 1 # section 4.4 of specification + _logger.debug("validate[%d] %d:%d" % (fx, address, count)) + return self.__val_callbacks[self.decode(fx)](address, count) + + def getValues(self, fx, address, count=1): + ''' Validates the request to make sure it is in range + + :param fx: The function we are working with + :param address: The starting address + :param count: The number of values to retrieve + :returns: The requested values from a:a+c + ''' + address = address + 1 # section 4.4 of specification + _logger.debug("getValues[%d] %d:%d" % (fx, address, count)) + return self.__get_callbacks[self.decode(fx)](address, count) + + def setValues(self, fx, address, values): + ''' Sets the datastore with the supplied values + + :param fx: The function we are working with + :param address: The starting address + :param values: The new values to be set + ''' + address = address + 1 # section 4.4 of specification + _logger.debug("setValues[%d] %d:%d" % (fx, address, len(values))) + self.__set_callbacks[self.decode(fx)](address, values) + + #--------------------------------------------------------------------------# + # Redis Helper Methods + #--------------------------------------------------------------------------# + def __get_prefix(self, key): + ''' This is a helper to abstract getting bit values + + :param key: The key prefix to use + :returns: The key prefix to redis + ''' + return "%s:%s" % (self.prefix, key) + + def __build_mapping(self): + ''' + A quick helper method to build the function + code mapper. + ''' + self.__val_callbacks = { + 'd' : lambda o, c: self.__val_bit('d', o, c), + 'c' : lambda o, c: self.__val_bit('c', o, c), + 'h' : lambda o, c: self.__val_reg('h', o, c), + 'i' : lambda o, c: self.__val_reg('i', o, c), + } + self.__get_callbacks = { + 'd' : lambda o, c: self.__get_bit('d', o, c), + 'c' : lambda o, c: self.__get_bit('c', o, c), + 'h' : lambda o, c: self.__get_reg('h', o, c), + 'i' : lambda o, c: self.__get_reg('i', o, c), + } + self.__set_callbacks = { + 'd' : lambda o, v: self.__set_bit('d', o, v), + 'c' : lambda o, v: self.__set_bit('c', o, v), + 'h' : lambda o, v: self.__set_reg('h', o, v), + 'i' : lambda o, v: self.__set_reg('i', o, v), + } + + #--------------------------------------------------------------------------# + # Redis discrete implementation + #--------------------------------------------------------------------------# + __bit_size = 16 + __bit_default = '\x00' * (__bit_size % 8) + + def __get_bit_values(self, key, offset, count): + ''' This is a helper to abstract getting bit values + + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + ''' + key = self.__get_prefix(key) + s = divmod(offset, self.__bit_size)[0] + e = divmod(offset + count, self.__bit_size)[0] + + request = ('%s:%s' % (key, v) for v in range(s, e + 1)) + response = self.client.mget(request) + return response + + def __val_bit(self, key, offset, count): + ''' Validates that the given range is currently set in redis. + If any of the keys return None, then it is invalid. + + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + ''' + response = self.__get_bit_values(key, offset, count) + return None not in response + + def __get_bit(self, key, offset, count): + ''' + + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + ''' + response = self.__get_bit_values(key, offset, count) + response = (r or self.__bit_default for r in response) + result = ''.join(response) + result = unpack_bitstring(result) + return result[offset:offset + count] + + def __set_bit(self, key, offset, values): + ''' + + :param key: The key prefix to use + :param offset: The address offset to start at + :param values: The values to set + ''' + count = len(values) + s = divmod(offset, self.__bit_size)[0] + e = divmod(offset + count, self.__bit_size)[0] + value = pack_bitstring(values) + + current = self.__get_bit_values(key, offset, count) + current = (r or self.__bit_default for r in current) + current = ''.join(current) + current = current[0:offset] + value + current[offset + count:] + final = (current[s:s + self.__bit_size] for s in range(0, count, self.__bit_size)) + + key = self.__get_prefix(key) + request = ('%s:%s' % (key, v) for v in range(s, e + 1)) + request = dict(zip(request, final)) + self.client.mset(request) + + #--------------------------------------------------------------------------# + # Redis register implementation + #--------------------------------------------------------------------------# + __reg_size = 16 + __reg_default = '\x00' * (__reg_size % 8) + + def __get_reg_values(self, key, offset, count): + ''' This is a helper to abstract getting register values + + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + ''' + key = self.__get_prefix(key) + #s = divmod(offset, self.__reg_size)[0] + #e = divmod(offset+count, self.__reg_size)[0] + + #request = ('%s:%s' % (key, v) for v in range(s, e + 1)) + request = ('%s:%s' % (key, v) for v in range(offset, count + 1)) + response = self.client.mget(request) + return response + + def __val_reg(self, key, offset, count): + ''' Validates that the given range is currently set in redis. + If any of the keys return None, then it is invalid. + + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + ''' + response = self.__get_reg_values(key, offset, count) + return None not in response + + def __get_reg(self, key, offset, count): + ''' + + :param key: The key prefix to use + :param offset: The address offset to start at + :param count: The number of bits to read + ''' + response = self.__get_reg_values(key, offset, count) + response = [r or self.__reg_default for r in response] + return response[offset:offset + count] + + def __set_reg(self, key, offset, values): + ''' + + :param key: The key prefix to use + :param offset: The address offset to start at + :param values: The values to set + ''' + count = len(values) + #s = divmod(offset, self.__reg_size) + #e = divmod(offset+count, self.__reg_size) + + #current = self.__get_reg_values(key, offset, count) + + key = self.__get_prefix(key) + request = ('%s:%s' % (key, v) for v in range(offset, count + 1)) + request = dict(zip(request, values)) + self.client.mset(request) From e1393c4882d4b179d084d592a26bd209b32a0f53 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 15:56:17 +0530 Subject: [PATCH 18/33] Small fixes --- examples/common/dbstore-update-server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/common/dbstore-update-server.py b/examples/common/dbstore-update-server.py index 517ad0c68..219862455 100644 --- a/examples/common/dbstore-update-server.py +++ b/examples/common/dbstore-update-server.py @@ -15,7 +15,7 @@ from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSequentialDataBlock from pymodbus.datastore import ModbusServerContext -from pymodbus.datastore.database import +from pymodbus.datastore.database import DatabaseSlaveContext from pymodbus.datastore.database import DatabaseSlaveContext from pymodbus.transaction import ModbusRtuFramer, ModbusAsciiFramer From eed9f05b7ae260cf68f218501bf56f60d73174e5 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 15:57:45 +0530 Subject: [PATCH 19/33] Small fixes --- examples/common/dbstore-update-server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/common/dbstore-update-server.py b/examples/common/dbstore-update-server.py index 219862455..7958b3bcf 100644 --- a/examples/common/dbstore-update-server.py +++ b/examples/common/dbstore-update-server.py @@ -2,8 +2,8 @@ Pymodbus Server With Updating Thread -------------------------------------------------------------------------- This is an example of having a background thread updating the -context while the server is operating. This can also be done with -a python thread:: +context in an SQLite4 database while the server is operating. +This can also be done with a python thread:: from threading import Thread thread = Thread(target=updating_writer, args=(context,)) thread.start() @@ -37,11 +37,11 @@ #---------------------------------------------------------------------------# def updating_writer(a): ''' A worker process that runs every so often and - updates live values of the context. It should be noted - that there is a race condition for the update. + updates live values of the context which resides in an SQLite3 database. + It should be noted that there is a race condition for the update. :param arguments: The input arguments to the call ''' - log.debug("updating the context") + log.debug("Updating the database context") context = a[0] readfunction = 0x03 # read holding registers writefunction = 0x10 @@ -50,7 +50,7 @@ def updating_writer(a): values = context[slave_id].getValues(readfunction, address, count=3) - log.debug("new values: " + str(values)) + log.debug("New values from datastore: " + str(values)) #---------------------------------------------------------------------------# From b05d9051d50b6ca71393fda6b97f0886e65fa382 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 15:58:20 +0530 Subject: [PATCH 20/33] Small fixes --- examples/common/dbstore-update-server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/common/dbstore-update-server.py b/examples/common/dbstore-update-server.py index 7958b3bcf..fd0de0c93 100644 --- a/examples/common/dbstore-update-server.py +++ b/examples/common/dbstore-update-server.py @@ -16,7 +16,6 @@ from pymodbus.datastore import ModbusSequentialDataBlock from pymodbus.datastore import ModbusServerContext from pymodbus.datastore.database import DatabaseSlaveContext -from pymodbus.datastore.database import DatabaseSlaveContext from pymodbus.transaction import ModbusRtuFramer, ModbusAsciiFramer #---------------------------------------------------------------------------# From e36c32c6f70ba5964d0d7cb319df192eb994cef4 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 21 Nov 2017 16:00:19 +0530 Subject: [PATCH 21/33] Cleanup --- .../database-store/database-datastore.py | 173 ------------- .../database-store/redis-datastore.py | 243 ------------------ 2 files changed, 416 deletions(-) delete mode 100644 pymodbus/datastore/database-store/database-datastore.py delete mode 100644 pymodbus/datastore/database-store/redis-datastore.py diff --git a/pymodbus/datastore/database-store/database-datastore.py b/pymodbus/datastore/database-store/database-datastore.py deleted file mode 100644 index c1c48b161..000000000 --- a/pymodbus/datastore/database-store/database-datastore.py +++ /dev/null @@ -1,173 +0,0 @@ -import sqlalchemy -import sqlalchemy.types as sqltypes -from sqlalchemy.sql import and_ -from sqlalchemy.schema import UniqueConstraint -from sqlalchemy.sql.expression import bindparam - -from pymodbus.exceptions import NotImplementedException -from pymodbus.interfaces import IModbusSlaveContext - -#---------------------------------------------------------------------------# -# Logging -#---------------------------------------------------------------------------# -import logging; -_logger = logging.getLogger(__name__) - - -#---------------------------------------------------------------------------# -# Context -#---------------------------------------------------------------------------# -class DatabaseSlaveContext(IModbusSlaveContext): - ''' - This creates a modbus data model with each data access - stored in its own personal block - ''' - - def __init__(self, *args, **kwargs): - ''' Initializes the datastores - - :param kwargs: Each element is a ModbusDataBlock - ''' - self.table = kwargs.get('table', 'pymodbus') - self.database = kwargs.get('database', 'sqlite:///pymodbus.db') - self.__db_create(self.table, self.database) - - def __str__(self): - ''' Returns a string representation of the context - - :returns: A string representation of the context - ''' - return "Modbus Slave Context" - - def reset(self): - ''' Resets all the datastores to their default values ''' - self._metadata.drop_all() - self.__db_create(self.table, self.database) - raise NotImplementedException() # TODO drop table? - - def validate(self, fx, address, count=1): - ''' Validates the request to make sure it is in range - - :param fx: The function we are working with - :param address: The starting address - :param count: The number of values to test - :returns: True if the request in within range, False otherwise - ''' - address = address + 1 # section 4.4 of specification - _logger.debug("validate[%d] %d:%d" % (fx, address, count)) - return self.__validate(self.decode(fx), address, count) - - def getValues(self, fx, address, count=1): - ''' Validates the request to make sure it is in range - - :param fx: The function we are working with - :param address: The starting address - :param count: The number of values to retrieve - :returns: The requested values from a:a+c - ''' - address = address + 1 # section 4.4 of specification - _logger.debug("get-values[%d] %d:%d" % (fx, address, count)) - return self.__get(self.decode(fx), address, count) - - def setValues(self, fx, address, values): - ''' Sets the datastore with the supplied values - - :param fx: The function we are working with - :param address: The starting address - :param values: The new values to be set - ''' - address = address + 1 # section 4.4 of specification - _logger.debug("set-values[%d] %d:%d" % (fx, address, len(values))) - self.__set(self.decode(fx), address, values) - - #--------------------------------------------------------------------------# - # Sqlite Helper Methods - #--------------------------------------------------------------------------# - def __db_create(self, table, database): - ''' A helper method to initialize the database and handles - - :param table: The table name to create - :param database: The database uri to use - ''' - self._engine = sqlalchemy.create_engine(database, echo=False) - self._metadata = sqlalchemy.MetaData(self._engine) - self._table = sqlalchemy.Table(table, self._metadata, - sqlalchemy.Column('type', sqltypes.String(1)), - sqlalchemy.Column('index', sqltypes.Integer), - sqlalchemy.Column('value', sqltypes.Integer), - UniqueConstraint('type', 'index', name='key')) - self._table.create(checkfirst=True) - self._connection = self._engine.connect() - - def __get(self, type, offset, count): - ''' - - :param type: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - :returns: The resulting values - ''' - query = self._table.select(and_( - self._table.c.type == type, - self._table.c.index >= offset, - self._table.c.index <= offset + count)) - query = query.order_by(self._table.c.index.asc()) - result = self._connection.execute(query).fetchall() - return [row.value for row in result] - - def __build_set(self, type, offset, values, p=''): - ''' A helper method to generate the sql update context - - :param type: The key prefix to use - :param offset: The address offset to start at - :param values: The values to set - ''' - result = [] - for index, value in enumerate(values): - result.append({ - p + 'type' : type, - p + 'index' : offset + index, - 'value' : value - }) - return result - - def __set(self, type, offset, values): - ''' - - :param key: The type prefix to use - :param offset: The address offset to start at - :param values: The values to set - ''' - context = self.__build_set(type, offset, values) - query = self._table.insert() - result = self._connection.execute(query, context) - return result.rowcount == len(values) - - def __update(self, type, offset, values): - ''' - - :param type: The type prefix to use - :param offset: The address offset to start at - :param values: The values to set - ''' - context = self.__build_set(type, offset, values, p='x_') - query = self._table.update().values(name='value') - query = query.where(and_( - self._table.c.type == bindparam('x_type'), - self._table.c.index == bindparam('x_index'))) - result = self._connection.execute(query, context) - return result.rowcount == len(values) - - def __validate(self, key, offset, count): - ''' - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - :returns: The result of the validation - ''' - query = self._table.select(and_( - self._table.c.type == type, - self._table.c.index >= offset, - self._table.c.index <= offset + count)) - result = self._connection.execute(query) - return result.rowcount == count diff --git a/pymodbus/datastore/database-store/redis-datastore.py b/pymodbus/datastore/database-store/redis-datastore.py deleted file mode 100644 index ef44c6544..000000000 --- a/pymodbus/datastore/database-store/redis-datastore.py +++ /dev/null @@ -1,243 +0,0 @@ -import redis -from pymodbus.interfaces import IModbusSlaveContext -from pymodbus.utilities import pack_bitstring, unpack_bitstring - -#---------------------------------------------------------------------------# -# Logging -#---------------------------------------------------------------------------# -import logging; -_logger = logging.getLogger(__name__) - - -#---------------------------------------------------------------------------# -# Context -#---------------------------------------------------------------------------# -class RedisSlaveContext(IModbusSlaveContext): - ''' - This is a modbus slave context using redis as a backing - store. - ''' - - def __init__(self, **kwargs): - ''' Initializes the datastores - - :param host: The host to connect to - :param port: The port to connect to - :param prefix: A prefix for the keys - ''' - host = kwargs.get('host', 'localhost') - port = kwargs.get('port', 6379) - self.prefix = kwargs.get('prefix', 'pymodbus') - self.client = kwargs.get('client', redis.Redis(host=host, port=port)) - self.__build_mapping() - - def __str__(self): - ''' Returns a string representation of the context - - :returns: A string representation of the context - ''' - return "Redis Slave Context %s" % self.client - - def reset(self): - ''' Resets all the datastores to their default values ''' - self.client.flushall() - - def validate(self, fx, address, count=1): - ''' Validates the request to make sure it is in range - - :param fx: The function we are working with - :param address: The starting address - :param count: The number of values to test - :returns: True if the request in within range, False otherwise - ''' - address = address + 1 # section 4.4 of specification - _logger.debug("validate[%d] %d:%d" % (fx, address, count)) - return self.__val_callbacks[self.decode(fx)](address, count) - - def getValues(self, fx, address, count=1): - ''' Validates the request to make sure it is in range - - :param fx: The function we are working with - :param address: The starting address - :param count: The number of values to retrieve - :returns: The requested values from a:a+c - ''' - address = address + 1 # section 4.4 of specification - _logger.debug("getValues[%d] %d:%d" % (fx, address, count)) - return self.__get_callbacks[self.decode(fx)](address, count) - - def setValues(self, fx, address, values): - ''' Sets the datastore with the supplied values - - :param fx: The function we are working with - :param address: The starting address - :param values: The new values to be set - ''' - address = address + 1 # section 4.4 of specification - _logger.debug("setValues[%d] %d:%d" % (fx, address, len(values))) - self.__set_callbacks[self.decode(fx)](address, values) - - #--------------------------------------------------------------------------# - # Redis Helper Methods - #--------------------------------------------------------------------------# - def __get_prefix(self, key): - ''' This is a helper to abstract getting bit values - - :param key: The key prefix to use - :returns: The key prefix to redis - ''' - return "%s:%s" % (self.prefix, key) - - def __build_mapping(self): - ''' - A quick helper method to build the function - code mapper. - ''' - self.__val_callbacks = { - 'd' : lambda o, c: self.__val_bit('d', o, c), - 'c' : lambda o, c: self.__val_bit('c', o, c), - 'h' : lambda o, c: self.__val_reg('h', o, c), - 'i' : lambda o, c: self.__val_reg('i', o, c), - } - self.__get_callbacks = { - 'd' : lambda o, c: self.__get_bit('d', o, c), - 'c' : lambda o, c: self.__get_bit('c', o, c), - 'h' : lambda o, c: self.__get_reg('h', o, c), - 'i' : lambda o, c: self.__get_reg('i', o, c), - } - self.__set_callbacks = { - 'd' : lambda o, v: self.__set_bit('d', o, v), - 'c' : lambda o, v: self.__set_bit('c', o, v), - 'h' : lambda o, v: self.__set_reg('h', o, v), - 'i' : lambda o, v: self.__set_reg('i', o, v), - } - - #--------------------------------------------------------------------------# - # Redis discrete implementation - #--------------------------------------------------------------------------# - __bit_size = 16 - __bit_default = '\x00' * (__bit_size % 8) - - def __get_bit_values(self, key, offset, count): - ''' This is a helper to abstract getting bit values - - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - ''' - key = self.__get_prefix(key) - s = divmod(offset, self.__bit_size)[0] - e = divmod(offset + count, self.__bit_size)[0] - - request = ('%s:%s' % (key, v) for v in range(s, e + 1)) - response = self.client.mget(request) - return response - - def __val_bit(self, key, offset, count): - ''' Validates that the given range is currently set in redis. - If any of the keys return None, then it is invalid. - - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - ''' - response = self.__get_bit_values(key, offset, count) - return None not in response - - def __get_bit(self, key, offset, count): - ''' - - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - ''' - response = self.__get_bit_values(key, offset, count) - response = (r or self.__bit_default for r in response) - result = ''.join(response) - result = unpack_bitstring(result) - return result[offset:offset + count] - - def __set_bit(self, key, offset, values): - ''' - - :param key: The key prefix to use - :param offset: The address offset to start at - :param values: The values to set - ''' - count = len(values) - s = divmod(offset, self.__bit_size)[0] - e = divmod(offset + count, self.__bit_size)[0] - value = pack_bitstring(values) - - current = self.__get_bit_values(key, offset, count) - current = (r or self.__bit_default for r in current) - current = ''.join(current) - current = current[0:offset] + value + current[offset + count:] - final = (current[s:s + self.__bit_size] for s in range(0, count, self.__bit_size)) - - key = self.__get_prefix(key) - request = ('%s:%s' % (key, v) for v in range(s, e + 1)) - request = dict(zip(request, final)) - self.client.mset(request) - - #--------------------------------------------------------------------------# - # Redis register implementation - #--------------------------------------------------------------------------# - __reg_size = 16 - __reg_default = '\x00' * (__reg_size % 8) - - def __get_reg_values(self, key, offset, count): - ''' This is a helper to abstract getting register values - - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - ''' - key = self.__get_prefix(key) - #s = divmod(offset, self.__reg_size)[0] - #e = divmod(offset+count, self.__reg_size)[0] - - #request = ('%s:%s' % (key, v) for v in range(s, e + 1)) - request = ('%s:%s' % (key, v) for v in range(offset, count + 1)) - response = self.client.mget(request) - return response - - def __val_reg(self, key, offset, count): - ''' Validates that the given range is currently set in redis. - If any of the keys return None, then it is invalid. - - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - ''' - response = self.__get_reg_values(key, offset, count) - return None not in response - - def __get_reg(self, key, offset, count): - ''' - - :param key: The key prefix to use - :param offset: The address offset to start at - :param count: The number of bits to read - ''' - response = self.__get_reg_values(key, offset, count) - response = [r or self.__reg_default for r in response] - return response[offset:offset + count] - - def __set_reg(self, key, offset, values): - ''' - - :param key: The key prefix to use - :param offset: The address offset to start at - :param values: The values to set - ''' - count = len(values) - #s = divmod(offset, self.__reg_size) - #e = divmod(offset+count, self.__reg_size) - - #current = self.__get_reg_values(key, offset, count) - - key = self.__get_prefix(key) - request = ('%s:%s' % (key, v) for v in range(offset, count + 1)) - request = dict(zip(request, values)) - self.client.mset(request) From 9f6b23c45ae86be453e49718cae0214fcbe3a644 Mon Sep 17 00:00:00 2001 From: rahul Date: Wed, 22 Nov 2017 12:08:28 +0530 Subject: [PATCH 22/33] PYM-8: Updated the example to first write a random value at a random afddress to a database and then read from that address --- examples/common/dbstore-update-server.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/common/dbstore-update-server.py b/examples/common/dbstore-update-server.py index fd0de0c93..cbb8a34e9 100644 --- a/examples/common/dbstore-update-server.py +++ b/examples/common/dbstore-update-server.py @@ -3,6 +3,11 @@ -------------------------------------------------------------------------- This is an example of having a background thread updating the context in an SQLite4 database while the server is operating. + +This scrit generates a random address range (within 0 - 65000) and a random +value and stores it in a database. It then reads the same address to verify +that the process works as expected + This can also be done with a python thread:: from threading import Thread thread = Thread(target=updating_writer, args=(context,)) @@ -17,6 +22,7 @@ from pymodbus.datastore import ModbusServerContext from pymodbus.datastore.database import DatabaseSlaveContext from pymodbus.transaction import ModbusRtuFramer, ModbusAsciiFramer +import random #---------------------------------------------------------------------------# # import the twisted libraries we need @@ -45,11 +51,18 @@ def updating_writer(a): readfunction = 0x03 # read holding registers writefunction = 0x10 slave_id = 0x01 # slave address - address = 16 # adress : 400017 + count = 50 + + # import pdb; pdb.set_trace() + rand_value = random.randint(0, 9999) + rand_addr = random.randint(0, 65000) + log.debug("Writing to datastore: {}, {}".format(rand_addr, rand_value)) + # import pdb; pdb.set_trace() + context[slave_id].setValues(writefunction, rand_addr, [rand_value]) + values = context[slave_id].getValues(readfunction, rand_addr, count) + log.debug("Values from datastore: " + str(values)) - values = context[slave_id].getValues(readfunction, address, count=3) - log.debug("New values from datastore: " + str(values)) #---------------------------------------------------------------------------# From eec4eb5fcc6b6e2372c1ec758b17f9b8477e3692 Mon Sep 17 00:00:00 2001 From: rahul Date: Wed, 22 Nov 2017 12:16:34 +0530 Subject: [PATCH 23/33] PYM-8: Added neccessary checks and methods to allow hassle free writes to database. The process first checks if the address and value are already present in the database before performing a write. This ensures that database transaction errors will now occur in cases where repetetive data is being written. --- .../datastore/database/database_datastore.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pymodbus/datastore/database/database_datastore.py b/pymodbus/datastore/database/database_datastore.py index d3fbed07a..c69b51ebf 100644 --- a/pymodbus/datastore/database/database_datastore.py +++ b/pymodbus/datastore/database/database_datastore.py @@ -131,6 +131,11 @@ def _build_set(self, type, offset, values, p=''): }) return result + def _check(self, type, offset, values): + result = self._get(type, offset, count=1) + # import pdb; pdb.set_trace() + return False if len(result) > 0 else True + def _set(self, type, offset, values): ''' @@ -138,10 +143,14 @@ def _set(self, type, offset, values): :param offset: The address offset to start at :param values: The values to set ''' - context = self._build_set(type, offset, values) - query = self._table.insert() - result = self._connection.execute(query, context) - return result.rowcount == len(values) + # import pdb; pdb.set_trace() + if self._check(type, offset, values): + context = self._build_set(type, offset, values) + query = self._table.insert() + result = self._connection.execute(query, context) + return result.rowcount == len(values) + else: + return False def _update(self, type, offset, values): ''' From 1e0c3082a4383fe4bf11891dee9a2411bdd847c2 Mon Sep 17 00:00:00 2001 From: rahul Date: Wed, 22 Nov 2017 12:17:57 +0530 Subject: [PATCH 24/33] Cleanup: Removing pdb placed during testing and other comments --- pymodbus/datastore/database/database_datastore.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pymodbus/datastore/database/database_datastore.py b/pymodbus/datastore/database/database_datastore.py index c69b51ebf..7128fd655 100644 --- a/pymodbus/datastore/database/database_datastore.py +++ b/pymodbus/datastore/database/database_datastore.py @@ -133,7 +133,6 @@ def _build_set(self, type, offset, values, p=''): def _check(self, type, offset, values): result = self._get(type, offset, count=1) - # import pdb; pdb.set_trace() return False if len(result) > 0 else True def _set(self, type, offset, values): @@ -143,7 +142,6 @@ def _set(self, type, offset, values): :param offset: The address offset to start at :param values: The values to set ''' - # import pdb; pdb.set_trace() if self._check(type, offset, values): context = self._build_set(type, offset, values) query = self._table.insert() From c86a02aaf25f2f697f4ec045f34ab145d60fc0ba Mon Sep 17 00:00:00 2001 From: sanjay Date: Wed, 22 Nov 2017 14:21:33 +0000 Subject: [PATCH 25/33] bitbucket-pipelines.yml deleted online with Bitbucket --- bitbucket-pipelines.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 bitbucket-pipelines.yml diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml deleted file mode 100644 index e6359056d..000000000 --- a/bitbucket-pipelines.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This is a sample build configuration for Python. -# Check our guides at https://confluence.atlassian.com/x/x4UWN for more examples. -# Only use spaces to indent your .yml configuration. -# ----- -# You can specify a custom docker image from Docker Hub as your build environment. -image: python:3.5.1 - -pipelines: - default: - - step: - caches: - - pip - script: # Modify the commands below to build your repository. - - pip install virtualenv - - scripts/travis.sh - - pip install --requirement=requirements-checks.txt - - pip install --requirement=requirements-tests.txt - - make test \ No newline at end of file From 37b8727c70de932af1f8f5414ef0d0593ce7aaff Mon Sep 17 00:00:00 2001 From: sanjay Date: Fri, 24 Nov 2017 14:45:30 +0530 Subject: [PATCH 26/33] #240 Fix PR failures --- requirements-tests.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 85623bb57..9dd8aeda1 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,5 +8,6 @@ Twisted>=17.1.0 zope.interface>=4.4.0 pyasn1>=0.2.3 pycrypto>=2.6.1 +sqlalchemy>=1.1.15 #wsgiref>=0.1.2 -cryptography>=1.8.1 \ No newline at end of file +cryptography>=1.8.1 From 8937d3477d1d77bad6fab898a78019f9ad5349a6 Mon Sep 17 00:00:00 2001 From: sanjay Date: Fri, 24 Nov 2017 15:12:57 +0530 Subject: [PATCH 27/33] #240 fix Travis build failures --- pymodbus/datastore/database/__init__.py | 6 +++--- .../database/{database_datastore.py => sql_datastore.py} | 2 +- requirements-tests.txt | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) rename pymodbus/datastore/database/{database_datastore.py => sql_datastore.py} (99%) diff --git a/pymodbus/datastore/database/__init__.py b/pymodbus/datastore/database/__init__.py index e8be9c255..dbb2609a4 100644 --- a/pymodbus/datastore/database/__init__.py +++ b/pymodbus/datastore/database/__init__.py @@ -1,7 +1,7 @@ -from database_datastore import DatabaseSlaveContext -from redis_datastore import RedisSlaveContext +from pymodbus.datastore.database.sql_datastore import SqlSlaveContext +from pymodbus.datastore.database.redis_datastore import RedisSlaveContext #---------------------------------------------------------------------------# # Exported symbols #---------------------------------------------------------------------------# -__all__ = ["DatabaseSlaveContext", "RedisSlaveContext"] +__all__ = ["SqlSlaveContext", "RedisSlaveContext"] diff --git a/pymodbus/datastore/database/database_datastore.py b/pymodbus/datastore/database/sql_datastore.py similarity index 99% rename from pymodbus/datastore/database/database_datastore.py rename to pymodbus/datastore/database/sql_datastore.py index 7128fd655..c1e7ddf5d 100644 --- a/pymodbus/datastore/database/database_datastore.py +++ b/pymodbus/datastore/database/sql_datastore.py @@ -17,7 +17,7 @@ #---------------------------------------------------------------------------# # Context #---------------------------------------------------------------------------# -class DatabaseSlaveContext(IModbusSlaveContext): +class SqlSlaveContext(IModbusSlaveContext): ''' This creates a modbus data model with each data access stored in its own personal block diff --git a/requirements-tests.txt b/requirements-tests.txt index 9dd8aeda1..5c4639d1b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,6 +8,8 @@ Twisted>=17.1.0 zope.interface>=4.4.0 pyasn1>=0.2.3 pycrypto>=2.6.1 +pyserial>=3.4 +redis>=2.10.5 sqlalchemy>=1.1.15 #wsgiref>=0.1.2 -cryptography>=1.8.1 +cryptography>=1.8.1 \ No newline at end of file From 3b7643ac4917a5519195f9de94a01f3efc742871 Mon Sep 17 00:00:00 2001 From: sanjay Date: Fri, 24 Nov 2017 15:17:27 +0530 Subject: [PATCH 28/33] #190 fix import error in dbstore-update-server example --- examples/common/dbstore-update-server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/common/dbstore-update-server.py b/examples/common/dbstore-update-server.py index cbb8a34e9..b0ce87bf0 100644 --- a/examples/common/dbstore-update-server.py +++ b/examples/common/dbstore-update-server.py @@ -20,7 +20,7 @@ from pymodbus.device import ModbusDeviceIdentification from pymodbus.datastore import ModbusSequentialDataBlock from pymodbus.datastore import ModbusServerContext -from pymodbus.datastore.database import DatabaseSlaveContext +from pymodbus.datastore.database import SqlSlaveContext from pymodbus.transaction import ModbusRtuFramer, ModbusAsciiFramer import random @@ -69,7 +69,7 @@ def updating_writer(a): # initialize your data store #---------------------------------------------------------------------------# block = ModbusSequentialDataBlock(0x00, [0]*0xff) -store = DatabaseSlaveContext(block) +store = SqlSlaveContext(block) context = ModbusServerContext(slaves={1: store}, single=False) From 0e492e72750babdf505ec49cc740f21181894986 Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 27 Nov 2017 12:51:20 +0530 Subject: [PATCH 29/33] Small changes and typo fixed in pymodbus utilities and redis datastore helpers --- .../datastore/database/redis_datastore.py | 102 +++++++++--------- pymodbus/utilities.py | 4 +- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/pymodbus/datastore/database/redis_datastore.py b/pymodbus/datastore/database/redis_datastore.py index ef44c6544..634dc1552 100644 --- a/pymodbus/datastore/database/redis_datastore.py +++ b/pymodbus/datastore/database/redis_datastore.py @@ -29,7 +29,7 @@ def __init__(self, **kwargs): port = kwargs.get('port', 6379) self.prefix = kwargs.get('prefix', 'pymodbus') self.client = kwargs.get('client', redis.Redis(host=host, port=port)) - self.__build_mapping() + self._build_mapping() def __str__(self): ''' Returns a string representation of the context @@ -52,7 +52,7 @@ def validate(self, fx, address, count=1): ''' address = address + 1 # section 4.4 of specification _logger.debug("validate[%d] %d:%d" % (fx, address, count)) - return self.__val_callbacks[self.decode(fx)](address, count) + return self._val_callbacks[self.decode(fx)](address, count) def getValues(self, fx, address, count=1): ''' Validates the request to make sure it is in range @@ -64,7 +64,7 @@ def getValues(self, fx, address, count=1): ''' address = address + 1 # section 4.4 of specification _logger.debug("getValues[%d] %d:%d" % (fx, address, count)) - return self.__get_callbacks[self.decode(fx)](address, count) + return self._get_callbacks[self.decode(fx)](address, count) def setValues(self, fx, address, values): ''' Sets the datastore with the supplied values @@ -75,12 +75,12 @@ def setValues(self, fx, address, values): ''' address = address + 1 # section 4.4 of specification _logger.debug("setValues[%d] %d:%d" % (fx, address, len(values))) - self.__set_callbacks[self.decode(fx)](address, values) + self._set_callbacks[self.decode(fx)](address, values) #--------------------------------------------------------------------------# # Redis Helper Methods #--------------------------------------------------------------------------# - def __get_prefix(self, key): + def _get_prefix(self, key): ''' This is a helper to abstract getting bit values :param key: The key prefix to use @@ -88,52 +88,52 @@ def __get_prefix(self, key): ''' return "%s:%s" % (self.prefix, key) - def __build_mapping(self): + def _build_mapping(self): ''' A quick helper method to build the function code mapper. ''' - self.__val_callbacks = { - 'd' : lambda o, c: self.__val_bit('d', o, c), - 'c' : lambda o, c: self.__val_bit('c', o, c), - 'h' : lambda o, c: self.__val_reg('h', o, c), - 'i' : lambda o, c: self.__val_reg('i', o, c), + self._val_callbacks = { + 'd' : lambda o, c: self._val_bit('d', o, c), + 'c' : lambda o, c: self._val_bit('c', o, c), + 'h' : lambda o, c: self._val_reg('h', o, c), + 'i' : lambda o, c: self._val_reg('i', o, c), } - self.__get_callbacks = { - 'd' : lambda o, c: self.__get_bit('d', o, c), - 'c' : lambda o, c: self.__get_bit('c', o, c), - 'h' : lambda o, c: self.__get_reg('h', o, c), - 'i' : lambda o, c: self.__get_reg('i', o, c), + self._get_callbacks = { + 'd' : lambda o, c: self._get_bit('d', o, c), + 'c' : lambda o, c: self._get_bit('c', o, c), + 'h' : lambda o, c: self._get_reg('h', o, c), + 'i' : lambda o, c: self._get_reg('i', o, c), } - self.__set_callbacks = { - 'd' : lambda o, v: self.__set_bit('d', o, v), - 'c' : lambda o, v: self.__set_bit('c', o, v), - 'h' : lambda o, v: self.__set_reg('h', o, v), - 'i' : lambda o, v: self.__set_reg('i', o, v), + self._set_callbacks = { + 'd' : lambda o, v: self._set_bit('d', o, v), + 'c' : lambda o, v: self._set_bit('c', o, v), + 'h' : lambda o, v: self._set_reg('h', o, v), + 'i' : lambda o, v: self._set_reg('i', o, v), } #--------------------------------------------------------------------------# # Redis discrete implementation #--------------------------------------------------------------------------# - __bit_size = 16 - __bit_default = '\x00' * (__bit_size % 8) + _bit_size = 16 + _bit_default = '\x00' * (_bit_size % 8) - def __get_bit_values(self, key, offset, count): + def _get_bit_values(self, key, offset, count): ''' This is a helper to abstract getting bit values :param key: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read ''' - key = self.__get_prefix(key) - s = divmod(offset, self.__bit_size)[0] - e = divmod(offset + count, self.__bit_size)[0] + key = self._get_prefix(key) + s = divmod(offset, self._bit_size)[0] + e = divmod(offset + count, self._bit_size)[0] request = ('%s:%s' % (key, v) for v in range(s, e + 1)) response = self.client.mget(request) return response - def __val_bit(self, key, offset, count): + def _val_bit(self, key, offset, count): ''' Validates that the given range is currently set in redis. If any of the keys return None, then it is invalid. @@ -141,23 +141,23 @@ def __val_bit(self, key, offset, count): :param offset: The address offset to start at :param count: The number of bits to read ''' - response = self.__get_bit_values(key, offset, count) - return None not in response + response = self._get_bit_values(key, offset, count) + return True if None not in response else False - def __get_bit(self, key, offset, count): + def _get_bit(self, key, offset, count): ''' :param key: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read ''' - response = self.__get_bit_values(key, offset, count) - response = (r or self.__bit_default for r in response) + response = self._get_bit_values(key, offset, count) + response = (r or self._bit_default for r in response) result = ''.join(response) result = unpack_bitstring(result) return result[offset:offset + count] - def __set_bit(self, key, offset, values): + def _set_bit(self, key, offset, values): ''' :param key: The key prefix to use @@ -165,17 +165,17 @@ def __set_bit(self, key, offset, values): :param values: The values to set ''' count = len(values) - s = divmod(offset, self.__bit_size)[0] - e = divmod(offset + count, self.__bit_size)[0] + s = divmod(offset, self._bit_size)[0] + e = divmod(offset + count, self._bit_size)[0] value = pack_bitstring(values) - current = self.__get_bit_values(key, offset, count) - current = (r or self.__bit_default for r in current) + current = self._get_bit_values(key, offset, count) + current = (r or self._bit_default for r in current) current = ''.join(current) current = current[0:offset] + value + current[offset + count:] - final = (current[s:s + self.__bit_size] for s in range(0, count, self.__bit_size)) + final = (current[s:s + self._bit_size] for s in range(0, count, self._bit_size)) - key = self.__get_prefix(key) + key = self._get_prefix(key) request = ('%s:%s' % (key, v) for v in range(s, e + 1)) request = dict(zip(request, final)) self.client.mset(request) @@ -183,17 +183,17 @@ def __set_bit(self, key, offset, values): #--------------------------------------------------------------------------# # Redis register implementation #--------------------------------------------------------------------------# - __reg_size = 16 - __reg_default = '\x00' * (__reg_size % 8) + _reg_size = 16 + _reg_default = '\x00' * (_reg_size % 8) - def __get_reg_values(self, key, offset, count): + def _get_reg_values(self, key, offset, count): ''' This is a helper to abstract getting register values :param key: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read ''' - key = self.__get_prefix(key) + key = self._get_prefix(key) #s = divmod(offset, self.__reg_size)[0] #e = divmod(offset+count, self.__reg_size)[0] @@ -202,7 +202,7 @@ def __get_reg_values(self, key, offset, count): response = self.client.mget(request) return response - def __val_reg(self, key, offset, count): + def _val_reg(self, key, offset, count): ''' Validates that the given range is currently set in redis. If any of the keys return None, then it is invalid. @@ -210,21 +210,21 @@ def __val_reg(self, key, offset, count): :param offset: The address offset to start at :param count: The number of bits to read ''' - response = self.__get_reg_values(key, offset, count) + response = self._get_reg_values(key, offset, count) return None not in response - def __get_reg(self, key, offset, count): + def _get_reg(self, key, offset, count): ''' :param key: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read ''' - response = self.__get_reg_values(key, offset, count) - response = [r or self.__reg_default for r in response] + response = self._get_reg_values(key, offset, count) + response = [r or self._reg_default for r in response] return response[offset:offset + count] - def __set_reg(self, key, offset, values): + def _set_reg(self, key, offset, values): ''' :param key: The key prefix to use @@ -237,7 +237,7 @@ def __set_reg(self, key, offset, values): #current = self.__get_reg_values(key, offset, count) - key = self.__get_prefix(key) + key = self._get_prefix(key) request = ('%s:%s' % (key, v) for v in range(offset, count + 1)) request = dict(zip(request, values)) self.client.mset(request) diff --git a/pymodbus/utilities.py b/pymodbus/utilities.py index e3ef421e2..d0c45e170 100644 --- a/pymodbus/utilities.py +++ b/pymodbus/utilities.py @@ -96,8 +96,8 @@ def unpack_bitstring(string): def make_byte_string(s): """ Returns byte string from a given string, python3 specific fix - :param s: - :return: + :param s: + :return: """ if IS_PYTHON3 and isinstance(s, string_types): s = s.encode() From 83f349e1b1ca5fbb9db5f9e6ef29f443c068e91d Mon Sep 17 00:00:00 2001 From: rahul Date: Mon, 27 Nov 2017 12:51:48 +0530 Subject: [PATCH 30/33] Added tests for redis datastore helpers --- test/test_datastore.py | 121 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/test/test_datastore.py b/test/test_datastore.py index b6b401517..94e7742e0 100644 --- a/test/test_datastore.py +++ b/test/test_datastore.py @@ -1,7 +1,12 @@ #!/usr/bin/env python import unittest +import mock +from mock import MagicMock +import redis from pymodbus.datastore import * from pymodbus.datastore.store import BaseModbusDataBlock +from pymodbus.datastore.database import SqlSlaveContext +from pymodbus.datastore.database import RedisSlaveContext from pymodbus.exceptions import NotImplementedException from pymodbus.exceptions import NoSuchSlaveException from pymodbus.exceptions import ParameterException @@ -113,7 +118,7 @@ def testModbusSlaveContext(self): } context = ModbusSlaveContext(**store) self.assertNotEqual(str(context), None) - + for fx in [1,2,3,4]: context.setValues(fx, 0, [True]*10) self.assertTrue(context.validate(fx, 0,10)) @@ -132,6 +137,120 @@ def _set(ctx): self.assertRaises(NoSuchSlaveException, lambda: _set(context)) self.assertRaises(NoSuchSlaveException, lambda: context[0xffff]) + +class RedisDataStoreTest(unittest.TestCase): + ''' + This is the unittest for the pymodbus.datastore.database.redis module + ''' + + def setUp(self): + self.slave = RedisSlaveContext() + + def tearDown(self): + ''' Cleans up the test environment ''' + pass + + def testStr(self): + # slave = RedisSlaveContext() + self.assertEqual(str(self.slave), "Redis Slave Context %s" % self.slave.client) + + def testReset(self): + assert isinstance(self.slave.client, redis.Redis) + self.slave.client = MagicMock() + self.slave.reset() + self.slave.client.flushall.assert_called_once_with() + + def testValCallbacksSuccess(self): + self.slave._build_mapping() + mock_count = 3 + mock_offset = 0 + self.slave.client.mset = MagicMock() + self.slave.client.mget = MagicMock(return_value=['11']) + + for key in ('d', 'c', 'h', 'i'): + self.assertTrue( + self.slave._val_callbacks[key](mock_offset, mock_count) + ) + + def testValCallbacksFailure(self): + self.slave._build_mapping() + mock_count = 3 + mock_offset = 0 + self.slave.client.mset = MagicMock() + self.slave.client.mget = MagicMock(return_value=['11', None]) + + for key in ('d', 'c', 'h', 'i'): + self.assertFalse( + self.slave._val_callbacks[key](mock_offset, mock_count) + ) + + def testGetCallbacks(self): + self.slave._build_mapping() + mock_count = 3 + mock_offset = 0 + self.slave.client.mset = MagicMock() + self.slave.client.mget = MagicMock(return_value="11") + + for key in ('d', 'c'): + resp = self.slave._get_callbacks[key](mock_offset, mock_count) + self.assertEqual(resp, [True, False, False]) + + for key in ('h', 'i'): + resp = self.slave._get_callbacks[key](mock_offset, mock_count) + self.assertEqual(resp, ['1', '1']) + + def testSetCallbacks(self): + self.slave._build_mapping() + mock_values = [3] + mock_offset = 0 + self.slave.client.mset = MagicMock() + self.slave.client.mget = MagicMock() + + for key in ['c', 'd']: + self.slave._set_callbacks[key](mock_offset, [3]) + k = "pymodbus:{}:{}".format(key, mock_offset) + self.slave.client.mset.assert_called_with( + {k: '\x01'} + ) + + for key in ('h', 'i'): + self.slave._set_callbacks[key](mock_offset, [3]) + k = "pymodbus:{}:{}".format(key, mock_offset) + self.slave.client.mset.assert_called_with( + {k: mock_values[0]} + ) + + def testValidate(self): + self.slave.client.mget = MagicMock(return_value=[123]) + self.assertTrue(self.slave.validate(0x01, 3000)) + + def testSetValue(self): + self.slave.client.mset = MagicMock() + self.slave.client.mget = MagicMock() + self.assertEqual(self.slave.setValues(0x01, 1000, [12]), None) + + def testGetValue(self): + self.slave.client.mget = MagicMock(return_value=["123"]) + self.assertEqual(self.slave.getValues(0x01, 23), []) + + +class SqlDataStoreTest(unittest.TestCase): + ''' + This is the unittest for the pymodbus.datastore.database.SqlSlaveContesxt + module + ''' + + def setUp(self): + pass + + def tearDown(self): + ''' Cleans up the test environment ''' + pass + + def testStr(self): + pass + + #---------------------------------------------------------------------------# # Main #---------------------------------------------------------------------------# From f6b45f8f90fbaef041a34d5f4892d15ff95794e5 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 28 Nov 2017 10:42:48 +0530 Subject: [PATCH 31/33] Minor fixes to SQL datastore --- pymodbus/datastore/database/sql_datastore.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pymodbus/datastore/database/sql_datastore.py b/pymodbus/datastore/database/sql_datastore.py index c1e7ddf5d..a02894251 100644 --- a/pymodbus/datastore/database/sql_datastore.py +++ b/pymodbus/datastore/database/sql_datastore.py @@ -100,7 +100,6 @@ def _db_create(self, table, database): def _get(self, type, offset, count): ''' - :param type: The key prefix to use :param offset: The address offset to start at :param count: The number of bits to read @@ -115,18 +114,19 @@ def _get(self, type, offset, count): result = self._connection.execute(query).fetchall() return [row.value for row in result] - def _build_set(self, type, offset, values, p=''): + def _build_set(self, type, offset, values, prefix=''): ''' A helper method to generate the sql update context :param type: The key prefix to use :param offset: The address offset to start at :param values: The values to set + :param prefix: Prefix fields index and type, defaults to empty string ''' result = [] for index, value in enumerate(values): result.append({ - p + 'type' : type, - p + 'index' : offset + index, + prefix + 'type' : type, + prefix + 'index' : offset + index, 'value' : value }) return result @@ -157,7 +157,7 @@ def _update(self, type, offset, values): :param offset: The address offset to start at :param values: The values to set ''' - context = self._build_set(type, offset, values, p='x_') + context = self._build_set(type, offset, values, prefix='x_') query = self._table.update().values(name='value') query = query.where(and_( self._table.c.type == bindparam('x_type'), From d9f1043438cacb8d79373c8020aad910a43d5c7c Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 28 Nov 2017 10:43:09 +0530 Subject: [PATCH 32/33] Unit tests for SQL datastore - 100% coverage --- test/test_datastore.py | 122 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 2 deletions(-) diff --git a/test/test_datastore.py b/test/test_datastore.py index 94e7742e0..f42003825 100644 --- a/test/test_datastore.py +++ b/test/test_datastore.py @@ -3,6 +3,7 @@ import mock from mock import MagicMock import redis +import random from pymodbus.datastore import * from pymodbus.datastore.store import BaseModbusDataBlock from pymodbus.datastore.database import SqlSlaveContext @@ -234,6 +235,12 @@ def testGetValue(self): self.assertEqual(self.slave.getValues(0x01, 23), []) +class MockSqlResult(object): + def __init__(self, rowcount=0, value=0): + self.rowcount = rowcount + self.value = value + + class SqlDataStoreTest(unittest.TestCase): ''' This is the unittest for the pymodbus.datastore.database.SqlSlaveContesxt @@ -241,15 +248,126 @@ class SqlDataStoreTest(unittest.TestCase): ''' def setUp(self): - pass + self.slave = SqlSlaveContext() + self.slave._metadata.drop_all = MagicMock() + self.slave._db_create = MagicMock() + self.slave._table = MagicMock() + self.slave._connection = MagicMock() + + self.mock_addr = random.randint(0, 65000) + self.mock_values = random.sample(range(1, 100), 5) + self.mock_function = 0x01 + self.mock_type = 'h' + self.mock_offset = 0 + self.mock_count = 1 + + self.function_map = {2: 'd', 4: 'i'} + self.function_map.update([(i, 'h') for i in [3, 6, 16, 22, 23]]) + self.function_map.update([(i, 'c') for i in [1, 5, 15]]) def tearDown(self): ''' Cleans up the test environment ''' pass def testStr(self): - pass + self.assertEqual(str(self.slave), "Modbus Slave Context") + + def testReset(self): + self.slave.reset() + + self.slave._metadata.drop_all.assert_called_once_with() + self.slave._db_create.assert_called_once_with( + self.slave.table, self.slave.database + ) + def testValidateSuccess(self): + mock_result = MockSqlResult( + rowcount=len(self.mock_values) + ) + self.slave._connection.execute = MagicMock(return_value=mock_result) + self.assertTrue(self.slave.validate( + self.mock_function, self.mock_addr, len(self.mock_values)) + ) + + def testValidateFailure(self): + wrong_count = 9 + mock_result = MockSqlResult(rowcount=len(self.mock_values)) + self.slave._connection.execute = MagicMock(return_value=mock_result) + self.assertFalse(self.slave.validate( + self.mock_function, self.mock_addr, wrong_count) + ) + + def testBuildSet(self): + mock_set = [ + { + 'index': 0, + 'type': 'h', + 'value': 11 + }, + { + 'index': 1, + 'type': 'h', + 'value': 12 + } + ] + self.assertListEqual(self.slave._build_set('h', 0, [11, 12]), mock_set) + + def testCheckSuccess(self): + mock_success_results = [1, 2, 3] + self.slave._get = MagicMock(return_value=mock_success_results) + self.assertFalse(self.slave._check('h', 0, 1)) + + def testCheckFailure(self): + mock_success_results = [] + self.slave._get = MagicMock(return_value=mock_success_results) + self.assertTrue(self.slave._check('h', 0, 1)) + + def testGetValues(self): + self.slave._get = MagicMock() + + for key, value in self.function_map.items(): + self.slave.getValues(key, self.mock_addr, self.mock_count) + self.slave._get.assert_called_with( + value, self.mock_addr + 1, self.mock_count + ) + + def testSetValues(self): + self.slave._set = MagicMock() + + for key, value in self.function_map.items(): + self.slave.setValues(key, self.mock_addr, self.mock_values) + self.slave._set.assert_called_with( + value, self.mock_addr + 1, self.mock_values + ) + def testSet(self): + self.slave._check = MagicMock(return_value=True) + self.slave._connection.execute = MagicMock( + return_value=MockSqlResult(rowcount=len(self.mock_values)) + ) + self.assertTrue(self.slave._set( + self.mock_type, self.mock_offset, self.mock_values) + ) + + self.slave._check = MagicMock(return_value=False) + self.assertFalse( + self.slave._set(self.mock_type, self.mock_offset, self.mock_values) + ) + + def testUpdateSuccess(self): + self.slave._connection.execute = MagicMock( + return_value=MockSqlResult(rowcount=len(self.mock_values)) + ) + self.assertTrue( + self.slave._update(self.mock_type, self.mock_offset, self.mock_values) + ) + + def testUpdateFailure(self): + self.slave._connection.execute = MagicMock( + return_value=MockSqlResult(rowcount=100) + ) + self.assertFalse( + self.slave._update(self.mock_type, self.mock_offset, self.mock_values) + ) #---------------------------------------------------------------------------# # Main From f527c1d43cf5df459c019f1ee75a35745b7c4042 Mon Sep 17 00:00:00 2001 From: rahul Date: Tue, 28 Nov 2017 11:17:03 +0530 Subject: [PATCH 33/33] Tests now compatible with python3 and python2 --- pymodbus/datastore/database/redis_datastore.py | 2 +- pymodbus/utilities.py | 5 ++++- test/test_datastore.py | 5 ++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pymodbus/datastore/database/redis_datastore.py b/pymodbus/datastore/database/redis_datastore.py index 634dc1552..b7c74b013 100644 --- a/pymodbus/datastore/database/redis_datastore.py +++ b/pymodbus/datastore/database/redis_datastore.py @@ -172,7 +172,7 @@ def _set_bit(self, key, offset, values): current = self._get_bit_values(key, offset, count) current = (r or self._bit_default for r in current) current = ''.join(current) - current = current[0:offset] + value + current[offset + count:] + current = current[0:offset] + value.decode('utf-8') + current[offset + count:] final = (current[s:s + self._bit_size] for s in range(0, count, self._bit_size)) key = self._get_prefix(key) diff --git a/pymodbus/utilities.py b/pymodbus/utilities.py index d0c45e170..a15515acf 100644 --- a/pymodbus/utilities.py +++ b/pymodbus/utilities.py @@ -86,7 +86,10 @@ def unpack_bitstring(string): byte_count = len(string) bits = [] for byte in range(byte_count): - value = byte2int(string[byte]) + if IS_PYTHON3: + value = byte2int(int(string[byte])) + else: + value = byte2int(string[byte]) for _ in range(8): bits.append((value & 1) == 1) value >>= 1 diff --git a/test/test_datastore.py b/test/test_datastore.py index f42003825..c1d99c33c 100644 --- a/test/test_datastore.py +++ b/test/test_datastore.py @@ -189,8 +189,7 @@ def testGetCallbacks(self): self.slave._build_mapping() mock_count = 3 mock_offset = 0 - self.slave.client.mset = MagicMock() - self.slave.client.mget = MagicMock(return_value="11") + self.slave.client.mget = MagicMock(return_value='11') for key in ('d', 'c'): resp = self.slave._get_callbacks[key](mock_offset, mock_count) @@ -251,7 +250,7 @@ def setUp(self): self.slave = SqlSlaveContext() self.slave._metadata.drop_all = MagicMock() self.slave._db_create = MagicMock() - self.slave._table = MagicMock() + self.slave._table.select = MagicMock() self.slave._connection = MagicMock() self.mock_addr = random.randint(0, 65000)