diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a08f75b10..3b35144d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,8 @@ jobs: name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} runs-on: ${{ matrix.os.runs-on }} container: ${{ matrix.os.container[matrix.python.docker] }} + # present runtime seems to be about 1 minute 30 seconds + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -30,7 +32,6 @@ jobs: os: - name: Linux runs-on: ubuntu-latest - python_platform: linux matrix: linux container: 2.7: docker://python:2.7-buster @@ -40,55 +41,67 @@ jobs: 3.9: docker://python:3.9-buster pypy2: docker://pypy:2-jessie pypy3: docker://pypy:3-stretch -# - name: Windows -# runs-on: windows-latest -# python_platform: win32 -# matrix: windows -# - name: macOS -# runs-on: macos-latest -# python_platform: darwin -# matrix: macos + - name: macOS + runs-on: macos-latest + matrix: macos + - name: Windows + runs-on: windows-latest + matrix: windows + openssl: + x86: win32 + x64: win64 python: - name: CPython 2.7 tox: py27 action: 2.7 docker: 2.7 + matrix: 2.7 implementation: cpython - name: PyPy 2.7 tox: pypy27 action: pypy-2.7 docker: pypy2.7 + matrix: 2.7 implementation: pypy + openssl_msvc_version: 2019 - name: CPython 3.6 tox: py36 action: 3.6 docker: 3.6 + matrix: 3.6 implementation: cpython - name: CPython 3.7 tox: py37 action: 3.7 docker: 3.7 + matrix: 3.7 implementation: cpython - name: CPython 3.8 tox: py38 action: 3.8 docker: 3.8 + matrix: 3.8 implementation: cpython - name: CPython 3.9 tox: py39 action: 3.9 docker: 3.9 + matrix: 3.9 implementation: cpython - name: PyPy 3.6 tox: pypy36 action: pypy-3.6 docker: pypy3.6 + matrix: 3.6 implementation: pypy + openssl_msvc_version: 2019 - name: PyPy 3.7 tox: pypy37 action: pypy-3.7 docker: pypy3.7 + matrix: 3.7 implementation: pypy + openssl_msvc_version: 2019 arch: - name: x86 action: x86 @@ -105,6 +118,12 @@ jobs: matrix: macos arch: matrix: x86 + - os: + matrix: windows + python: + implementation: pypy + arch: + matrix: x64 env: # Should match name above JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} @@ -129,7 +148,37 @@ jobs: pip install --upgrade pip setuptools wheel pip install --upgrade tox - uses: twisted/python-info-action@v1.0.1 + - name: Add PyPy Externals + if: ${{ matrix.os.matrix == 'windows' && matrix.python.implementation == 'pypy'}} + env: + PYPY_EXTERNALS_PATH: ${{ github.workspace }}/pypy_externals + shell: bash + run: | + echo $PYPY_EXTERNALS_PATH + mkdir --parents $(dirname $PYPY_EXTERNALS_PATH) + hg clone https://foss.heptapod.net/pypy/externals/ $PYPY_EXTERNALS_PATH + dir $PYPY_EXTERNALS_PATH + cd $PYPY_EXTERNALS_PATH && hg update win32_14x + echo "INCLUDE=$PYPY_EXTERNALS_PATH/include;$INCLUDE" >> $GITHUB_ENV + echo "LIB=$PYPY_EXTERNALS_PATH/lib;$LIB" >> $GITHUB_ENV +# echo "CL=${{ matrix.PYTHON.CL_FLAGS }}" >> $GITHUB_ENV + - name: Add Brew + if: ${{ matrix.os.matrix == 'macos' && matrix.python.implementation == 'pypy'}} + shell: bash + run: | + brew install openssl@1.1 rust + echo "LDFLAGS=-L$(brew --prefix openssl@1.1)/lib" >> $GITHUB_ENV + echo "CFLAGS=-I$(brew --prefix openssl@1.1)/include" >> $GITHUB_ENV + - name: rustup + if: ${{ matrix.os.matrix == 'windows' && matrix.python.implementation == 'pypy'}} + shell: bash + run: | + rustup target add i686-pc-windows-msvc - name: Test + env: + # When compiling Cryptography for PyPy on Windows there is a cleanup + # failure. This is CI, it doesn't matter. + PIP_NO_CLEAN: 1 run: | tox -vv -e ${{ matrix.python.tox }} - name: Coverage Processing @@ -161,7 +210,6 @@ jobs: os: - name: Linux runs-on: ubuntu-latest - python_platform: linux matrix: linux container: 3.8: docker://python:3.8-buster @@ -195,6 +243,7 @@ jobs: # Should match JOB_NAME below name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }} runs-on: ${{ matrix.os.runs-on }} + if: always() needs: - test container: ${{ matrix.os.container[matrix.python.docker] }} @@ -208,7 +257,6 @@ jobs: os: - name: Linux runs-on: ubuntu-latest - python_platform: linux matrix: linux container: 3.8: docker://python:3.8-buster diff --git a/pymodbus/server/sync.py b/pymodbus/server/sync.py index 442fa6dd1..041069aab 100644 --- a/pymodbus/server/sync.py +++ b/pymodbus/server/sync.py @@ -327,16 +327,17 @@ def __init__(self, context, framer=None, identity=None, self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) self.handler = handler or ModbusConnectedRequestHandler - self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + self.ignore_missing_slaves = kwargs.pop('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) - self.broadcast_enable = kwargs.get('broadcast_enable', + self.broadcast_enable = kwargs.pop('broadcast_enable', Defaults.broadcast_enable) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) socketserver.ThreadingTCPServer.__init__(self, self.address, - self.handler) + self.handler, + **kwargs) def process_request(self, request, client): """ Callback for connecting a new client thread @@ -456,16 +457,16 @@ def __init__(self, context, framer=None, identity=None, address=None, self.control = ModbusControlBlock() self.address = address or ("", Defaults.Port) self.handler = handler or ModbusDisconnectedRequestHandler - self.ignore_missing_slaves = kwargs.get('ignore_missing_slaves', + self.ignore_missing_slaves = kwargs.pop('ignore_missing_slaves', Defaults.IgnoreMissingSlaves) - self.broadcast_enable = kwargs.get('broadcast_enable', + self.broadcast_enable = kwargs.pop('broadcast_enable', Defaults.broadcast_enable) if isinstance(identity, ModbusDeviceIdentification): self.control.Identity.update(identity) socketserver.ThreadingUDPServer.__init__(self, - self.address, self.handler) + self.address, self.handler, **kwargs) # self._BaseServer__shutdown_request = True def process_request(self, request, client): diff --git a/requirements-tests.txt b/requirements-tests.txt index 5eca1922c..ad0d59022 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -14,6 +14,11 @@ sqlalchemy>=1.1.15 #wsgiref>=0.1.2 verboselogs >= 1.5 tornado==4.5.3 -Twisted[serial]>=20.3.0 +# using platform_python_implementation rather than +# implementation_name for Python 2 support +Twisted[conch,serial]>=20.3.0; platform_python_implementation != "PyPy" or sys_platform != "win32" +# pywin32 isn't supported on pypy +# https://github.com/mhammond/pywin32/issues/1289 +Twisted[conch]>=20.3.0; platform_python_implementation == "PyPy" and sys_platform == "win32" zope.interface>=4.4.0 asynctest>=0.10.0 diff --git a/setup.py b/setup.py index d20e1f572..89c16326a 100644 --- a/setup.py +++ b/setup.py @@ -97,8 +97,12 @@ 'sphinx_rtd_theme', 'humanfriendly'], 'twisted': [ - 'twisted[serial] >= 20.3.0', - 'pyasn1 >= 0.1.4', + # using platform_python_implementation rather than + # implementation_name for Python 2 support + 'Twisted[conch,serial]>=20.3.0; platform_python_implementation != "PyPy" or sys_platform != "win32"', + # pywin32 isn't supported on pypy + # https://github.com/mhammond/pywin32/issues/1289 + 'Twisted[conch]>=20.3.0; platform_python_implementation == "PyPy" and sys_platform == "win32"', ], 'tornado': [ 'tornado == 4.5.3' diff --git a/test/test_client_async.py b/test/test_client_async.py index 97aaae8bd..34a1e2db3 100644 --- a/test/test_client_async.py +++ b/test/test_client_async.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +import contextlib +import sys import unittest import pytest from pymodbus.compat import IS_PYTHON3, PYTHON_VERSION @@ -30,13 +32,18 @@ import ssl IS_DARWIN = platform.system().lower() == "darwin" +IS_WINDOWS = platform.system().lower() == "windows" OSX_SIERRA = LooseVersion("10.12") if IS_DARWIN: IS_HIGH_SIERRA_OR_ABOVE = LooseVersion(platform.mac_ver()[0]) SERIAL_PORT = '/dev/ttyp0' if not IS_HIGH_SIERRA_OR_ABOVE else '/dev/ptyp0' else: IS_HIGH_SIERRA_OR_ABOVE = False - SERIAL_PORT = "/dev/ptmx" + if IS_WINDOWS: + # the use is mocked out + SERIAL_PORT = "" + else: + SERIAL_PORT = "/dev/ptmx" # ---------------------------------------------------------------------------# # Fixture @@ -47,6 +54,15 @@ def mock_asyncio_gather(coro): return coro +@contextlib.contextmanager +def maybe_manage(condition, manager): + if condition: + with manager as value: + yield value + else: + yield None + + class TestAsynchronousClient(object): """ This is the unittest for the pymodbus.client.asynchronous module @@ -175,6 +191,10 @@ def testUdpAsycioClient(self, mock_gather, mock_event_loop): # Test Serial client # -----------------------------------------------------------------------# + @pytest.mark.skipif( + sys.platform == 'win32' and platform.python_implementation() == 'PyPy', + reason='Twisted serial requires pywin32 which is not compatible with PyPy', + ) @pytest.mark.parametrize("method, framer", [("rtu", ModbusRtuFramer), ("socket", ModbusSocketFramer), ("binary", ModbusBinaryFramer), @@ -185,30 +205,30 @@ def testSerialTwistedClient(self, method, framer): with patch("serial.Serial") as mock_sp: from twisted.internet import reactor from twisted.internet.serialport import SerialPort + with maybe_manage(sys.platform == 'win32', patch.object(SerialPort, "_finishPortSetup")): + with patch('twisted.internet.reactor') as mock_reactor: - with patch('twisted.internet.reactor') as mock_reactor: - - protocol, client = AsyncModbusSerialClient(schedulers.REACTOR, - method=method, - port=SERIAL_PORT, - proto_cls=ModbusSerClientProtocol) + protocol, client = AsyncModbusSerialClient(schedulers.REACTOR, + method=method, + port=SERIAL_PORT, + proto_cls=ModbusSerClientProtocol) - assert (isinstance(client, SerialPort)) - assert (isinstance(client.protocol, ModbusSerClientProtocol)) - assert (0 == len(list(client.protocol.transaction))) - assert (isinstance(client.protocol.framer, framer)) - assert (client.protocol._connected) + assert (isinstance(client, SerialPort)) + assert (isinstance(client.protocol, ModbusSerClientProtocol)) + assert (0 == len(list(client.protocol.transaction))) + assert (isinstance(client.protocol.framer, framer)) + assert (client.protocol._connected) - def handle_failure(failure): - assert (isinstance(failure.exception(), ConnectionException)) + def handle_failure(failure): + assert (isinstance(failure.exception(), ConnectionException)) - d = client.protocol._buildResponse(0x00) - d.addCallback(handle_failure) + d = client.protocol._buildResponse(0x00) + d.addCallback(handle_failure) - assert (client.protocol._connected) - client.protocol.close() - protocol.stop() - assert (not client.protocol._connected) + assert (client.protocol._connected) + client.protocol.close() + protocol.stop() + assert (not client.protocol._connected) @pytest.mark.parametrize("method, framer", [("rtu", ModbusRtuFramer), ("socket", ModbusSocketFramer), @@ -216,24 +236,26 @@ def handle_failure(failure): ("ascii", ModbusAsciiFramer)]) def testSerialTornadoClient(self, method, framer): """ Test the serial tornado client client initialize """ - protocol, future = AsyncModbusSerialClient(schedulers.IO_LOOP, method=method, port=SERIAL_PORT) - client = future.result() - assert(isinstance(client, AsyncTornadoModbusSerialClient)) - assert(0 == len(list(client.transaction))) - assert(isinstance(client.framer, framer)) - assert(client.port == SERIAL_PORT) - assert(client._connected) - - def handle_failure(failure): - assert(isinstance(failure.exception(), ConnectionException)) - - d = client._build_response(0x00) - d.add_done_callback(handle_failure) - - assert(client._connected) - client.close() - protocol.stop() - assert(not client._connected) + from serial import Serial + with maybe_manage(sys.platform in ('darwin', 'win32'), patch.object(Serial, "open")): + protocol, future = AsyncModbusSerialClient(schedulers.IO_LOOP, method=method, port=SERIAL_PORT) + client = future.result() + assert(isinstance(client, AsyncTornadoModbusSerialClient)) + assert(0 == len(list(client.transaction))) + assert(isinstance(client.framer, framer)) + assert(client.port == SERIAL_PORT) + assert(client._connected) + + def handle_failure(failure): + assert(isinstance(failure.exception(), ConnectionException)) + + d = client._build_response(0x00) + d.add_done_callback(handle_failure) + + assert(client._connected) + client.close() + protocol.stop() + assert(not client._connected) @pytest.mark.skipif(IS_PYTHON3 , reason="requires python2.7") def testSerialAsyncioClientPython2(self): diff --git a/test/test_client_sync.py b/test/test_client_sync.py index 1b679ed85..d7c7f7be2 100755 --- a/test/test_client_sync.py +++ b/test/test_client_sync.py @@ -11,6 +11,9 @@ import socket import serial import ssl +import sys + +import pytest from pymodbus.client.sync import ModbusTcpClient, ModbusUdpClient from pymodbus.client.sync import ModbusSerialClient, BaseModbusClient @@ -47,6 +50,15 @@ def setblocking(self, flag): return None def in_waiting(self): return None +inet_pton_skipif = pytest.mark.skipif( + sys.platform == "win32" and sys.version_info < (3, 4), + reason=( + "Uses socket.inet_pton() which wasn't available on Windows until" + " 3.4.", + ) +) + + # ---------------------------------------------------------------------------# # Fixture @@ -128,6 +140,7 @@ def testBasicSyncUdpClient(self): self.assertEqual("ModbusUdpClient(127.0.0.1:502)", str(client)) + @inet_pton_skipif def testUdpClientAddressFamily(self): ''' Test the Udp client get address family method''' client = ModbusUdpClient() @@ -135,6 +148,7 @@ def testUdpClientAddressFamily(self): client._get_address_family('127.0.0.1')) self.assertEqual(socket.AF_INET6, client._get_address_family('::1')) + @inet_pton_skipif def testUdpClientConnect(self): ''' Test the Udp client connection method''' with patch.object(socket, 'socket') as mock_method: @@ -151,6 +165,7 @@ def settimeout(self, *a, **kwa): client = ModbusUdpClient() self.assertFalse(client.connect()) + @inet_pton_skipif def testUdpClientIsSocketOpen(self): ''' Test the udp client is_socket_open method''' client = ModbusUdpClient() diff --git a/test/test_server_async.py b/test/test_server_async.py index 4fe04add3..c88f6e4db 100644 --- a/test/test_server_async.py +++ b/test/test_server_async.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from pymodbus.compat import IS_PYTHON3 import unittest +import pytest if IS_PYTHON3: # Python 3 from unittest.mock import patch, Mock, MagicMock else: # Python 2 @@ -32,6 +33,11 @@ IS_HIGH_SIERRA_OR_ABOVE = False SERIAL_PORT = "/dev/ptmx" +no_twisted_serial_on_windows_with_pypy = pytest.mark.skipif( + sys.platform == 'win32' and platform.python_implementation() == 'PyPy', + reason='Twisted serial requires pywin32 which is not compatible with PyPy', +) + class AsynchronousServerTest(unittest.TestCase): ''' @@ -188,6 +194,7 @@ def testUdpServerStartup(self): self.assertEqual(mock_reactor.listenUDP.call_count, 1) self.assertEqual(mock_reactor.run.call_count, 1) + @no_twisted_serial_on_windows_with_pypy @patch("twisted.internet.serialport.SerialPort") def testSerialServerStartup(self, mock_sp): ''' Test that the modbus serial asynchronous server starts correctly ''' @@ -195,6 +202,7 @@ def testSerialServerStartup(self, mock_sp): StartSerialServer(context=None, port=SERIAL_PORT) self.assertEqual(mock_reactor.run.call_count, 1) + @no_twisted_serial_on_windows_with_pypy @patch("twisted.internet.serialport.SerialPort") def testStopServerFromMainThread(self, mock_sp): """ @@ -207,6 +215,7 @@ def testStopServerFromMainThread(self, mock_sp): StopServer() self.assertEqual(mock_reactor.stop.call_count, 1) + @no_twisted_serial_on_windows_with_pypy @patch("twisted.internet.serialport.SerialPort") def testStopServerFromThread(self, mock_sp): """ diff --git a/test/test_server_asyncio.py b/test/test_server_asyncio.py index 84c1b025f..c0b9b647d 100755 --- a/test/test_server_asyncio.py +++ b/test/test_server_asyncio.py @@ -216,7 +216,9 @@ def connection_made(self, transport): transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1', port=random_port) yield from step1 - # await asyncio.sleep(1) + # On Windows we seem to need to give this an extra chance to finish, + # otherwise there ends up being an active connection at the assert. + yield from asyncio.sleep(0.0) self.assertTrue(len(server.active_connections) == 1) protocol.transport.close() # close isn't synchronous and there's no notification that it's done @@ -252,6 +254,9 @@ def connection_made(self, transport): transport, protocol = yield from self.loop.create_connection(BasicClient, host='127.0.0.1',port=random_port) yield from step1 + # On Windows we seem to need to give this an extra chance to finish, + # otherwise there ends up being an active connection at the assert. + yield from asyncio.sleep(0.0) server.server_close() # close isn't synchronous and there's no notification that it's done diff --git a/test/test_server_sync.py b/test/test_server_sync.py index 526da1a5f..8b37b9ac2 100644 --- a/test/test_server_sync.py +++ b/test/test_server_sync.py @@ -260,13 +260,12 @@ def _callback(a, b): #-----------------------------------------------------------------------# def testTcpServerClose(self): ''' test that the synchronous TCP server closes correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: - identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusTcpServer(context=None, identity=identity) - server.threads.append(Mock(**{'running': True})) - server.server_close() - self.assertEqual(server.control.Identity.VendorName, 'VendorName') - self.assertFalse(server.threads[0].running) + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = ModbusTcpServer(context=None, identity=identity, bind_and_activate=False) + server.threads.append(Mock(**{'running': True})) + server.server_close() + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.assertFalse(server.threads[0].running) def testTcpServerProcess(self): ''' test that the synchronous TCP server processes requests ''' @@ -280,30 +279,33 @@ def testTcpServerProcess(self): #-----------------------------------------------------------------------# def testTlsServerInit(self): ''' test that the synchronous TLS server intial correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: + with patch.object(socketserver.TCPServer, 'server_activate'): with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusTlsServer(context=None, identity=identity) + server = ModbusTlsServer(context=None, identity=identity, + bind_and_activate=False) + server.server_activate() self.assertIsNotNone(server.sslctx) self.assertEqual(type(server.socket), ssl.SSLSocket) server.server_close() sslctx = ssl.create_default_context() server = ModbusTlsServer(context=None, identity=identity, - sslctx=sslctx) + sslctx=sslctx, bind_and_activate=False) + server.server_activate() self.assertEqual(server.sslctx, sslctx) self.assertEqual(type(server.socket), ssl.SSLSocket) server.server_close() def testTlsServerClose(self): ''' test that the synchronous TLS server closes correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: - with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: - identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusTlsServer(context=None, identity=identity) - server.threads.append(Mock(**{'running': True})) - server.server_close() - self.assertEqual(server.control.Identity.VendorName, 'VendorName') - self.assertFalse(server.threads[0].running) + with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = ModbusTlsServer(context=None, identity=identity, + bind_and_activate=False) + server.threads.append(Mock(**{'running': True})) + server.server_close() + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.assertFalse(server.threads[0].running) def testTlsServerProcess(self): ''' test that the synchronous TLS server processes requests ''' @@ -318,13 +320,14 @@ def testTlsServerProcess(self): #-----------------------------------------------------------------------# def testUdpServerClose(self): ''' test that the synchronous UDP server closes correctly ''' - with patch.object(socket.socket, 'bind') as mock_socket: - identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) - server = ModbusUdpServer(context=None, identity=identity) - server.threads.append(Mock(**{'running': True})) - server.server_close() - self.assertEqual(server.control.Identity.VendorName, 'VendorName') - self.assertFalse(server.threads[0].running) + identity = ModbusDeviceIdentification(info={0x00: 'VendorName'}) + server = ModbusUdpServer(context=None, identity=identity, + bind_and_activate=False) + server.server_activate() + server.threads.append(Mock(**{'running': True})) + server.server_close() + self.assertEqual(server.control.Identity.VendorName, 'VendorName') + self.assertFalse(server.threads[0].running) def testUdpServerProcess(self): ''' test that the synchronous UDP server processes requests ''' @@ -383,15 +386,13 @@ def testSerialServerClose(self): def testStartTcpServer(self): ''' Test the tcp server starting factory ''' with patch.object(ModbusTcpServer, 'serve_forever') as mock_server: - with patch.object(socketserver.TCPServer, 'server_bind') as mock_binder: - StartTcpServer() + StartTcpServer(bind_and_activate=False) def testStartTlsServer(self): ''' Test the tls server starting factory ''' with patch.object(ModbusTlsServer, 'serve_forever') as mock_server: - with patch.object(socketserver.TCPServer, 'server_bind') as mock_binder: - with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: - StartTlsServer() + with patch.object(ssl.SSLContext, 'load_cert_chain') as mock_method: + StartTlsServer(bind_and_activate=False) def testStartUdpServer(self): ''' Test the udp server starting factory ''' diff --git a/tox.ini b/tox.ini index d84c48458..ccd8544ce 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,10 @@ envlist = py{27,py27,36,37,38,39,py36,py37} deps = -r requirements-tests.txt commands = pytest {posargs:--cov=pymodbus/ --cov-report=term-missing --cov-report=xml} +passenv = + INCLUDE + LIB + PIP_* setenv = with_gmp=no