Fix the Python RPC tests on Big Endian architectures

.travis.yml

@@ -479,12 +479,21 @@ before_install:
  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew uninstall --ignore-dependencies gnupg; fi
 #Fix a broken homebrew python upgrade - see https://github.com/Homebrew/homebrew-core/issues/26358
  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew upgrade python || true; fi
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then if [ ! -d /usr/local/sbin ]; then sudo mkdir -p /usr/local/sbin && sudo chown -R $(whoami) /usr/local/sbin; fi; fi
- - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=/usr/local/opt/ccache/libexec:$PATH; fi # Use ccache on Mac too
+#Uninstall some homebrew packages which aren't required so we don't end up updating and rebuilding them. Force so it doesn't reinstall
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew uninstall --ignore-dependencies --force cgal liblwgeom libspatialite mercurial postgis postgres sfcgal; fi
+#Having fixed everything up, update to the bleeding edge
+#Skip updating for now as it seems to be breaking the build, also now done via the addon
+# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
  # This is no longer allowed, so build all Mac builds as protobuf latest
  #- if [ "$TRAVIS_OS_NAME" == "osx" -a "$PROTOBUF" != "latest" ]; then brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/024ca9a4730a1f26ceede43485fbf62ef6f41179%5E/Formula/protobuf@3.1.rb; fi # install protobuf 3.1 as that's the latest we support everywhere
  #- if [ "$TRAVIS_OS_NAME" == "osx" -a "$PROTOBUF" != "latest" ]; then brew link -f https://raw.githubusercontent.com/Homebrew/homebrew-core/024ca9a4730a1f26ceede43485fbf62ef6f41179%5E/Formula/protobuf@3.1.rb; export PKG_CONFIG_PATH=/usr/local/opt/protobuf@3.1/lib/pkgconfig; brew install --build-from-source --ignore-dependencies --env=std protobuf-c; fi # While protobuf is not on the latest release
+ - if [ "$TRAVIS_OS_NAME" == "osx" -a "$PROTOBUF" != "latest" ]; then brew install protobuf@3.6; fi
+ - if [ "$TRAVIS_OS_NAME" == "osx" -a "$PROTOBUF" != "latest" ]; then brew link -f protobuf@3.6; export PKG_CONFIG_PATH=/usr/local/opt/protobuf@3.6/lib/pkgconfig; brew install --build-from-source --ignore-dependencies --env=std protobuf-c; fi # When protobuf is not on the latest release
  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then mkdir -p ${HOME}/Library/Python/2.7/lib/python/site-packages; echo 'import site; site.addsitedir("/usr/local/lib/python2.7/site-packages")' >> ${HOME}/Library/Python/2.7/lib/python/site-packages/homebrew.pth; fi
+ - if [ "$TRAVIS_OS_NAME" == "osx" -a "$CPPUNIT" != "1.14" ]; then brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/e6e43cf6a3%5E/Formula/cppunit.rb; fi # install a slightly older cppunit, as latest needs C++11 support
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=/usr/local/opt/ccache/libexec:$PATH; fi # Use ccache on Mac too
+#Put back the old pip numpy we need to work
+ - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --upgrade --no-deps --force-reinstall --user numpy; fi
 #Coverity doesn't work with g++ 5 or 6, so only upgrade to g++ 4.9 for that
  - if [ "$TRAVIS_OS_NAME" == "linux" -a \( "$TASK" = "compile" -o "$TASK" = "coverage" -o "$TASK" = "doxygen" \) -a "$CXX" = "g++" ]; then export CXX="ccache g++-10" CC="ccache gcc-10"; fi
  - if [ "$TASK" = "coverity" -a "$CXX" = "g++" ]; then export CXX="g++-4.9" CC="gcc-4.9"; fi

NEWS

@@ -9,7 +9,8 @@ x/y/2020 ola-0.10.9
  * 
 
  Bugs:
- * 
+ * Renamed EndpointNoticationEvent(sic) to EndpointNotificationEvent in the
+   E1.33 EndpointManager code
 
  Internal:
  *

python/examples/Makefile.mk

@@ -3,6 +3,7 @@ dist_noinst_SCRIPTS += \
     python/examples/ola_artnet_params.py \
     python/examples/ola_candidate_ports.py \
     python/examples/ola_devices.py \
+    python/examples/ola_fetch_dmx.py \
     python/examples/ola_patch_unpatch.py \
     python/examples/ola_plugin_info.py \
     python/examples/ola_rdm_discover.py \

python/examples/ola_fetch_dmx.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# ola_fetch_dmx.py
+# Copyright (C) 2020 Peter Newman
+
+"""Gets a current frame of DMX for a universe."""
+
+from __future__ import print_function
+from ola.ClientWrapper import ClientWrapper
+import getopt
+import textwrap
+import sys
+
+__author__ = 'nomis52@gmail.com (Simon Newton)'
+
+wrapper = None
+
+
+def DMXData(status, universe, data):
+  if status.Succeeded():
+    print(data)
+
+  else:
+    print('Error: %s' % status.message, file=sys.stderr)
+
+  global wrapper
+  if wrapper:
+    wrapper.Stop()
+
+
+def Usage():
+  print(textwrap.dedent("""
+  Usage: ola_fetch_dmx.py --universe <universe>
+
+  Fetch the current DXM512 data for the universe and exit.
+
+  -h, --help                Display this help message and exit.
+  -u, --universe <universe> Universe number."""))
+
+
+def main():
+  try:
+      opts, args = getopt.getopt(sys.argv[1:], "hu:", ["help", "universe="])
+  except getopt.GetoptError as err:
+    print(str(err))
+    Usage()
+    sys.exit(2)
+
+  universe = 1
+  for o, a in opts:
+    if o in ("-h", "--help"):
+      Usage()
+      sys.exit()
+    elif o in ("-u", "--universe"):
+      universe = int(a)
+
+  global wrapper
+  wrapper = ClientWrapper()
+  client = wrapper.Client()
+  client.FetchDmx(universe, DMXData)
+  wrapper.Run()
+
+
+if __name__ == '__main__':
+  main()

python/ola/ClientWrapperTest.py

@@ -24,6 +24,7 @@
 import unittest
 from ola.ClientWrapper import ClientWrapper
 from ola.ClientWrapper import _Event
+from ola.TestUtils import handleRPCByteOrder
 
 
 """Test cases for the Event and Event loop of ClientWrapper class."""
@@ -192,22 +193,22 @@ def d():
     # Called immediately
     a_diff = results.a_called - self.start
     self.assertAlmostEqual(a_diff, datetime.timedelta(milliseconds=0),
-                           delta=datetime.timedelta(microseconds=500))
+                           delta=datetime.timedelta(microseconds=750))
 
     # Called in 5 milliseconds
     b_diff = results.b_called - self.start
     self.assertAlmostEqual(b_diff, datetime.timedelta(milliseconds=5),
-                           delta=datetime.timedelta(microseconds=500))
+                           delta=datetime.timedelta(microseconds=750))
 
     # Called in 10 milliseconds
     c_diff = results.c_called - self.start
     self.assertAlmostEqual(c_diff, datetime.timedelta(milliseconds=10),
-                           delta=datetime.timedelta(microseconds=500))
+                           delta=datetime.timedelta(microseconds=750))
 
     # Called in 15 milliseconds
     d_diff = results.d_called - self.start
     self.assertAlmostEqual(d_diff, datetime.timedelta(milliseconds=15),
-                           delta=datetime.timedelta(microseconds=500))
+                           delta=datetime.timedelta(microseconds=750))
 
     sockets[0].close()
     sockets[1].close()
@@ -223,12 +224,12 @@ class results:
 
     def DataCallback(self):
       data = sockets[1].recv(4096)
-      expected = binascii.unhexlify(
+      expected = handleRPCByteOrder(binascii.unhexlify(
         "7d000010080110001a0d557064617465446d784461746122680801126400000"
         "000000000000000000000000000000000000000000000000000000000000000"
         "000000000000000000000000000000000000000000000000000000000000000"
         "000000000000000000000000000000000000000000000000000000000000000"
-        "000000")
+        "000000"))
       self.assertEqual(data, expected,
                        msg="Regression check failed. If protocol change "
                        "was intended set expected to: " +
@@ -242,11 +243,78 @@ def DataCallback(self):
 
     wrapper.Run()
 
+    sockets[0].close()
+    sockets[1].close()
+
     self.assertTrue(results.gotdata)
 
+  # @timeout_decorator.timeout(2)
+  def testFetchDmx(self):
+    """uses client to send a FetchDMX with mocked olad.
+    Regression test that confirms sent message is correct and
+    sends fixed response message."""
+    sockets = socket.socketpair()
+    wrapper = ClientWrapper(sockets[0])
+    client = wrapper.Client()
+
+    class results:
+      got_request = False
+      got_response = False
+
+    def DataCallback(self):
+      # request and response for
+      # ola_fetch_dmx.py -u 0
+      # enable logging in rpc/StreamRpcChannel.py
+      data = sockets[1].recv(4096)
+      expected = handleRPCByteOrder(binascii.unhexlify(
+        "10000010080110001a06476574446d7822020800"))
+      self.assertEqual(data, expected,
+                       msg="Regression check failed. If protocol change "
+                       "was intended set expected to: " +
+                       str(binascii.hexlify(data)))
+      results.got_request = True
+      response = handleRPCByteOrder(binascii.unhexlify(
+          "0c020010080210002285040800128004"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000000000000000000000000000000000"
+          "00000000000000000000000000000000"))
+      sent_bytes = sockets[1].send(response)
+      self.assertEqual(sent_bytes, len(response))
+
+    def ResponseCallback(self, status, universe, data):
+      results.got_response = True
+      self.assertTrue(status.Succeeded())
+      self.assertEqual(universe, 0)
+      self.assertEqual(len(data), 512)
+      self.assertEqual(data, array.array('B', [0] * 512))
+      wrapper.AddEvent(0, wrapper.Stop)
+
+    wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self))
+
+    client.FetchDmx(0, lambda x, y, z: ResponseCallback(self, x, y, z))
+
+    wrapper.Run()
+
     sockets[0].close()
     sockets[1].close()
 
+    self.assertTrue(results.got_request)
+    self.assertTrue(results.got_response)
+
 
 if __name__ == '__main__':
   unittest.main()

python/ola/RDMTest.py

@@ -26,6 +26,7 @@
 from ola.OlaClient import RDMNack
 from ola.RDMAPI import RDMAPI
 from ola.UID import UID
+from ola.TestUtils import handleRPCByteOrder
 
 
 """Test cases for RDM device commands."""
@@ -57,18 +58,18 @@ def DataCallback(self):
       # against olad dummy plugin
       # enable logging in rpc/StreamRpcChannel.py
       data = sockets[1].recv(4096)
-      expected = binascii.unhexlify(
+      expected = handleRPCByteOrder(binascii.unhexlify(
         "29000010080110001a0a52444d436f6d6d616e6422170801120908f0f4011500"
-        "ffffff180020602a0030003800")
+        "ffffff180020602a0030003800"))
       self.assertEqual(data, expected,
                        msg="Regression check failed. If protocol change "
                        "was intended set expected to: " +
                        str(binascii.hexlify(data)))
       results.got_request = True
-      response = binascii.unhexlify(
+      response = handleRPCByteOrder(binascii.unhexlify(
         "3f0000100802100022390800100018002213010000017fff0000000300050204"
         "00010000032860300038004a0908f0f4011500ffffff520908f0f40115ac1100"
-        "02580a")
+        "02580a"))
       sent_bytes = sockets[1].send(response)
       self.assertEqual(sent_bytes, len(response))
 
@@ -124,18 +125,18 @@ def DataCallback(self):
       # against olad dummy plugin
       # enable logging in rpc/StreamRpcChannel.py
       data = sockets[1].recv(4096)
-      expected = binascii.unhexlify(
+      expected = handleRPCByteOrder(binascii.unhexlify(
         "2b000010080110001a0a52444d436f6d6d616e6422190801120908f0f4011500"
-        "ffffff180020e1012a010230003800")
+        "ffffff180020e1012a010230003800"))
       self.assertEqual(data, expected,
                        msg="Regression check failed. If protocol change "
                        "was intended set expected to: " +
                        str(binascii.hexlify(data)))
       results.got_request = True
-      response = binascii.unhexlify(
+      response = handleRPCByteOrder(binascii.unhexlify(
         "3d0000100802100022370800100018002210020005506572736f6e616c697479"
         "203228e101300038004a0908f0f4011500ffffff520908f0f40115ac107de058"
-        "29")
+        "29"))
       sent_bytes = sockets[1].send(response)
       self.assertEqual(sent_bytes, len(response))
 
@@ -184,17 +185,17 @@ def DataCallback(self):
       # against olad dummy plugin
       # enable logging in rpc/StreamRpcChannel.py
       data = sockets[1].recv(4096)
-      expected = binascii.unhexlify(
+      expected = handleRPCByteOrder(binascii.unhexlify(
         "2b000010080110001a0a52444d436f6d6d616e6422190801120908f0f401150"
-        "0ffffff180020e0012a010a30013800")
+        "0ffffff180020e0012a010a30013800"))
       self.assertEqual(data, expected,
                        msg="Regression check failed. If protocol change "
                        "was intended set expected to: " +
                        str(binascii.hexlify(data)))
       results.got_request = True
-      response = binascii.unhexlify(
+      response = handleRPCByteOrder(binascii.unhexlify(
         "2f0000100802100022290800100218002202000628e001300138004a0908f0f"
-        "4011500ffffff520908f0f40115ac107de05831")
+        "4011500ffffff520908f0f40115ac107de05831"))
       sent_bytes = sockets[1].send(response)
       self.assertEqual(sent_bytes, len(response))
 

python/ola/TestUtils.py

@@ -17,6 +17,8 @@
 # Copyright (C) 2020 Bruce Lowekamp
 
 import itertools
+import struct
+import sys
 
 """Common utils for ola python tests"""
 
@@ -32,3 +34,13 @@ def allHashNotEqual(testCase, t):
   h = map(hash, t)
   for pair in itertools.combinations(h, 2):
     testCase.assertNotEqual(pair[0], pair[1])
+
+
+def handleRPCByteOrder(expected):
+  # The RPC header (version and size) is encoded in native format, so flip that
+  # part of the expected data where necessary (as our expected is from a
+  # little endian source)
+  if sys.byteorder == 'big':
+    expected = (struct.pack('=L', struct.unpack_from('<L', expected)[0]) +
+                expected[4:])
+  return expected

tools/e133/EndpointManager.cpp

@@ -82,7 +82,7 @@ void EndpointManager::EndpointIDs(vector<uint16_t> *id_list) const {
  * @param callback the Callback to run. Ownership is not transferred.
  */
 void EndpointManager::RegisterNotification(
-    EndpointNoticationEvent event_type,
+    EndpointNotificationEvent event_type,
     EndpointNotificationCallback *callback) {
   // if this callback already exists update it
   vector<EndpointNotification>::iterator iter = m_callbacks.begin();
@@ -121,7 +121,7 @@ bool EndpointManager::UnRegisterNotification(
  * @param event_type the type of notifications to trigger.
  */
 void EndpointManager::RunNotifications(uint16_t endpoint_id,
-                                       EndpointNoticationEvent event_type) {
+                                       EndpointNotificationEvent event_type) {
   vector<EndpointNotification>::iterator iter = m_callbacks.begin();
   for (; iter != m_callbacks.end(); ++iter) {
     if (iter->event_type == event_type || event_type == BOTH)

tools/e133/EndpointManager.h

@@ -41,7 +41,7 @@
 class EndpointManager {
  public:
     typedef ola::Callback1<void, uint16_t> EndpointNotificationCallback;
-    typedef enum { ADD, REMOVE, BOTH } EndpointNoticationEvent;
+    typedef enum { ADD, REMOVE, BOTH } EndpointNotificationEvent;
 
     EndpointManager()
         : m_list_change_number(0) {
@@ -60,7 +60,7 @@ class EndpointManager {
     void EndpointIDs(std::vector<uint16_t> *id_list) const;
 
     // control notifications
-    void RegisterNotification(EndpointNoticationEvent event_type,
+    void RegisterNotification(EndpointNotificationEvent event_type,
                               EndpointNotificationCallback *callback);
     bool UnRegisterNotification(EndpointNotificationCallback *callback);
 
@@ -73,12 +73,12 @@ class EndpointManager {
 
     // list of callbacks to run
     typedef struct {
-      EndpointNoticationEvent event_type;
+      EndpointNotificationEvent event_type;
       EndpointNotificationCallback *callback;
     } EndpointNotification;
     std::vector<EndpointNotification> m_callbacks;
 
     void RunNotifications(uint16_t endpoint_id,
-                          EndpointNoticationEvent event_type);
+                          EndpointNotificationEvent event_type);
 };
 #endif  // TOOLS_E133_ENDPOINTMANAGER_H_