Merge pull request #111 from dls-controls/imalcolm

Imalcolm

.travis.yml

@@ -6,20 +6,29 @@ python:
   - "3.4"
   - "3.5"
 
+addons:
+  apt:
+    packages:
+      - qt4-qmake
+      - libqt4-dev
+
 cache:
   directories:
     - $HOME/.cache/pip
-    - $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages
-    - $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/bin
+    - ${VIRTUAL_ENV}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages
+    - ${VIRTUAL_ENV}/bin
 
 install:
   - env
-  - ls -al $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages
-  - ls -al $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/bin  
+  - ls -al ${VIRTUAL_ENV}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages
+  - ls -al ${VIRTUAL_ENV}/bin
+  - ci/install_pyqt.sh
+  - python -c "import PyQt4;print(PyQt4)"
   - pip install -r requirements/test.txt
   - pip install coveralls
-  - ls -al $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/lib/python$TRAVIS_PYTHON_VERSION/site-packages
-  - ls -al $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/bin  
+  - ls -al ${VIRTUAL_ENV}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages  
+  - ldd ${VIRTUAL_ENV}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/PyQt4/*.so
+  - ls -al ${VIRTUAL_ENV}/bin
   - python setup.py bdist_egg
 
 # command to run tests

ci/install_pyqt.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+# sip
+SIP_VER=4.13.3
+SIP_DIR=sip-${SIP_VER}
+SIP_TAR=${SIP_DIR}.tar.gz
+# pyqt
+PYQT_VER=4.9.4
+PYQT_DIR=PyQt-x11-gpl-${PYQT_VER}
+PYQT_TAR=${PYQT_DIR}.tar.gz
+SITE_PACKAGES=${VIRTUAL_ENV}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages
+set -ex
+if [ ! -e ${SITE_PACKAGES}/PyQt4 ]; then
+    # Install sip
+    wget http://sourceforge.net/projects/pyqt/files/sip/${SIP_DIR}/${SIP_TAR}
+    tar -xzf ${SIP_TAR}
+    cd ${SIP_DIR}
+    python ./configure.py \
+        --bindir=$(pwd)/prefix/bin \
+        --sipdir=$(pwd)/prefix/share/sip \
+        --incdir=$(pwd)/prefix/include \
+        --destdir=${SITE_PACKAGES}
+    make -j 2
+    make install
+    cd ..
+    # Install pyqt
+    export PATH=$PATH:$(pwd)/${SIP_DIR}/prefix/bin
+    wget http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-${PYQT_VER}/${PYQT_TAR}
+    tar -xzf ${PYQT_TAR}
+    rm ${PYQT_TAR}
+    cd ${PYQT_DIR}
+    mkdir prefix
+    python ./configure.py \
+        --confirm-license \
+        --bindir=$(pwd)/prefix/bin \
+        --destdir=${SITE_PACKAGES}
+    make -j 2
+    make install
+    cd ..
+fi

examples/hello_example.yaml

@@ -0,0 +1,2 @@
+assemblies.demo.hello:
+  name: hello

malcolm/assemblies/__init__.py

@@ -0,0 +1,6 @@
+# import subpackages
+from malcolm.packageutil import import_sub_packages
+
+__all__ = import_sub_packages(globals(), __name__)
+
+del import_sub_packages

malcolm/assemblies/demo/__init__.py

@@ -0,0 +1,6 @@
+# import subpackages
+from malcolm.assemblyutil import make_all_assemblies
+
+__all__ = make_all_assemblies(globals(), __name__)
+
+del make_all_assemblies

malcolm/assemblies/demo/hello.yaml

@@ -0,0 +1 @@
+controllers.HelloController:

malcolm/core/collection.py → malcolm/assemblyutil.py

@@ -1,48 +1,71 @@
+import os
 from collections import OrderedDict
+import logging
 
 from ruamel import yaml
 
-import malcolm.parameters
 import malcolm.controllers
+import malcolm.parameters
 import malcolm.parts
-import malcolm.collections
-from malcolm.core.methodmeta import takes, REQUIRED
-from malcolm.vmetas import StringMeta
 from malcolm.compat import base_string
-
-
-def make_collection(text):
+from malcolm.core import REQUIRED, method_takes
+from malcolm.core.vmetas import StringMeta
+
+
+def make_all_assemblies(globals_d, package_name):
+    func_dict = {}
+    # this is the path to the package
+    package_relative = package_name.split(".")[1:]
+    package_fs_path = os.path.join(os.path.dirname(__file__), *package_relative)
+
+    for fname in os.listdir(package_fs_path):
+        split = fname.split(".")
+        if split[-1] == "yaml":
+            assert len(split) == 2, \
+                "Expected <something_without_dots>.yaml, got %r" % fname
+            yaml_path = os.path.join(package_fs_path, fname)
+            logging.debug("Parsing %s", yaml_path)
+            with open(yaml_path) as f:
+                text = f.read()
+                func = make_assembly(text)
+                func_dict[split[0]] = func
+
+    globals_d.update(func_dict)
+    __all__ = list(func_dict)
+    return __all__
+
+def make_assembly(text):
     """Make a collection function that will create a list of blocks
 
     Args:
         text (str): YAML text specifying parameters, controllers, parts and
-            other collections to be instantiated
+            other assemblies to be instantiated
 
     Returns:
         function: A collection function decorated with @takes. This can be
-            used in other collections or instantiated by the process. If the
+            used in other assemblies or instantiated by the process. If the
             YAML text specified controllers or parts then a block instance
             with the given name will be instantiated. If there are any
-            collections listed then they will be called. All created blocks by
-            this or any sub collection will be returned
+            assemblies listed then they will be called. All created blocks
+            by this or any sub collection will be returned
     """
     ds = yaml.load(text, Loader=yaml.RoundTripLoader)
 
     sections = split_into_sections(ds)
 
     # If we have parts then check we have a maximum of one controller
     if sections["controllers"] or sections["parts"]:
-        num_controllers = len(sections["controllers"])
-        assert num_controllers in (0, 1), \
-            "Expected 0 or 1 controller with parts, got %s" % num_controllers
+        ncontrollers = len(sections["controllers"])
+        assert ncontrollers in (0, 1), \
+            "Expected 0 or 1 controller with parts, got %s" % ncontrollers
         # We will be creating a block here, so we need a name
         include_name = True
     else:
-        # No name needed as just a collection of other collections
+        # No name needed as just a collection of other assemblies
         include_name = False
 
     @with_takes_from(sections["parameters"], include_name)
-    def collection(params, process):
+    def collection(process, params):
         substitute_params(sections, params)
         ret = []
 
@@ -52,17 +75,17 @@ def collection(params, process):
                 params["name"], process,
                 sections["controllers"], sections["parts"]))
 
-        # It we have any other collections
-        for name, d in sections["collections"].items():
-            ret += call_with_map(malcolm.collections, name, d, process)
+        # It we have any other assemblies
+        for name, d in sections["assemblies"].items():
+            import malcolm.assemblies
+            ret += call_with_map(malcolm.assemblies, name, d, process)
 
         return ret
 
     return collection
 
-
 def split_into_sections(ds):
-    """Split a dictionary into parameters, controllers, parts and collections
+    """Split a dictionary into parameters, controllers, parts and assemblies
 
     Args:
         ds (dict): Dictionary of section: params. E.g.
@@ -83,7 +106,7 @@ def split_into_sections(ds):
             }
     """
     # First separate them into their relevant sections
-    sections = dict(parameters={}, controllers={}, parts={}, collections={})
+    sections = dict(parameters={}, controllers={}, parts={}, assemblies={})
     for name, d in ds.items():
         section, subsection = name.split(".", 1)
         if section in sections:
@@ -93,7 +116,6 @@ def split_into_sections(ds):
 
     return sections
 
-
 def with_takes_from(parameters, include_name):
     """Create an @takes decorator from parameters dict.
 
@@ -114,8 +136,7 @@ def with_takes_from(parameters, include_name):
         takes_arguments = []
     for name, d in parameters.items():
         takes_arguments += call_with_map(malcolm.parameters, name, d)
-    return takes(*takes_arguments)
-
+    return method_takes(*takes_arguments)
 
 def substitute_params(d, params):
     """Substitute a dictionary in place with $(attr) macros in it with values
@@ -138,7 +159,6 @@ def substitute_params(d, params):
             elif isinstance(v, dict):
                 substitute_params(v, params)
 
-
 def make_block_instance(name, process, controllers_d, parts_d):
     """Make a block subclass from a series of parts.* and controllers.* dicts
 
@@ -148,7 +168,7 @@ def make_block_instance(name, process, controllers_d, parts_d):
         controllers_d (dict): Controllers sub dictionary. E.g.
             {"ManagerController": None}
         parts_d (dict): Parts sub dictionary. E.g.
-            {"ca.CADoublePart": {"pv": "MY:PV:STRING"}}
+            {"ca.CADoublePart": {"name": "me", "pv": "MY:PV:STRING"}}
 
     Returns:
         Block: The created block instance as managed by the controller with
@@ -158,18 +178,17 @@ def make_block_instance(name, process, controllers_d, parts_d):
     for cls_name, d in parts_d.items():
         # Require all parts to have a name
         # TODO: make sure this is added from gui?
-        name = d[name]
-        parts[name] = call_with_map(malcolm.parts, cls_name, d)
+        parts[d["name"]] = call_with_map(malcolm.parts, cls_name, d, process)
     if controllers_d:
+        ob = malcolm.controllers
         cls_name, d = list(controllers_d.items())[0]
-        controller = call_with_map(
-            malcolm.controllers, cls_name, d, name, process, parts)
     else:
-        controller = call_with_map(
-            malcolm.core.controller, "Controller", d, name, process, parts)
+        ob = malcolm.core.controller
+        cls_name = "Controller"
+        d = None
+    controller = call_with_map(ob, cls_name, d, name, process, parts)
     return controller.block
 
-
 def call_with_map(ob, name, d, *args):
     """Keep recursing down from ob using dotted name, then call it with d, *args
 
@@ -192,4 +211,4 @@ def call_with_map(ob, name, d, *args):
 
     params = ob.MethodMeta.prepare_input_map(d)
     args += (params,)
-    return ob(*args)
+    return ob(*args)

malcolm/comms/websocket/websocketclientcomms.py

@@ -4,9 +4,8 @@
 from tornado.ioloop import IOLoop
 from tornado.websocket import websocket_connect
 
-from malcolm.core.clientcomms import ClientComms
-from malcolm.core.request import Request, Subscribe
-from malcolm.core.serializable import Serializable
+from malcolm.core import ClientComms, Request, Subscribe, Response, \
+    deserialize_object, serialize_object
 
 
 class WebsocketClientComms(ClientComms):
@@ -38,7 +37,7 @@ def on_message(self, message):
         try:
             self.log_debug("Got message %s", message)
             d = json.loads(message, object_pairs_hook=OrderedDict)
-            response = Serializable.from_dict(d)
+            response = deserialize_object(d, Response)
             self.send_to_caller(response)
         except Exception as e:
             # If we don't catch the exception here, tornado will spew odd
@@ -51,7 +50,7 @@ def send_to_server(self, request):
         Args:
             request(Request): The message to pass to the server
         """
-        message = json.dumps(request.to_dict())
+        message = json.dumps(serialize_object(request))
         self.conn.result().write_message(message)
 
     def stop_recv_loop(self):

malcolm/comms/websocket/websocketservercomms.py

@@ -7,9 +7,8 @@
 from tornado.web import Application
 from tornado.httpserver import HTTPServer
 
-from malcolm.core.servercomms import ServerComms
-from malcolm.core.serializable import Serializable
-from malcolm.core.request import Request
+from malcolm.core import ServerComms, deserialize_object, serialize_object, \
+    Request
 
 
 class MalcolmWebSocketHandler(WebSocketHandler):
@@ -26,7 +25,7 @@ def on_message(self, message):
 
         logging.debug(message)
         d = json.loads(message, object_pairs_hook=OrderedDict)
-        request = Serializable.from_dict(d)
+        request = deserialize_object(d, Request)
         request.context = self
         self.servercomms.on_request(request)
 
@@ -55,7 +54,7 @@ def send_to_client(self, response):
             response(Response): The message to pass to the client
         """
 
-        message = json.dumps(response.to_dict())
+        message = json.dumps(serialize_object(response))
         self.log_debug("Sending to client %s", message)
         response.context.write_message(message)
 

malcolm/controllers/__init__.py

@@ -1,10 +1,6 @@
 # make the import path nice
-from malcolm.util import import_child_packages
+from malcolm.packageutil import import_methodmeta_decorated_classes
 
-class_dict = import_child_packages("controllers")
+__all__ = import_methodmeta_decorated_classes(globals(), __name__)
 
-globals().update(class_dict)
-__all__ = list(class_dict)
-
-del class_dict
-del import_child_packages
+del import_methodmeta_decorated_classes