Merge pull request #252 from simonsobs/koopman/move-test-utils

Move useful integration testing utils to main library

docs/developer/testing.rst

@@ -0,0 +1,37 @@
+Testing
+=======
+
+Writing tests for your OCS Agents is important for the long term maintanence of
+your Agent. Tests allow other developers to contribute to your Agent and easily
+confirm that their changes did not break functionality within the Agent. With
+some setup, tests can also allow you to test your Agent without access to the
+hardware that it controls.
+
+Testing within OCS comes in two forms, unit tests and integration tests.  Unit
+tests test functionality of the Agent code directly, without running the Agent
+itself (or any supporting parts, such as the crossbar server, or a piece of
+hardware to connect to.)
+
+Integration tests run a small OCS network, starting up the crossbar server,
+your Agent, and any supporting programs that your Agent might need (for
+instance, a program accepting serial connections for you Agent to connect to).
+As a result, integration tests are more involved than unit tests, requiring
+more setup and thus taking longer to execute.
+
+Both types of testing can be important for fully testing the functionality of
+your Agent.
+
+Running Tests
+-------------
+
+.. include:: ../../tests/README.rst
+    :start-line: 2
+
+Testing API
+-----------
+
+This section details the helper functions within OCS for assisting with testing
+your Agents.
+
+.. automodule:: ocs.testing
+    :members:

docs/index.rst

@@ -48,6 +48,7 @@ write OCS Agents or Clients.
     developer/clients
     developer/data
     developer/web
+    developer/testing
 
 
 .. toctree::

ocs/testing.py

@@ -0,0 +1,126 @@
+import os
+import time
+import pytest
+import signal
+import subprocess
+import coverage.data
+import urllib.request
+
+from urllib.error import URLError
+
+from ocs.ocs_client import OCSClient
+
+
+def create_agent_runner_fixture(agent_path, agent_name, args=None):
+    """Create a pytest fixture for running a given OCS Agent.
+
+    Parameters:
+        agent_path (str): Relative path to Agent,
+            i.e. '../agents/fake_data/fake_data_agent.py'
+        agent_name (str): Short, unique name for the agent
+        args (list): Additional CLI arguments to add when starting the Agent
+
+    """
+    @pytest.fixture()
+    def run_agent(cov):
+        env = os.environ.copy()
+        env['COVERAGE_FILE'] = f'.coverage.agent.{agent_name}'
+        env['OCS_CONFIG_DIR'] = os.getcwd()
+        cmd = ['coverage', 'run',
+               '--rcfile=./.coveragerc',
+               agent_path,
+               '--site-file',
+               './default.yaml']
+        if args is not None:
+            cmd.extend(args)
+        agentproc = subprocess.Popen(cmd,
+                                     env=env,
+                                     stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     preexec_fn=os.setsid)
+
+        # wait for Agent to startup
+        time.sleep(1)
+
+        yield
+
+        # shutdown Agent
+        agentproc.send_signal(signal.SIGINT)
+        time.sleep(1)
+
+        # report coverage
+        agentcov = coverage.data.CoverageData(
+            basename=f'.coverage.agent.{agent_name}')
+        agentcov.read()
+        # protect against missing --cov flag
+        if cov is not None:
+            cov.get_data().update(agentcov)
+
+    return run_agent
+
+
+def create_client_fixture(instance_id, timeout=30):
+    """Create the fixture that provides tests a Client object.
+
+    Parameters:
+        instance_id (str): Agent instance-id to connect the Client to
+        timeout (int): Approximate timeout in seconds for the connection.
+            Connection attempts will be made X times, with a 1 second pause
+            between attempts. This is useful if it takes some time for the
+            Agent to start accepting connections, which varies depending on the
+            Agent.
+
+    """
+    @pytest.fixture()
+    def client_fixture():
+        # Set the OCS_CONFIG_DIR so we read the local default.yaml file
+        os.environ['OCS_CONFIG_DIR'] = os.getcwd()
+        print(os.environ['OCS_CONFIG_DIR'])
+        attempts = 0
+
+        while attempts < timeout:
+            try:
+                client = OCSClient(instance_id)
+                break
+            except RuntimeError as e:
+                print(f"Caught error: {e}")
+                print("Attempting to reconnect.")
+
+            time.sleep(1)
+            attempts += 1
+
+        return client
+
+    return client_fixture
+
+
+def check_crossbar_connection(port=18001, interval=5, max_attempts=6):
+    """Check that the crossbar server is up and available for an Agent to
+    connect to.
+
+    Parameters:
+        port (int): Port the crossbar server is configured to run on for
+            testing.
+        interval (float): Amount of time in seconds to wait between checks.
+        max_attempts (int): Maximum number of attempts before giving up.
+
+    Notes:
+        For this check to work the crossbar server needs the `Node Info Service
+        <https://crossbar.io/docs/Node-Info-Service/>`_ running at the path
+        /info.
+
+    """
+    attempts = 0
+
+    while attempts < max_attempts:
+        try:
+            url = f"http://localhost:{port}/info"
+            code = urllib.request.urlopen(url).getcode()
+        except (URLError, ConnectionResetError):
+            print("Crossbar server not online yet, waiting 5 seconds.")
+            time.sleep(interval)
+
+        attempts += 1
+
+    assert code == 200
+    print("Crossbar server online.")

tests/README.rst

@@ -1,5 +1,5 @@
 Tests
------
+=====
 We use pytest as our test runner for OCS. To run all of the tests, from within
 this ``tests/`` directory, run pytest::
 
@@ -10,7 +10,7 @@ unit tests will run quickly, however the integration test will take some time,
 and might require some first time setup.
 
 Test Organization
-`````````````````
+-----------------
 Tests that test the functionality of the core ocs library are kept in the root
 of the ``tests/`` directory. They are named identically to the core library
 module filenames with a ``test_`` prefix.
@@ -24,7 +24,7 @@ Finally, integration tests that test the Agents and interaction with them
 through Clients are kept in ``tests/integration``.
 
 Unit Tests
-``````````
+----------
 The unit tests are built to run quickly and test functionality of individual
 parts of the OCS library. These are run automatically on every commit to a
 branch/PR on GitHub, using GitHub Actions.  However, you might want to run them
@@ -43,7 +43,7 @@ tests, leaving just the unit tests.
         $ docker run --rm -w="/app/ocs/tests/" ocs sh -c "python3 -m pytest -m 'not integtest'"
 
 Integration Tests
-`````````````````
+-----------------
 These tests are built to test the running OCS system, and as such need several
 running components. This includes a crossbar server and each core OCS Agent. In
 order to run these in an isolated environment we make use of Docker and Docker
@@ -70,7 +70,7 @@ all integration tests with::
     crossbar on port 18001 instead of port 8001.)
 
 Reducing Turnaround Time in Testing
-...................................
+```````````````````````````````````
 Since the integration tests depend on docker containers you need to have the
 docker images built prior to running the tests. You can build all of the docker
 images from the root of the ocs repo::
@@ -99,7 +99,7 @@ Agent/container you are working on. For example, in the fake-data-agent::
       - "--site-http=http://crossbar:18001/call"
 
 Code Coverage
-`````````````
+-------------
 Code coverage reports can be produced with the ``--cov`` flag::
 
   python3 -m pytest -m 'not integtest' --cov

tests/integration/test_aggregator_agent_integration.py

@@ -3,9 +3,12 @@
 import ocs
 from ocs.base import OpCode
 
-from integration.util import (
+from ocs.testing import (
     create_agent_runner_fixture,
     create_client_fixture,
+)
+
+from integration.util import (
     create_crossbar_fixture
 )
 

tests/integration/test_fake_data_agent_integration.py

@@ -3,10 +3,13 @@
 import ocs
 from ocs.base import OpCode
 
-from integration.util import (
+from ocs.testing import (
     create_agent_runner_fixture,
     create_client_fixture,
-    create_crossbar_fixture
+)
+
+from integration.util import (
+    create_crossbar_fixture,
 )
 
 pytest_plugins = ("docker_compose")

tests/integration/test_host_master_agent_integration.py

@@ -3,9 +3,12 @@
 
 from ocs.base import OpCode
 
-from integration.util import (
+from ocs.testing import (
     create_agent_runner_fixture,
     create_client_fixture,
+)
+
+from integration.util import (
     create_crossbar_fixture
 )
 

tests/integration/test_influxdb_publisher_integration.py

@@ -3,9 +3,12 @@
 import ocs
 from ocs.base import OpCode
 
-from integration.util import (
+from ocs.testing import (
     create_agent_runner_fixture,
     create_client_fixture,
+)
+
+from integration.util import (
     create_crossbar_fixture
 )
 

tests/integration/test_registry_agent_integration.py

@@ -1,9 +1,12 @@
 import os
 import pytest
 
-from integration.util import (
+from ocs.testing import (
     create_agent_runner_fixture,
     create_client_fixture,
+)
+
+from integration.util import (
     create_crossbar_fixture
 )