From 93cc8b2e4f174112f4db1a942dc9d0e3a1ca002e Mon Sep 17 00:00:00 2001 From: Matthew Hasselfield Date: Tue, 16 May 2023 13:38:27 -0400 Subject: [PATCH] Revist address_root != "observatory" (#327) * ocs-local-support: warn if crossbar/scf disagree on realm/address_root * registry subscribes to {address_root}.* not observatory.* Also changed OCSAgent to fail if any startup subscriptions fail. * aggregator subscribes to {address_root}.* not observatory.* * influxdb_publisher subscribes to {address_root}.* not observatory.* * InfluxDBAgent: exit cleanly on ctrl-c ... even when looping on failed connections to influxdb. * Tweak a few references to "observatory" in docstrings * docs: clarify "observatory" as default address_root * Remove all references to "registry_address" This SCF param actually wasn't used anywhere, except optionally in the ocs-agent-cli (which now simply defaults to {address_root}.registry and can still be overridden). The command-line argument remains, but is doc'ed as deprecated and has no effect on anything. * Fix registry tests (needs args.address_root) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * ocs-local-support: better error message if crossbar not managed ... in response to requests for "generate_crossbar_config" and "start crossbar". Also update erroneous docs for the former. * ocs-crossbar image can be configured more, with env Now the crossbar config can be bind-mounted in, or else some envvars are used to generate one from a template before crossbar is started. * ocs_agent: more informative error message on realm mismatch Also the SCF docs warn about updating the crossbar config. * Update documentation related to crossbar configuration --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Brian Koopman --- docker/crossbar/Dockerfile | 9 +- .../{config.json => config.json.template} | 10 +- docker/crossbar/run-crossbar.sh | 28 +++ docs/agents/aggregator.rst | 4 +- docs/developer/writing_an_agent/docker.rst | 1 - docs/user/crossbar_config.rst | 167 ++++++++++-------- docs/user/docker_config.rst | 4 +- docs/user/quickstart.rst | 1 - docs/user/site_config.rst | 13 +- example/miniobs/default.yaml | 1 - ocs/agents/aggregator/agent.py | 2 +- ocs/agents/host_manager/agent.py | 3 - ocs/agents/influxdb_publisher/agent.py | 3 +- ocs/agents/influxdb_publisher/drivers.py | 9 +- ocs/agents/registry/agent.py | 2 +- ocs/checkdata.py | 4 +- ocs/client_cli.py | 7 +- ocs/ocs_agent.py | 22 ++- ocs/ocsbow.py | 42 ++++- ocs/site_config.py | 15 +- tests/agents/test_registry_agent.py | 2 + tests/default.yaml | 1 - tests/test_site_config.py | 6 +- 23 files changed, 228 insertions(+), 128 deletions(-) rename docker/crossbar/{config.json => config.json.template} (92%) create mode 100644 docker/crossbar/run-crossbar.sh diff --git a/docker/crossbar/Dockerfile b/docker/crossbar/Dockerfile index b479604e..2adb8885 100644 --- a/docker/crossbar/Dockerfile +++ b/docker/crossbar/Dockerfile @@ -8,8 +8,11 @@ FROM crossbario/crossbar:cpy3-20.8.1 # Run as root to put config in place and chown USER root -# Copy in config and requirements files -COPY config.json /ocs/.crossbar/config.json +# Copy in config template and wrapper script +COPY run-crossbar.sh /ocs/run-crossbar.sh +RUN chmod a+x /ocs/run-crossbar.sh + +COPY config.json.template /ocs/.crossbar/config-with-address.json.template RUN chown -R crossbar:crossbar /ocs # Run image as crossbar user during normal operation @@ -20,4 +23,4 @@ EXPOSE 8001 # Run crossbar when the container launches # User made config.json should be mounted to /ocs/.crossbar/config.json -ENTRYPOINT ["crossbar", "start", "--cbdir", "/ocs/.crossbar"] +ENTRYPOINT ["/ocs/run-crossbar.sh"] diff --git a/docker/crossbar/config.json b/docker/crossbar/config.json.template similarity index 92% rename from docker/crossbar/config.json rename to docker/crossbar/config.json.template index 5f1519a0..c632c48c 100644 --- a/docker/crossbar/config.json +++ b/docker/crossbar/config.json.template @@ -10,13 +10,13 @@ }, "realms": [ { - "name": "test_realm", + "name": "{realm}", "roles": [ { "name": "iocs_agent", "permissions": [ { - "uri": "observatory.", + "uri": "{address_root}.", "match": "prefix", "allow": { "call": true, @@ -36,7 +36,7 @@ "name": "iocs_controller", "permissions": [ { - "uri": "observatory.", + "uri": "{address_root}.", "match": "prefix", "allow": { "call": true, @@ -60,7 +60,7 @@ "type": "web", "endpoint": { "type": "tcp", - "port": 8001 + "port": {port} }, "paths": { "ws": { @@ -81,7 +81,7 @@ }, "call": { "type": "caller", - "realm": "test_realm", + "realm": "{realm}", "role": "iocs_controller", "options": { } diff --git a/docker/crossbar/run-crossbar.sh b/docker/crossbar/run-crossbar.sh new file mode 100644 index 00000000..f074afc5 --- /dev/null +++ b/docker/crossbar/run-crossbar.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +CONFIG_DIR=/ocs/.crossbar +OCS_ADDRESS_ROOT=${OCS_ADDRESS_ROOT:-observatory} +OCS_CROSSBAR_REALM=${OCS_REALM:-test_realm} +OCS_CROSSBAR_PORT=${OCS_PORT:-8001} + +# Did user mount in a config.json? +if [ -e $CONFIG_DIR/config.json ]; then + echo Launching user-provided config.json + CONFIG_FILE=$CONFIG_DIR/config.json +else + pattern=" + s/{address_root}/${OCS_ADDRESS_ROOT}/g + s/{realm}/${OCS_CROSSBAR_REALM}/g + s/{port}/${OCS_CROSSBAR_PORT}/g + " + echo "Processing template with replacements:" + echo "$pattern" + echo + + CONFIG_FILE=$CONFIG_DIR/config-with-address.json + sed "$pattern" \ + $CONFIG_DIR/config-with-address.json.template \ + > $CONFIG_FILE +fi + +crossbar start --cbdir $CONFIG_DIR --config $CONFIG_FILE diff --git a/docs/agents/aggregator.rst b/docs/agents/aggregator.rst index 31e624b2..2e3c63d8 100644 --- a/docs/agents/aggregator.rst +++ b/docs/agents/aggregator.rst @@ -90,7 +90,9 @@ to register a feed so that it will be recorded by the aggregator. Unregistered providers will automatically be added when they send data, and stale providers will be removed if no data is received in a specified time period. -To do this, the aggregator monitors all feeds in the ``observatory`` namespace to find + +To do this, the aggregator monitors all feeds in the namespace defined +by the `{address_root}` prefix to find feeds that should be recorded. If the aggregator receives data from a feed registered with ``record=True``, it will automatically add that feed as a Provider, and will start putting incoming data into frames every ``frame_length`` diff --git a/docs/developer/writing_an_agent/docker.rst b/docs/developer/writing_an_agent/docker.rst index 2e4a7eea..e8d97656 100644 --- a/docs/developer/writing_an_agent/docker.rst +++ b/docs/developer/writing_an_agent/docker.rst @@ -84,7 +84,6 @@ the BarbonesAgent config to the ``ocs-docker`` host.: wamp_http: http://localhost:8001/call wamp_realm: test_realm address_root: observatory - registry_address: observatory.registry hosts: diff --git a/docs/user/crossbar_config.rst b/docs/user/crossbar_config.rst index dd1e5ac2..0a02a07a 100644 --- a/docs/user/crossbar_config.rst +++ b/docs/user/crossbar_config.rst @@ -12,48 +12,118 @@ the interface to the crossbar server. .. note:: - For most test deployments of OCS, you should not need to modify this file - and can use the one that comes with the ``simonsobs/ocs-crossbar`` Docker - Image. + For most simple lab deployments of OCS, you should not need to modify this + file and can use the one that comes with the ``simonsobs/ocs-crossbar`` + Docker Image. -Example Config --------------- -An example of the default OCS crossbar config that is bundled into -``simonsobs/ocs-crossbar`` can be found in the repository at -`ocs/docker/crossbar/config.json`_. This is based on the template in -`ocs/ocs/support/crossbar_config.json`_. +Configuration File Template +--------------------------- +The template that the default OCS crossbar config is built with is shown here: + +.. literalinclude:: ../../docker/crossbar/config.json.template -The unique parts of this to OCS are the realm name, "test_realm", defined -roles of "iocs_agent" and "iocs_controller, and "address_root" of -"observatory.". Additionally, we run on port 8001. +The variables `realm`, `address_root`, and `port` are all configurable and must +match configuration options set in your SCF. Keep reading this page to see how +to configure these variables. + +.. note:: + Changing the `address_root` has implications for the how your data is + stored and accessed in tools such as Grafana. It is recommended you pick + something reasonable when you first configure your system and do not change it + later. For further details on crossbar server configuration, see the crossbar `Router Configuration`_ page. -.. _`ocs/docker/crossbar/config.json`: https://github.com/simonsobs/ocs/blob/main/docker/crossbar/config.json -.. _`ocs/ocs/support/crossbar_config.json`: https://github.com/simonsobs/ocs/blob/main/ocs/support/crossbar_config.json .. _`Router Configuration`: https://crossbar.io/docs/Router-Configuration/ +Running with Docker +=================== + +We recommend running crossbar within a Docker container. We build the +``simonsobs/ocs-crossbar`` container from the official `crossbar.io Docker +image`_, specifically the cpy3 version. Bundled within the container is a +simple crossbar configuration file template with defaults that are +compatible with examples in this documentation. + +To adjust the crossbar configuration in the container, you can either: + +- Use environment variables to alter the most basic settings +- Generate and mount a new configuration file over + ``/ocs/.crossbar/config.json`` with the proper permissions + +.. _`crossbar.io Docker image`: https://hub.docker.com/r/crossbario/crossbar + +Environment variables in ocs-crossbar +------------------------------------- +The following environment variables can be set, to affect the +generation of the crossbar configuration file when the container +starts up: + +- OCS_ADDRESS_ROOT (default "observatory"): the base URI for OCS + entities (this needs to match the `address_root` set in the SCF). +- OCS_CROSSBAR_REALM (default "test_realm"): the WAMP realm to + configure for OCS. +- OCS_CROSSBAR_PORT (default 8001): the port on which crossbar will + accept requests. + +Here is an example of a docker-compose entry that overrides the +OCS_ADDRESS_ROOT:: + + crossbar: + image: simonsobs/ocs-crossbar:latest + ports: + - "127.0.0.1:8001:8001" # expose for OCS + environment: + - PYTHONUNBUFFERED=1 + - OCS_ADDRESS_ROOT=laboratory + +Bind Mounting the Configuration +------------------------------- +To instead mount a new configuration into the pre-built image, first chown +your file to be owned by user and group 242 (the default crossbar UID/GID), +then mount it appropriately in your docker-compose file. Here we assume you +put the configuration in the directory ``./dot_crossbar/``:: + + $ chown -R 242:242 dot_crossbar/ + +.. note:: + If you do not already have a configuration file to modify and use, see the + next section on generating one. + +Your docker-compose service should then be configured like:: + + crossbar: + image: simonsobs/ocs-crossbar + ports: + - "8001:8001" # expose for OCS + volumes: + - ./dot_crossbar:/ocs/.crossbar + environment: + - PYTHONUNBUFFERED=1 + Generating a New Config File ---------------------------- -``ocsbow`` can be used to generate a default configuration file, based on -options in your OSC file, which can then be modified if needed. +``ocs-local-support`` can be used to generate a default configuration +file, based on options in your SCF, which can then be modified if +needed. First, we make sure our ``OCS_CONFIG_DIR`` environment variable is set:: $ cd ocs-site-configs/ $ export OCS_CONFIG_DIR=`pwd` -We should make a directory for the crossbar config, let's call it -``dot_crossbar/`` (typically a dot directory, but for visibility we'll avoid -that):: +We should make a directory for the crossbar config, following along above let's +call it ``dot_crossbar/`` (typically a dot directory, but for visibility we'll +avoid that):: $ mkdir -p ocs-site-configs/dot_crossbar/ This directory needs to be configured as your crossbar 'config-dir' in your -ocs-site-config file. Now we can generate the config:: +ocs-site-config file. (See example in :ref:`site_config_user`.) Now we can +generate the config:: - $ ocsbow crossbar generate_config + $ ocs-local-support generate_crossbar_config The crossbar config-dir is set to: ./dot_crossbar/ Using @@ -68,54 +138,7 @@ modifications needed for your deployment. .. note:: - The crossbar 'config-dir' block and the 'agent-instance' block defining the - 'HostManager' Agent are both required for the system you are running ocsbow - on. Be sure to add these to your SCF if they do not exist. - -Running with Docker -=================== - -We recommend running crossbar within a Docker container. We build the -``simonsobs/ocs-crossbar`` container from the official `crossbar.io Docker -image`_, specifically the cpy3 version. Bundled within the container is a -simple OCS configuration that should work with the configuration -recommendations in this documentation. - -If changes need to be made, then you will need to generate your own -configuration file as described above. To use a modified configuration in the -container you can either: - -- Edit the default configuration file and rebuild the Docker image -- Mount the new configuration file over ``/ocs/.crossbar/config.json`` with the - proper permissions - -.. _`crossbar.io Docker image`: https://hub.docker.com/r/crossbario/crossbar - -Rebuilding the Docker Image ---------------------------- -To rebuild the Docker image after modifying ``ocs/docker/config.json`` run:: - - $ docker build -t ocs-crossbar . - -You should then update your configuration to use the new, local, -``ocs-crossbar`` image. - -Bind Mounting the Configuration -------------------------------- -To instead mount the new configuration into the pre-built image, first chown -your file to be owned by user and group 242 (the default crossbar UID/GID), -then mount it appropriately in your docker-compose file. Here we assume you -put the configuration in the directory ``./dot_crossbar/``:: - - $ chown -R 242:242 dot_crossbar/ - -Your docker-compose service should then be configured like:: - - crossbar: - image: simonsobs/ocs-crossbar - ports: - - "8001:8001" # expose for OCS - volumes: - - ./dot_crossbar:/ocs/.crossbar - environment: - - PYTHONUNBUFFERED=1 + The crossbar 'config-dir' block and the 'agent-instance' block + defining the 'HostManager' Agent are both required for the system + you are running `ocs-local-support` on. Be sure to add these to + your SCF if they do not exist. diff --git a/docs/user/docker_config.rst b/docs/user/docker_config.rst index 90d27853..9d0bb626 100644 --- a/docs/user/docker_config.rst +++ b/docs/user/docker_config.rst @@ -61,7 +61,7 @@ components):: ports: - "127.0.0.1:8001:8001" # expose for OCS environment: - - PYTHONUNBUFFERED=1 + - PYTHONUNBUFFERED=1 # -------------------------------------------------------------------------- # OCS Agents @@ -242,7 +242,7 @@ Where the separate compose files would look something like:: ports: - "127.0.0.1:8001:8001" # expose for OCS environment: - - PYTHONUNBUFFERED=1 + - PYTHONUNBUFFERED=1 :: diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index bf42ab5d..3886a0a9 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -48,7 +48,6 @@ structure. wamp_http: http://localhost:8001/call wamp_realm: test_realm address_root: observatory - registry_address: observatory.registry hosts: diff --git a/docs/user/site_config.rst b/docs/user/site_config.rst index b6e0760c..a5023c8a 100644 --- a/docs/user/site_config.rst +++ b/docs/user/site_config.rst @@ -35,7 +35,6 @@ instances (running two different classes of agent): wamp_http: http://10.10.10.3:8001/call wamp_realm: test_realm address_root: observatory - registry_agent: observatory.registry hosts: @@ -116,6 +115,18 @@ The `hub` section defines the connection parameters for the crossbar server. This entire section will likely remain unchanged, except for the ``wamp_server`` and ``wamp_http`` IP addresses. +The `address_root` setting determines the leading token in all agent +and feed addresses on the crossbar network. While "observatory" is +the default, it can be changed as long as the crossbar configuration +is also updated to permit operations on the `{address_root}.` uri. + +.. warning:: + The hub settings must match the crossbar configuration. If you + change `wamp_realm` or `address_root`, especially, be sure to + update your crossbar configuration accordingly. (If using the + ocs-crossbar docker image, this can be done through environment + variables in the ``docker-compose.yaml`` file.) + Under `hosts` we have defined a three hosts, `host-1`, `host-1-docker`, and `host-2`. This configuration example shows a mix of Agents running directly on hosts and running within Docker containers. diff --git a/example/miniobs/default.yaml b/example/miniobs/default.yaml index 3b3d7a44..43710ee8 100644 --- a/example/miniobs/default.yaml +++ b/example/miniobs/default.yaml @@ -5,7 +5,6 @@ hub: wamp_http: http://localhost:8001/call wamp_realm: test_realm address_root: observatory - registry_address: observatory.registry hosts: diff --git a/ocs/agents/aggregator/agent.py b/ocs/agents/aggregator/agent.py index 06a1bb34..3f16cb45 100644 --- a/ocs/agents/aggregator/agent.py +++ b/ocs/agents/aggregator/agent.py @@ -57,7 +57,7 @@ def __init__(self, agent, args): # If this ends up being too much data, we can add a tag '.record' # at the end of the address of recorded feeds, and filter by that. self.agent.subscribe_on_start(self._enqueue_incoming_data, - 'observatory..feeds.', + f'{args.address_root}..feeds.', options={'match': 'wildcard'}) record_on_start = (args.initial_state == 'record') diff --git a/ocs/agents/host_manager/agent.py b/ocs/agents/host_manager/agent.py index 36507b9a..dc5bb650 100644 --- a/ocs/agents/host_manager/agent.py +++ b/ocs/agents/host_manager/agent.py @@ -601,9 +601,6 @@ def main(args=None): os.dup2(null, stream.fileno()) os.close(null) - # To reduce "try again" noise, don't tell Registry about HostManager. - args.registry_address = 'none' - agent, runner = ocs_agent.init_site_agent(args) docker_composes = [] diff --git a/ocs/agents/influxdb_publisher/agent.py b/ocs/agents/influxdb_publisher/agent.py index aa274f3b..e35ef00a 100644 --- a/ocs/agents/influxdb_publisher/agent.py +++ b/ocs/agents/influxdb_publisher/agent.py @@ -50,7 +50,7 @@ def __init__(self, agent, args): self.loop_time = 1 self.agent.subscribe_on_start(self._enqueue_incoming_data, - 'observatory..feeds.', + f'{args.address_root}..feeds.', options={'match': 'wildcard'}) record_on_start = (args.initial_state == 'record') @@ -97,6 +97,7 @@ def record(self, session: ocs_agent.OpSession, params): port=self.args.port, protocol=self.args.protocol, gzip=self.args.gzip, + operate_callback=lambda: self.aggregate, ) session.set_status('running') diff --git a/ocs/agents/influxdb_publisher/drivers.py b/ocs/agents/influxdb_publisher/drivers.py index 7f09cba8..23fdc889 100644 --- a/ocs/agents/influxdb_publisher/drivers.py +++ b/ocs/agents/influxdb_publisher/drivers.py @@ -51,6 +51,9 @@ class Publisher: Protocol for writing data. Either 'line' or 'json'. gzip (bool, optional): compress influxdb requsts with gzip + operate_callback (callable, optional): + Function to call to see if failed connections should be + retried (to prevent a thread from locking). Attributes: host (str): @@ -66,7 +69,8 @@ class Publisher: """ - def __init__(self, host, database, incoming_data, port=8086, protocol='line', gzip=False): + def __init__(self, host, database, incoming_data, port=8086, protocol='line', + gzip=False, operate_callback=None): self.host = host self.port = port self.db = database @@ -88,6 +92,9 @@ def __init__(self, host, database, incoming_data, port=8086, protocol='line', gz LOG.error("Connection error, attempting to reconnect to DB.") self.client = InfluxDBClient(host=self.host, port=self.port, gzip=gzip) time.sleep(1) + if operate_callback and not operate_callback(): + break + db_names = [x['name'] for x in db_list] if self.db not in db_names: diff --git a/ocs/agents/registry/agent.py b/ocs/agents/registry/agent.py index 7970ccb2..f53d644d 100644 --- a/ocs/agents/registry/agent.py +++ b/ocs/agents/registry/agent.py @@ -98,7 +98,7 @@ def __init__(self, agent, args): self.agent_timeout = 5.0 # Removes agent after 5 seconds of no heartbeat. self.agent.subscribe_on_start( - self._register_heartbeat, 'observatory..feeds.heartbeat', + self._register_heartbeat, f'{args.address_root}..feeds.heartbeat', options={'match': 'wildcard'} ) diff --git a/ocs/checkdata.py b/ocs/checkdata.py index f3d21f03..b589f284 100644 --- a/ocs/checkdata.py +++ b/ocs/checkdata.py @@ -108,11 +108,11 @@ def _populate_instances(self): {FEED1: {"fields": {FIELD1: - {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD', + {'full_name': 'ADDRESS_ROOT.INSTANCE_ID.feeds.FEED.FIELD', 't_last': float, 'v_last': float}, FIELD2: - {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD', + {'full_name': 'ADDRESS_ROOT.INSTANCE_ID.feeds.FEED.FIELD', 't_last': float, 'v_last': float} }, diff --git a/ocs/client_cli.py b/ocs/client_cli.py index 7065f622..35b73724 100755 --- a/ocs/client_cli.py +++ b/ocs/client_cli.py @@ -62,7 +62,8 @@ def get_parser(): # scan p = client_sp.add_parser('scan', help="Gather and print list of Agents.") p.add_argument('--details', action='store_true', help="List all Operations with their current status OpCode.") - p.add_argument('--use-registry', action='store_true', help="Query the registry (faster than listening for heartbeats).") + p.add_argument('--use-registry', nargs='?', const='registry', help="Query the registry (faster than listening for heartbeats). " + "Pass the registry instance_id as an argument (default to 'registry').") # scan p = client_sp.add_parser('listen', help="Subscribe to feed(s) and dump to stdout.") @@ -122,9 +123,7 @@ def scan(parser, args): parser.error('Unable to find the OCS config; set OCS_CONFIG_DIR?') if args.use_registry: - reg_addr = args.registry_address - if reg_addr is None: - reg_addr = 'registry' + reg_addr = f'{args.address_root}.{args.use_registry}' try: c = OCSClient(get_instance_id(reg_addr, args), args=args) except RuntimeError as e: diff --git a/ocs/ocs_agent.py b/ocs/ocs_agent.py index 97aa97d0..fc06a6e8 100644 --- a/ocs/ocs_agent.py +++ b/ocs/ocs_agent.py @@ -4,7 +4,7 @@ txaio.use_twisted() from twisted.internet import reactor, task, threads -from twisted.internet.defer import inlineCallbacks, Deferred, DeferredList, FirstError +from twisted.internet.defer import inlineCallbacks, Deferred, DeferredList, FirstError, maybeDeferred from twisted.internet.error import ReactorNotRunning from twisted.python import log @@ -171,9 +171,16 @@ def onJoin(self, details): try: yield self.register(self._ops_handler, self.agent_address + '.ops') yield self.register(self._management_handler, self.agent_address) - except ApplicationError: - self.log.error('Failed to register basic handlers @ %s; ' - 'agent probably running already.' % self.agent_address) + except ApplicationError as e: + self.log.error('Failed to register basic handlers! ' + 'Error: {error}', error=e) + if e.error == 'wamp.error.not_authorized': + self.log.error('Are the WAMP realm and OCS address_root consistent ' + 'in OCS site config and crossbar config.json?') + elif e.error == 'wamp.error.procedure_already_exists': + self.log.error('Is this agent already running? ' + 'agent_address="{agent_address}"', + agent_address=self.agent_address) self.leave() return @@ -196,8 +203,13 @@ def heartbeat(): self.heartbeat_call.start(1.0) # Calls the hearbeat every second # Subscribe to startup_subs + def _subscribe_fail(*args, **kwargs): + self.log.error('Failed to subscribe to a feed or feed pattern; possible configuration problem.') + self.log.error(str(args) + str(kwargs)) + self.leave() + for sub in self.startup_subs: - self.subscribe(**sub) + maybeDeferred(self.subscribe, **sub).addErrback(_subscribe_fail) # Now do the startup activities, only the first time we join if self.first_time_startup: diff --git a/ocs/ocsbow.py b/ocs/ocsbow.py index 2915099f..e48192b6 100644 --- a/ocs/ocsbow.py +++ b/ocs/ocsbow.py @@ -156,16 +156,32 @@ def crossbar_test(args, site_config): '%s._crossbar_check_' % site.hub.data['address_root'], url=site.hub.data['wamp_http'], realm=site.hub.data['wamp_realm']) try: + # This is not expected to succeed, but the different errors + # tell us different things... client.call(client.agent_addr) except client_http.ControlClientError as ccex: suberr = ccex.args[0][4] - if suberr == 'client_http.error.connection_error': - ok, msg = False, 'http bridge not found at {wamp_http}.' - elif suberr == 'wamp.error.no_such_procedure': - ok, msg = True, 'http bridge reached at {wamp_http}.' + if suberr == 'wamp.error.no_such_procedure': + # This indicates we got through to the bridge, it liked + # the realm and our address_root. Return True. + ok, msg = True, 'http bridge reached at {wamp_http}.'.format(**site.hub.data) + elif suberr == 'client_http.error.connection_error': + # Possibly crossbar server is not running. + ok, msg = False, 'http bridge not found at {wamp_http}.'.format(**site.hub.data) + elif suberr == 'wamp.error.not_authorized': + # This is likely a configuration issue, print a banner and reraise it. + print('***** crossbar / ocs configuration mismatch *****') + print('The exception here indicates a likely configuration mismatch issue') + print('with the crossbar server and OCS. Specifically, the WAMP realm and') + print('the OCS address_root must match between the site config file') + print('and the crossbar config.') + print('*****\n') + raise ccex else: - ok, msg = True, 'unexpected bridge connection problem; raised %s' % (str(ccex)) - return ok, msg.format(**site.hub.data) + # I think this case hasn't been encountered much. + print('***** unhandled error case *****\n') + raise ccex + return ok, msg def get_status(args, site_config, restrict_hosts=None): @@ -399,9 +415,11 @@ def generate_crossbar_config(cm, site_config): print(line, end='') print('\n') print('To adopt the new config, remove %s and re-run me.' % cb_filename) + print() else: open(cb_filename, 'w').write(config_text) print('Wrote %s' % cb_filename) + print() class CrossbarManager: @@ -789,6 +807,15 @@ def update(self): 'running, but should start if you run "ocs-local-support start".')) self.analysis = solutions + def fail_on_missing_crossbar_config(self): + """Check if crossbar is managed by this config. If not, print + error message and exit(1).""" + if not self.crossbar['manage']: + print('Error! Crossbar config file not set.\n\n' + 'To start crossbar or to generate a config file, the site ' + 'config file must have a "crossbar" entry; see docs.\n') + sys.exit(1) + def main(args=None): args, site_config = get_args_and_site_config(args) @@ -953,8 +980,10 @@ def eligible(subsys): print('Trouble!') for text in fatals: print(_term_format(text, ' ', 4)) + sys.exit(1) if any([soln == 'crossbar' for soln, text in supports.analysis]): + supports.fail_on_missing_crossbar_config() print('Trying to start crossbar...') supports.crossbar['ctrl'].action('start', foreground=args.foreground) supports.update() # refresh .analysis @@ -995,5 +1024,6 @@ def eligible(subsys): print('No running crossbar detected, system is already "down".') elif action == 'generate_crossbar_config': + supports.fail_on_missing_crossbar_config() cm = supports.crossbar['ctrl'] generate_crossbar_config(cm, site_config) diff --git a/ocs/site_config.py b/ocs/site_config.py index 0315c207..f25fa375 100644 --- a/ocs/site_config.py +++ b/ocs/site_config.py @@ -187,12 +187,8 @@ def from_dict(cls, data, parent=None): ``address_root`` (required): The base address to be used by all OCS Agents. This is normally something simple like - ``observatory`` or ``detlab.system1``. (Command line - override: ``--address-root``.) - - ``registry_address`` (optional): The address of the OCS Registry - Agent. See :ref:`registry`. (Command line override: - ``--registry-address``.) + ``observatory`` or ``detlab``. (Command line override: + ``--address-root``.) """ self = cls() @@ -385,7 +381,7 @@ def add_arguments(parser=None): group.add_argument('--instance-id', help="""Look in the SCF for Agent-instance specific configuration options, and use those to launch the Agent.""") group.add_argument('--address-root', help="""Override the site default address root.""") - group.add_argument('--registry-address', help="""Override the site default registry address.""") + group.add_argument('--registry-address', help="""Deprecated.""") group.add_argument('--log-dir', help="""Set the logging directory.""") group.add_argument('--working-dir', help="""Propagate the working directory.""") return parser @@ -466,9 +462,6 @@ def get_config(args, agent_class=None): if args.site_realm is not None: site_config.hub.data['wamp_realm'] = args.site_realm - if args.registry_address is not None: - site_config.hub.data['registry_address'] = args.registry_address - # Identify our agent-instance. instance_config = None if no_dev_match: @@ -515,8 +508,6 @@ def add_site_attributes(args, site, host=None): args.site_realm = site.hub.data['wamp_realm'] if args.address_root is None: args.address_root = site.hub.data['address_root'] - if args.registry_address is None: - args.registry_address = site.hub.data.get('registry_address') if (args.log_dir is None) and (host is not None): args.log_dir = host.log_dir diff --git a/tests/agents/test_registry_agent.py b/tests/agents/test_registry_agent.py index 20a15edb..6eb0339f 100644 --- a/tests/agents/test_registry_agent.py +++ b/tests/agents/test_registry_agent.py @@ -4,8 +4,10 @@ from agents.util import create_session, create_agent_fixture from ocs.agents.registry.agent import RegisteredAgent, Registry, make_parser +from ocs import site_config parser = make_parser() +site_config.add_arguments(parser) args = parser.parse_args(['--wait-time', '0.1']) agent = create_agent_fixture(Registry, agent_kwargs=dict(args=args)) diff --git a/tests/default.yaml b/tests/default.yaml index 4207c592..449ac234 100644 --- a/tests/default.yaml +++ b/tests/default.yaml @@ -5,7 +5,6 @@ hub: wamp_http: http://127.0.0.1:18001/call wamp_realm: test_realm address_root: observatory - registry_address: observatory.registry hosts: diff --git a/tests/test_site_config.py b/tests/test_site_config.py index 0100374e..a2ed359d 100644 --- a/tests/test_site_config.py +++ b/tests/test_site_config.py @@ -28,8 +28,7 @@ def test_none_client_type_w_wamp_http_site(self): mock_site = MagicMock() mock_site.hub.data = {'wamp_http': 'http://127.0.0.1:8001', 'wamp_realm': 'test_realm', - 'address_root': 'observatory', - 'registry_address': 'observatory.registry'} + 'address_root': 'observatory'} get_control_client('test', site=mock_site, client_type=None) @@ -40,8 +39,7 @@ def test_none_client_type_wo_wamp_http_site(self): """ mock_site = MagicMock() mock_site.hub.data = {'wamp_realm': 'test_realm', - 'address_root': 'observatory', - 'registry_address': 'observatory.registry'} + 'address_root': 'observatory'} with pytest.raises(ValueError): get_control_client('test', site=mock_site, client_type=None)