Merge pull request #284 from simonsobs/koopman/ocs-plugins

Create OCS Plugin system
simonsobs · Oct 10, 2022 · 68cdac5 · 68cdac5
2 parents cebd430 + 54a8e67
commit 68cdac5
Show file tree

Hide file tree

Showing 68 changed files with 869 additions and 372 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -36,7 +36,7 @@ jobs:
     # Integration Tests
     - name: Build images for integration tests
       run: |
-        docker-compose build ocs ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
+        docker-compose build ocs ocs-crossbar
 
     - name: Run integration tests
       working-directory: ./tests

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -36,7 +36,7 @@ jobs:
     # Integration Tests
     - name: Build images for integration tests
       run: |
-        docker-compose build ocs ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
+        docker-compose build ocs ocs-crossbar
 
     - name: Run integration tests
       working-directory: ./tests

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -44,7 +44,7 @@ jobs:
     # Integration Tests
     - name: Build images for integration tests
       run: |
-        docker-compose build ocs ocs-fake-data-agent ocs-aggregator-agent ocs-crossbar
+        docker-compose build ocs ocs-crossbar
 
     - name: Run integration tests
       working-directory: ./tests

diff --git a/Dockerfile b/Dockerfile
@@ -11,9 +11,16 @@ ENV LANG C.UTF-8
 RUN groupadd -g 9000 ocs && \
     useradd -m -l -u 9000 -g 9000 ocs
 
+# aggregator: Prepare data directory for mount
+RUN mkdir -p /data && \
+    chown ocs:ocs /data
+
 # Setup configuration environment
 ENV OCS_CONFIG_DIR=/config
 
+# Disable output buffer
+ENV PYTHONUNBUFFERED=1
+
 # Install python and pip
 RUN apt-get update && apt-get install -y python3 \
     python3-pip \
@@ -35,3 +42,9 @@ COPY . /app/ocs/
 
 # Install ocs
 RUN pip3 install .
+
+# Reset workdir to avoid local imports
+WORKDIR /
+
+# Run agent on container startup
+ENTRYPOINT ["dumb-init", "ocs-agent-cli"]
diff --git a/agents/aggregator/Dockerfile b/agents/aggregator/Dockerfile
diff --git a/agents/barebones_agent/Dockerfile b/agents/barebones_agent/Dockerfile
diff --git a/agents/fake_data/Dockerfile b/agents/fake_data/Dockerfile
diff --git a/agents/influxdb_publisher/Dockerfile b/agents/influxdb_publisher/Dockerfile
diff --git a/agents/registry/Dockerfile b/agents/registry/Dockerfile
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -24,52 +24,3 @@ services:
     build: ./docker/ocs-util/
     depends_on:
       - "ocs"
-
-  # --------------------------------------------------------------------------
-  # OCS Core Agents
-  # --------------------------------------------------------------------------
-
-  # --------------------------------------------------------------------------
-  # The OCS Registry keeps track of all other running Agents
-  # --------------------------------------------------------------------------
-  ocs-registry-agent:
-    image: "ocs-registry-agent"
-    build: ./agents/registry/
-    depends_on:
-      - "ocs"
-
-  # --------------------------------------------------------------------------
-  # The OCS Aggregator collects and writes data to disk in the sog3 format.
-  # --------------------------------------------------------------------------
-  ocs-aggregator-agent:
-    image: "ocs-aggregator-agent"
-    build: ./agents/aggregator/
-    depends_on:
-      - "ocs"
-
-  # --------------------------------------------------------------------------
-  # The OCS InfluxDB Publisher publishes data feeds to InfluxDB.
-  # --------------------------------------------------------------------------
-  ocs-influxdb-publisher-agent:
-    image: "ocs-influxdb-publisher-agent"
-    build: ./agents/influxdb_publisher/
-    depends_on:
-      - "ocs"
-
-  # --------------------------------------------------------------------------
-  # The OCS Aggregator collects and writes data to disk in the sog3 format.
-  # --------------------------------------------------------------------------
-  ocs-fake-data-agent:
-    image: "ocs-fake-data-agent"
-    build: ./agents/fake_data/
-    depends_on:
-      - "ocs"
-
-  # --------------------------------------------------------------------------
-  # The OCS Barebones Agent
-  # --------------------------------------------------------------------------
-  ocs-barebones-agent:
-    image: "ocs-barebones-agent"
-    build: ./agents/barebones_agent/
-    depends_on:
-      - "ocs"
diff --git a/docs/agents/aggregator.rst b/docs/agents/aggregator.rst
@@ -17,9 +17,9 @@ to disk in the `.g3` file format.
     data to disk.
 
 .. argparse::
-   :module: agents.aggregator.aggregator_agent
+   :module: ocs.agents.aggregator.agent
    :func: make_parser
-   :prog: aggregator_agent.py
+   :prog: agent.py
 
 Dependencies
 ------------
@@ -65,12 +65,13 @@ The docker image for the aggregator agent is simonsobs/ocs-aggregator-agent
 Here is an example configuration::
 
     ocs-aggregator:
-        image: simonsobs/ocs-aggregator-agent:latest
+        image: simonsobs/ocs:latest
         container_name: ocs-aggregator
         hostname: ocs-docker
         user: "9000"
         environment:
           - LOGLEVEL=info
+          - INSTANCE_ID=aggregator
         volumes:
           - ${OCS_CONFIG_DIR}:/config
           - /path/to/host/data:/data
@@ -186,21 +187,21 @@ data and a vector of timestamps.
 
 Agent API
 ---------
-.. autoclass:: agents.aggregator.aggregator_agent.AggregatorAgent
+.. autoclass:: ocs.agents.aggregator.agent.AggregatorAgent
     :members:
 
 Supporting APIs
 ---------------
 .. _agg_provider_api:
 
-.. autoclass:: ocs.agent.aggregator.Provider
+.. autoclass:: ocs.agents.aggregator.drivers.Provider
     :members:
     :noindex:
 
-.. autoclass:: ocs.agent.aggregator.G3FileRotator
+.. autoclass:: ocs.agents.aggregator.drivers.G3FileRotator
     :members:
     :noindex:
 
-.. autoclass:: ocs.agent.aggregator.Aggregator
+.. autoclass:: ocs.agents.aggregator.drivers.Aggregator
     :members:
     :noindex:
diff --git a/docs/agents/barebones.rst b/docs/agents/barebones.rst
@@ -24,8 +24,22 @@ available arguments::
      'instance-id': 'barebones-1',
      'arguments': []},
 
+Docker Compose
+``````````````
+
+An example docker-compose configuration::
+
+    ocs-barebones-1:
+        image: simonsobs/ocs:latest
+        hostname: ocs-docker
+        environment:
+          - LOGLEVEL=info
+          - INSTANCE_ID=barebones-1
+        volumes:
+          - ${OCS_CONFIG_DIR}:/config:ro
+
 Agent API
 ---------
 
-.. autoclass:: agents.barebones_agent.barebones_agent.BarebonesAgent
+.. autoclass:: ocs.agents.barebones.agent.BarebonesAgent
     :members:
diff --git a/docs/agents/fake_data.rst b/docs/agents/fake_data.rst
@@ -9,9 +9,9 @@ with data aggregation and display. It will generate random data and pass it to
 an OCS feed.
 
 .. argparse::
-   :module: agents.fake_data.fake_data_agent
+   :module: ocs.agents.fake_data.agent
    :func: add_agent_args
-   :prog: fake_data_agent.py
+   :prog: agent.py
 
 Configuration File Examples
 ---------------------------
@@ -39,17 +39,16 @@ The Fake Data Agent can also be run in a Docker container. An example
 docker-compose service configuration is shown here::
 
     fake-data1:
-        image: simonsobs/ocs-fake-data-agent:latest
+        image: simonsobs/ocs:latest
         hostname: ocs-docker
         environment:
           - LOGLEVEL=info
+          - INSTANCE_ID=fake-data1
         volumes:
           - ${OCS_CONFIG_DIR}:/config:ro
-        command:
-          - "--instance-id=fake-data1"
 
 Agent API
 ---------
 
-.. autoclass:: agents.fake_data.fake_data_agent.FakeDataAgent
+.. autoclass:: ocs.agents.fake_data.agent.FakeDataAgent
     :members:
diff --git a/docs/agents/host_manager.rst b/docs/agents/host_manager.rst
@@ -15,9 +15,9 @@ For a complete discussion of this Agent and how to best use it, see
 
 
 .. argparse::
-   :module: agents.host_manager.host_manager
+   :module: ocs.agents.host_manager.agent
    :func: make_parser
-   :prog: host_manager.py
+   :prog: agent.py
 
 
 
@@ -52,12 +52,12 @@ Please see :ref:`centralized_management`.
 Agent API
 ---------
 
-.. autoclass:: agents.host_manager.host_manager.HostManager
+.. autoclass:: ocs.agents.host_manager.agent.HostManager
     :members:
 
 Supporting APIs
 ---------------
 
-.. automethod:: agents.host_manager.host_manager.HostManager._process_target_states
+.. automethod:: ocs.agents.host_manager.agent.HostManager._process_target_states
 
-.. automethod:: agents.host_manager.host_manager.HostManager._reload_config
+.. automethod:: ocs.agents.host_manager.agent.HostManager._reload_config
diff --git a/docs/agents/influxdb_publisher.rst b/docs/agents/influxdb_publisher.rst
@@ -11,9 +11,9 @@ writing to file it will publish all recorded OCS data feeds to an InfluxDB
 instance running somewhere on the network.
 
 .. argparse::
-   :module: agents.influxdb_publisher.influxdb_publisher
+   :module: ocs.agents.influxdb_publisher.agent
    :func: make_parser
-   :prog: influxdb_publisher.py
+   :prog: agent.py
 
 Configuration File Examples
 ---------------------------
@@ -42,8 +42,10 @@ Docker Compose
 Add the InfluxDB Publisher Agent container to your docker-compose file::
 
   ocs-influxdb-publisher:
-    image: simonsobs/ocs-influxdb-publisher-agent:latest
+    image: simonsobs/ocs:latest
     hostname: ocs-docker
+    environment:
+      - INSTANCE_ID=influxagent
     volumes:
       - ${OCS_CONFIG_DIR}:/config:ro
 
@@ -117,12 +119,12 @@ For more information about using InfluxDB in Grafana, see the `Grafana Documenta
 Agent API
 ---------
 
-.. autoclass:: agents.influxdb_publisher.influxdb_publisher.InfluxDBAgent
+.. autoclass:: ocs.agents.influxdb_publisher.agent.InfluxDBAgent
     :members:
 
 
 Supporting APIs
 ---------------
 
-.. autoclass:: ocs.agent.influxdb_publisher.Publisher
+.. autoclass:: ocs.agents.influxdb_publisher.agent.Publisher
     :members: