pipeline.yaml

# docker compose file for the ladder99 pipeline.
# can override these settings in individual setup pipeline.yamls.
# run with eg `shell/docker ccs-pa start broker adapter agent`

version: '3.8'

services:
  # ladder99 agent - takes in shdr and outputs html/json.
  agent:
    container_name: agent
    profiles:
      - all
      - agent
    # see https://github.com/Ladder99/mtconnect-agent
    image: ladder99/mtconnect-agent:latest
    command: agent debug /data/agent/agent.cfg
    volumes:
      - ./$SETUP/volumes/agent:/data/agent # should have agent.cfg, devices.xml
    ports:
      - 5000:5000
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # database - stores structure and time-series data
  database:
    container_name: database
    image: timescale/timescaledb:2.3.0-pg13
    ports:
      - $PGPORT:5432/tcp
    profiles:
      - all
      - database
    # init: true
    environment:
      POSTGRES_DB: $PGDATABASE
      POSTGRES_USERNAME: $PGUSER
      POSTGRES_PASSWORD: $PGPASSWORD
    volumes:
      - ./$SETUP/volumes/database/data:/var/lib/postgresql/data
    deploy:
      restart_policy:
        condition: on-failure
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # # database - stores structure and time-series data
  # database:
  #   container_name: database
  #   image: arangodb:3.7.12
  #   ports:
  #     - $ARANGO_PORT:8529
  #   profiles:
  #     - all
  #     - database
  #   # init: true
  #   # environment:
  #   #   ARANGO_HOST:
  #   #   ARANGO_PORT:
  #   #   ARANGO_DATABASE:
  #   #   ARANGO_USER:
  #   #   ARANGO_ROOT_PASSWORD:
  #   #   # ARANGO_PASSWORD:
  #   volumes:
  #     - ./$SETUP/volumes/database/data:/var/lib/arangodb3
  #     - ./$SETUP/volumes/database/apps:/var/lib/arangodb3-apps
  #   deploy:
  #     restart_policy:
  #       condition: on-failure
  #   networks:
  #     - ladder99
  #   logging:
  #     options:
  #       max-size: '1m'

  # influxdb - stores edge device data (for now - will move to postgres)
  influxdb:
    container_name: influxdb
    # image: influxdb:2.0 # not avail for 32bit - needs 64bit os
    image: influxdb:1.8 # need v1 for https://grafana.com/grafana/dashboards/12034
    ports:
      - '8086:8086'
    profiles:
      - all
      - edge
      - influxdb
    environment:
      INFLUXDB_DB:
      INFLUXDB_ADMIN_USER:
      INFLUXDB_ADMIN_PASSWORD:
      INFLUXDB_USER:
      INFLUXDB_PASSWORD:
    volumes:
      - ./$SETUP/volumes/influxdb/config:/etc/influxdb
      - ./$SETUP/volumes/influxdb/data:/var/lib/influxdb
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # telegraf - sends data from edge device to influxdb
  telegraf:
    container_name: telegraf
    image: telegraf:1.18
    profiles:
      - all
      - edge
      - telegraf
    environment:
      INFLUXDB_URL: http://influxdb:8086
      INFLUXDB_DB:
      INFLUXDB_PASSWORD:
    volumes:
      - ./$SETUP/volumes/telegraf:/etc/telegraf
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # grafana - shows real-time data from the databases, handles notifications
  grafana:
    container_name: grafana
    image: grafana/grafana:7.5.5
    profiles:
      - all
      - grafana
    ports:
      - $GRAFANA_PORT:3000/tcp
    environment:
      # need these for provisioning datasources
      # PGHOST: timescaledb
      PGHOST:
      PGPORT:
      PGDATABASE:
      PGUSER:
      PGPASSWORD:
      INFLUXDB_DB:
      INFLUXDB_USER:
      INFLUXDB_PASSWORD:
      # grafana will download and install these automatically
      # https://grafana.com/docs/grafana/latest/installation/docker/#install-plugins-in-the-docker-container
      GF_INSTALL_PLUGINS: natel-discrete-panel,natel-plotly-panel,michaeldmoore-scatter-panel
    volumes:
      # note: grafana.ini is located in /etc/grafana by default -
      # other folders are specified there.
      - ./$SETUP/volumes/grafana/etc:/etc/grafana
      - ./$SETUP/volumes/grafana/var:/var/lib/grafana
    # user: grafana
    #. get warning that these are blank, though echo $UID works - why?
    # but even if blank, now grafana is able to write to /var/lib/grafana.
    # gives permission error without this.
    # does blank here mean root?
    # user: '$UID:$GID'
    #.. don't use root here
    user: root
    deploy:
      restart_policy:
        condition: on-failure
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # pgadmin - admin console for postgres database
  pgadmin:
    container_name: pgadmin
    image: biarms/pgadmin4:latest
    profiles:
      - all
      - pgadmin
    ports:
      - '5050:5050/tcp'
    networks:
      - ladder99
    deploy:
      restart_policy:
        condition: on-failure

  # mqtt broker
  # gets mqtt data from devices and publishes it to mtconnect adapter.
  # we can playback mqtt data through this to simulate a device.
  broker:
    container_name: broker
    image: eclipse-mosquitto
    profiles:
      - all
      - broker
    ports:
      - 1883:1883
    volumes:
      - ./$SETUP/volumes/broker:/mosquitto # has config/mosquitto.conf
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # ladder99 adapter
  # converts machine data to shdr and sends to ladder99 agent
  adapter:
    container_name: adapter
    image: node:15.14-slim
    profiles:
      - all
      - adapter
    user: node
    working_dir: /home/node/app
    command: npm start
    volumes:
      - ../services/adapter:/home/node/app # has source code
      - ./$SETUP:/data/adapter # has setup.yaml
      - ../models:/data/models # has model yamls
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # ladder99 application - fetches data from agent and sends to db
  application:
    container_name: application
    image: node:15.14-slim
    profiles:
      - all
      - app
    user: node
    working_dir: /home/node/app
    # command: npm start
    command: node /home/node/app/src/application.js
    init: true
    # depends_on:
    #   - db
    environment:
      # override these as needed in setup compose.yaml
      AGENT_ENDPOINTS: http://agent:5000
      FETCH_INTERVAL: 2000
      FETCH_COUNT: 800
      PGHOST:
      PGPORT:
      PGDATABASE:
      PGUSER:
      PGPASSWORD:
      # ARANGO_HOST:
      # ARANGO_PORT:
      # ARANGO_DATABASE:
      # ARANGO_USER:
      # ARANGO_ROOT_PASSWORD:
      # # ARANGO_PASSWORD:
    # this fixes a problem on linux - works on mac okay also
    # see https://stackoverflow.com/a/67158212/243392
    extra_hosts:
      - host.docker.internal:host-gateway
    networks:
      - ladder99
    volumes:
      - ../services/application:/home/node/app # has source code
    logging:
      options:
        max-size: '1m'

  # play device recordings
  play:
    container_name: play
    build: ../services/recorder
    profiles:
      - play
    command: recorder --host broker --folder /data/models/$SETUP/recorder
    volumes:
      #   - ../../../services/simulator:/home/node/app
      - ../models:/data/models
    # environment:
    #   - HOST=broker
    #   - PORT=1883
    #   - FOLDER=../models/ccs-pa/recordings
    #   - LOOP=true
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # record a device
  record:
    container_name: record
    build: ../services/recorder
    profiles:
      - record
    command: recorder --mode record --host broker --folder ./models/$SETUP/recorder
    # environment:
    #   - HOST=broker
    #   - PORT=1883
    #   - FOLDER=../models/ccs-pa/recordings
    #   - LOOP=true
    networks:
      - ladder99
    logging:
      options:
        max-size: '1m'

  # # simulate device
  # simulator:
  #   container_name: simulator
  #   image: node:15.14-slim
  #   profiles:
  #     - simulator
  #   user: node
  #   working_dir: /home/node/app
  # #   command: npm start
  #   command: node src/simulator.js
  #   volumes:
  #     - ../../../services/simulator:/home/node/app
  #   networks:
  #     - ladder99
  #   logging:
  #     options:
  #       max-size: '1m'

  # portainer - docker container viewer/management
  portainer:
    container_name: portainer
    image: portainer/portainer-ce
    profiles:
      - all
      - portainer
    ports:
      - '9001:9000' #.... changed from 9000:9000 - make envar
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
      - /var/run/docker.sock:/var/run/docker.sock
      - ./$SETUP/volumes/portainer/data:/data
    restart: always
    networks:
      - ladder99
    logging:
      driver: 'none'

networks:
  ladder99:
    name: ladder99