Add loki logpush via promtail

docker-compose.yml

@@ -18,10 +18,13 @@ services:
         STAKER_SCRIPTS_VERSION: v0.1.1
     restart: unless-stopped
     environment:
-      SHUTTER_API_NODE_PRIVATEKEY: ""
-      SHUTTER_PUSH_METRICS_ENABLED: false
-      KEYPER_NAME: ""
-      ETHEREUM_WS: ""
+      - SHUTTER_API_NODE_PRIVATEKEY=""
+      - SHUTTER_PUSH_METRICS_ENABLED=false
+      - KEYPER_NAME=""
+      - ETHEREUM_WS=""
+      - SHUTTER_PUSH_LOGS_ENABLED=false
+      - PUSHGATEWAY_USERNAME=""
+      - PUSHGATEWAY_PASSWORD=""
     volumes:
       - chain:/chain
       - keyper-config:/keyper/config

setup-wizard.yml

@@ -31,6 +31,18 @@ fields:
       service: shutter
     required: true
 
+  - id: enable_push_logs
+    title: Enable Push Logs
+    description: |
+      Enable the push logs feature to send logs to an external server controlled by Shutter.
+    target:
+      type: environment
+      name: SHUTTER_PUSH_LOGS_ENABLED
+      service: [shutter, metrics]
+    enum:
+      - "true"
+      - "false"
+
   - id: enable_push_metrics
     title: Enable Push Metrics
     description: |
@@ -46,7 +58,7 @@ fields:
   - id: pushgateway_url
     title: Pushgateway URL
     description: |
-      The URL of the Pushgateway server to send metrics to.
+      The URL of the Pushgateway server to send metrics/logs to.
     target:
       type: environment
       name: PUSHGATEWAY_URL
@@ -61,7 +73,7 @@ fields:
     target:
       type: environment
       name: PUSHGATEWAY_USERNAME
-      service: metrics
+      service: [metrics, shutter]
     required: false
     if: { enable_push_metrics: { "enum": ["true"] } }
 
@@ -72,7 +84,7 @@ fields:
     target:
       type: environment
       name: PUSHGATEWAY_PASSWORD
-      service: metrics
+      service: [metrics, shutter]
     required: false
     secret: true
     if: { enable_push_metrics: { "enum": ["true"] } }

shutter/Dockerfile

@@ -1,7 +1,7 @@
 ARG ASSETS_VERSION
 ARG UPSTREAM_VERSION
 
-FROM ghcr.io/shutter-network/assets:${ASSETS_VERSION} as assets
+FROM ghcr.io/shutter-network/assets:${ASSETS_VERSION} AS assets
 
 RUN rsync -aq --delete /assets-source/ /assets/
 
@@ -40,6 +40,7 @@ ADD ${STAKER_SCRIPTS_URL}/dvt_lsd_tools.sh /etc/profile.d/
 
 COPY go-shutter-settings ${SHUTTER_SETTINGS_SRC_DIR}
 COPY supervisord.conf /etc/supervisord.conf
+COPY promtail_config.yaml /etc/promtail_config.yaml
 
 RUN go build -C ${SHUTTER_SETTINGS_SRC_DIR} -o /usr/local/bin/go_shutter_settings
 
@@ -49,9 +50,18 @@ RUN mkdir -p ${KEYPER_CONFIG_DIR} ${SHUTTER_CHAIN_DIR} ${ASSETS_DIR} /opt/superv
 COPY scripts /usr/local/bin/
 COPY --from=assets ${ASSETS_DIR}/ ${ASSETS_DIR}/
 
+# For pushing logs to loki
+RUN apt-get -y install wget gpg
+RUN mkdir -p /etc/apt/keyrings/
+RUN wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor > /etc/apt/keyrings/grafana.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list
+# promtail & rotatelogs (from apache2)
+RUN apt-get update && apt-get -y install promtail apache2
+
+
 # Placed here to rebuild less layers
 ENV CHAIN_PORT=${CHAIN_PORT} \
     SHUTTER_P2P_LISTENADDRESSES="/ip4/0.0.0.0/tcp/${KEYPER_PORT},/ip4/0.0.0.0/udp/${KEYPER_PORT}/quic-v1" \
     NETWORK=${NETWORK}
 
-ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]
+ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]

shutter/promtail_config.yaml

@@ -0,0 +1,46 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+client:
+  url: https://logs.metrics.shutter.network/insert/loki/api/v1/push
+  basic_auth:
+    username: ${PUSHGATEWAY_USERNAME}
+    password: ${PUSHGATEWAY_PASSWORD}
+
+scrape_configs:
+ - job_name: configure
+   pipeline_stages:
+   - docker:
+   static_configs:
+   - targets:
+      - localhost
+     labels:
+      job: configure
+      host: ${KEYPER_NAME}
+      __path__: /tmp/configure.log
+
+ - job_name: keyper
+   pipeline_stages:
+   - docker:
+   static_configs:
+   - targets:
+      - localhost
+     labels:
+      job: keyper
+      host: ${KEYPER_NAME}
+      __path__: /tmp/keyper.log
+
+ - job_name: chain
+   pipeline_stages:
+   - docker:
+   static_configs:
+   - targets:
+      - localhost
+     labels:
+      job: chain
+      host: ${KEYPER_NAME}
+      __path__: /tmp/chain.log

shutter/scripts/run_chain.sh

@@ -3,8 +3,12 @@
 run_chain() {
 
     echo "[INFO | chain] Starting chain..."
-
-    $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE"
+    if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]];
+    then
+        $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" |& rotatelogs -n 1 -e -c /tmp/chain.log 5M
+    else
+        $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE"
+    fi
 }
 
 run_chain

shutter/scripts/run_configure.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+
+if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]];
+then
+    configure.sh |& rotatelogs -n 1 -e -c /tmp/configure.log 5M
+else
+    configure.sh
+fi

shutter/scripts/run_keyper.sh

@@ -18,7 +18,12 @@ perform_chain_healthcheck() {
 }
 
 run_keyper() {
-    $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE"
+    if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]];
+    then
+        $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE" |& rotatelogs -n 1 -e -c /tmp/keyper.log 5M
+    else
+        $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE"
+    fi
 }
 
 perform_chain_healthcheck

shutter/scripts/run_promtail.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+run_promtail() {
+    if [[ $SHUTTER_PUSH_LOGS_ENABLED=true ]];
+    then
+        promtail -config.expand-env=true -log-config-reverse-order -print-config-stderr -config.file /etc/promtail_config.yaml
+    else
+        tail -f /dev/null
+    fi
+}
+
+run_promtail

shutter/supervisord.conf

@@ -18,7 +18,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 serverurl=unix:///var/run/supervisor.sock
 
 [program:configure]
-command = configure.sh
+command = run_configure.sh
 priority = 1
 autostart = true
 autorestart = false
@@ -47,4 +47,16 @@ autorestart = true
 stdout_logfile = /dev/stdout
 stdout_logfile_maxbytes = 0
 stderr_logfile = /dev/stderr
-stderr_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+
+[program:promtail]
+command = run_promtail.sh ; we ingest 'rotatelogs' managed logfiles -- there is potential for some missed log lines.
+priority = 4
+autostart = %(ENV_SHUTTER_PUSH_LOGS_ENABLED)s
+startretries = 9999  ; A large number enough to cover node updates
+autorestart = %(ENV_SHUTTER_PUSH_LOGS_ENABLED)s
+environment = KEYPER_NAME="%(ENV_KEYPER_NAME)s",PUSHGATEWAY_USERNAME="%(ENV_PUSHGATEWAY_USERNAME)s",PUSHGATEWAY_PASSWORD="%(ENV_PUSHGATEWAY_PASSWORD)s",SHUTTER_PUSH_LOGS_ENABLED="%(ENV_SHUTTER_PUSH_LOGS_ENABLED)s"
+stdout_logfile = /dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile = /dev/stderr
+stderr_logfile_maxbytes = 0