diff --git a/docker-compose.yml b/docker-compose.yml index 72d31bf..cc8597c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,10 +18,13 @@ services: STAKER_SCRIPTS_VERSION: v0.1.1 restart: unless-stopped environment: - SHUTTER_API_NODE_PRIVATEKEY: "" - SHUTTER_PUSH_METRICS_ENABLED: false - KEYPER_NAME: "" - ETHEREUM_WS: "" + - SHUTTER_API_NODE_PRIVATEKEY="" + - SHUTTER_PUSH_METRICS_ENABLED=false + - KEYPER_NAME="" + - ETHEREUM_WS="" + - SHUTTER_PUSH_LOGS_ENABLED=false + - PUSHGATEWAY_USERNAME="" + - PUSHGATEWAY_PASSWORD="" volumes: - chain:/chain - keyper-config:/keyper/config diff --git a/setup-wizard.yml b/setup-wizard.yml index 7a8a7f1..2ea3fed 100644 --- a/setup-wizard.yml +++ b/setup-wizard.yml @@ -31,6 +31,18 @@ fields: service: shutter required: true + - id: enable_push_logs + title: Enable Push Logs + description: | + Enable the push logs feature to send logs to an external server controlled by Shutter. + target: + type: environment + name: SHUTTER_PUSH_LOGS_ENABLED + service: [shutter, metrics] + enum: + - "true" + - "false" + - id: enable_push_metrics title: Enable Push Metrics description: | @@ -46,7 +58,7 @@ fields: - id: pushgateway_url title: Pushgateway URL description: | - The URL of the Pushgateway server to send metrics to. + The URL of the Pushgateway server to send metrics/logs to. target: type: environment name: PUSHGATEWAY_URL @@ -61,7 +73,7 @@ fields: target: type: environment name: PUSHGATEWAY_USERNAME - service: metrics + service: [metrics, shutter] required: false if: { enable_push_metrics: { "enum": ["true"] } } @@ -72,7 +84,7 @@ fields: target: type: environment name: PUSHGATEWAY_PASSWORD - service: metrics + service: [metrics, shutter] required: false secret: true if: { enable_push_metrics: { "enum": ["true"] } } diff --git a/shutter/Dockerfile b/shutter/Dockerfile index 94da8fc..244209f 100644 --- a/shutter/Dockerfile +++ b/shutter/Dockerfile @@ -1,7 +1,7 @@ ARG ASSETS_VERSION ARG UPSTREAM_VERSION -FROM ghcr.io/shutter-network/assets:${ASSETS_VERSION} as assets +FROM ghcr.io/shutter-network/assets:${ASSETS_VERSION} AS assets RUN rsync -aq --delete /assets-source/ /assets/ @@ -40,6 +40,7 @@ ADD ${STAKER_SCRIPTS_URL}/dvt_lsd_tools.sh /etc/profile.d/ COPY go-shutter-settings ${SHUTTER_SETTINGS_SRC_DIR} COPY supervisord.conf /etc/supervisord.conf +COPY promtail_config.yaml /etc/promtail_config.yaml RUN go build -C ${SHUTTER_SETTINGS_SRC_DIR} -o /usr/local/bin/go_shutter_settings @@ -49,9 +50,18 @@ RUN mkdir -p ${KEYPER_CONFIG_DIR} ${SHUTTER_CHAIN_DIR} ${ASSETS_DIR} /opt/superv COPY scripts /usr/local/bin/ COPY --from=assets ${ASSETS_DIR}/ ${ASSETS_DIR}/ +# For pushing logs to loki +RUN apt-get -y install wget gpg +RUN mkdir -p /etc/apt/keyrings/ +RUN wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor > /etc/apt/keyrings/grafana.gpg +RUN echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list +# promtail & rotatelogs (from apache2) +RUN apt-get update && apt-get -y install promtail apache2 + + # Placed here to rebuild less layers ENV CHAIN_PORT=${CHAIN_PORT} \ SHUTTER_P2P_LISTENADDRESSES="/ip4/0.0.0.0/tcp/${KEYPER_PORT},/ip4/0.0.0.0/udp/${KEYPER_PORT}/quic-v1" \ NETWORK=${NETWORK} -ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file +ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"] diff --git a/shutter/promtail_config.yaml b/shutter/promtail_config.yaml new file mode 100644 index 0000000..f241855 --- /dev/null +++ b/shutter/promtail_config.yaml @@ -0,0 +1,46 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +client: + url: https://logs.metrics.shutter.network/insert/loki/api/v1/push + basic_auth: + username: ${PUSHGATEWAY_USERNAME} + password: ${PUSHGATEWAY_PASSWORD} + +scrape_configs: + - job_name: configure + pipeline_stages: + - docker: + static_configs: + - targets: + - localhost + labels: + job: configure + host: ${KEYPER_NAME} + __path__: /tmp/configure.log + + - job_name: keyper + pipeline_stages: + - docker: + static_configs: + - targets: + - localhost + labels: + job: keyper + host: ${KEYPER_NAME} + __path__: /tmp/keyper.log + + - job_name: chain + pipeline_stages: + - docker: + static_configs: + - targets: + - localhost + labels: + job: chain + host: ${KEYPER_NAME} + __path__: /tmp/chain.log diff --git a/shutter/scripts/run_chain.sh b/shutter/scripts/run_chain.sh index 3167447..c5e7265 100755 --- a/shutter/scripts/run_chain.sh +++ b/shutter/scripts/run_chain.sh @@ -3,8 +3,12 @@ run_chain() { echo "[INFO | chain] Starting chain..." - - $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" + if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]]; + then + $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" |& rotatelogs -n 1 -e -c /tmp/chain.log 5M + else + $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" + fi } run_chain diff --git a/shutter/scripts/run_configure.sh b/shutter/scripts/run_configure.sh new file mode 100755 index 0000000..484dbf6 --- /dev/null +++ b/shutter/scripts/run_configure.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]]; +then + configure.sh |& rotatelogs -n 1 -e -c /tmp/configure.log 5M +else + configure.sh +fi diff --git a/shutter/scripts/run_keyper.sh b/shutter/scripts/run_keyper.sh index 85cede4..db6ae46 100755 --- a/shutter/scripts/run_keyper.sh +++ b/shutter/scripts/run_keyper.sh @@ -18,7 +18,12 @@ perform_chain_healthcheck() { } run_keyper() { - $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE" + if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]]; + then + $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE" |& rotatelogs -n 1 -e -c /tmp/keyper.log 5M + else + $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE" + fi } perform_chain_healthcheck diff --git a/shutter/scripts/run_promtail.sh b/shutter/scripts/run_promtail.sh new file mode 100755 index 0000000..f335d42 --- /dev/null +++ b/shutter/scripts/run_promtail.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +run_promtail() { + if [[ $SHUTTER_PUSH_LOGS_ENABLED=true ]]; + then + promtail -config.expand-env=true -log-config-reverse-order -print-config-stderr -config.file /etc/promtail_config.yaml + else + tail -f /dev/null + fi +} + +run_promtail diff --git a/shutter/supervisord.conf b/shutter/supervisord.conf index bfc6ccc..4c6b47c 100644 --- a/shutter/supervisord.conf +++ b/shutter/supervisord.conf @@ -18,7 +18,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface serverurl=unix:///var/run/supervisor.sock [program:configure] -command = configure.sh +command = run_configure.sh priority = 1 autostart = true autorestart = false @@ -47,4 +47,16 @@ autorestart = true stdout_logfile = /dev/stdout stdout_logfile_maxbytes = 0 stderr_logfile = /dev/stderr -stderr_logfile_maxbytes = 0 \ No newline at end of file +stderr_logfile_maxbytes = 0 + +[program:promtail] +command = run_promtail.sh ; we ingest 'rotatelogs' managed logfiles -- there is potential for some missed log lines. +priority = 4 +autostart = %(ENV_SHUTTER_PUSH_LOGS_ENABLED)s +startretries = 9999 ; A large number enough to cover node updates +autorestart = %(ENV_SHUTTER_PUSH_LOGS_ENABLED)s +environment = KEYPER_NAME="%(ENV_KEYPER_NAME)s",PUSHGATEWAY_USERNAME="%(ENV_PUSHGATEWAY_USERNAME)s",PUSHGATEWAY_PASSWORD="%(ENV_PUSHGATEWAY_PASSWORD)s",SHUTTER_PUSH_LOGS_ENABLED="%(ENV_SHUTTER_PUSH_LOGS_ENABLED)s" +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0