01-logstash-pipeline.conf

input {
  beats {
    port => 5044
  }
}

filter {


  grok {
    match => {
      "[docker][attrs][tag]"  => [
        # "blockchain|execution|nethermind|nethermind/nethermind:latest|blockchain_dc_execution_1|sha256:87432e29ccf560fb120ef8cc494a1ea4084f8c48351cf8a5ecab35eb0a2170a4|65c6097cf7694be2b454ee3eb817095d78c01b03ea91a4cf47e022d6d7a93f45"
        "%{WORD:stack}\|%{WORD:layer}\|%{WORD:client}\|%{DATA:image_name}\|%{DATA:container_name}\|%{DATA:full_id}\|%{WORD:id}",
        # "observability|grafana/grafana:main|grafana|sha256:c7f0d28043d769f26c8bc67add8b6cff10c53b2680d51ce1076aea02e5303999|74ec24494d32"
        "%{WORD:stack}\|%{DATA:image_name}\|%{DATA:container_name}\|%{DATA:full_id}\|%{WORD:id}"
      ]
    }
  }

  if [stack] == "blockchain" {


    if [client] == "nethermind" {

      grok {
        match => {
          # 2022-12-09 07:46:38.8872|Full Pruning In Progress: 11:54:36.3212318 174.00 mln nodes mirrored.
          "[message]"  => "%{TIMESTAMP_ISO8601:t}\|%{GREEDYDATA:msg}"

        }
     }

    }

    else if [client] == "besu" {

      grok {
        match => {
          # 2022-12-19 07:13:48.455+00:00 | main | INFO | JsonRpcHttpService | Starting JSON-RPC service on 0.0.0.0:8545
          # 2022-12-19 07:34:15.400+00:00 | Timer-0 | INFO | DNSResolver | Resolved 426 nodes
          # 2022-12-19 07:13:48.471+00:00 | vert.x-eventloop-thread-1 | INFO | JsonRpcHttpService | JSON-RPC service started and listening on 0.0.0.0:8545
          # Note: sometimes there are two spaces after level
          # 2022-12-19 07:15:35.205+00:00 | vert.x-worker-thread-0 | INFO  | AbstractBlockProcessor | Block processing error: transaction invalid Internal Error in Besu
          "[message]"  => "%{TIMESTAMP_ISO8601:t}\s\|\s%{GREEDYDATA:thread}\s\|\s%{LOGLEVEL:level}\s+\|\s%{WORD:module}\s\|\s%{GREEDYDATA:msg}"
        }

      }

    }

    else if [client] == "teku" {

      grok {
        match => {
          "[message]"  => [
            # 2023-02-13 14:30:16.021 INFO -  [37mSlot Event *** Slot: 4982551, Block: ... empty, Justified: 155703, Finalized: 155701, Peers: 8 [0m
            # 2023-02-13 14:29:04.467 WARN - Detected peer with inconsistent finalized epoch 155700; block at slot 4982400 for peer 16Uiu2HAmN5Y996jasrbztxmcTuJShX3g5MXDR6hsRZYDpmeFam2V. Local root: 0x8df7790e597e419e0c65433fc40aeaeab0d8c387778c21ccae438b0ba9bb5565; remote root: 0x04afb499b1ccf917adfd06949d4759784f08ceab57b4f9b523cc8b252b788b76
            # 2023-02-13 14:29:00.001 INFO - Updating number of persistent subnet subscriptions from 0 to 1
            "%{TIMESTAMP_ISO8601:t}\s%{LOGLEVEL:level}\s+\-\s+%{GREEDYDATA:msg}",
            # 2023-02-14 06:39:02,592 main INFO Logging includes validator duties: true
            "%{TIMESTAMP_ISO8601:t}\s%{WORD}\s%{LOGLEVEL:level}\s+%{GREEDYDATA:msg}"
            # Teku is shutting down
            # "%{GREEDYDATA:msg}"
          ]
        }

      }
    }

    else if [client] == "lodestar" {

      grok {
        match => {
          "[message]"  => [
            # Log are coloured by default so we had to add extra grok patter for them \d\dm%{LOGLEVEL:level}.*\[\d\dm
            "%{GREEDYDATA:t}\[%{DATA:origin}\]\s+.*\[\d\dm%{LOGLEVEL:level}.*\[\d\dm\:\s+%{GREEDYDATA:msg}"
          ]
        }

      }
    }

    # Geth, Erigon, Lighthouse  logs are valid JSON
    else {

      # {"lvl":"info","msg":"Starting Geth on Görli testnet...","t":"2022-11-24T07:43:38.070844459Z"}
      json {
        source => "message"
      }

      # Lighthouse timestamp field name is "ts"
      mutate {
        rename => {"ts" => "t"}
      }

    }


    # Overwrite @timestamp with the application timestamp
    date {
      timezone => "Etc/UTC"
      match => ["t" , "ISO8601", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSSS", "yyyy-MM-dd HH:mm:ss,SSS", "MMM-dd HH:mm:ss.SSS"]
      target => "@timestamp"
      remove_field => [ "t" ]
    }

    # Remove docker metadata
    mutate {
      remove_field => [ "[docker][container][labels]"]
    }

  }

}


output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    manage_template => false
    index => "%{[log_type]}-%{+YYYY.MM.dd}"
  }
}