(feat) logstash now parse application logs (manon.log)

DEPLOY.md

@@ -71,7 +71,6 @@ Application dockerized with [Jib](https://github.com/GoogleContainerTools/jib) a
   * Get application logs via: `GET /manon-app-*/_search`.
   * Get Nginx access logs via: `GET /manon-nginx-access-*/_search`.
   * You can delete these logs via: `DELETE /manon*`. Play with application and show logs again.
-
 * Optional: run Cerebro via Docker Compose: `./do upcerebro`.
   * Visit `http://localhost:9000` and select `Main Cluster` (it's an alias for `http://elasticsearch:9200`, see `config/docker/cerebro/cerebro.conf` file for details).
 

config/docker/docker-compose-elk.yml

@@ -16,7 +16,7 @@ services:
       - ~/manon-elastic-db:/usr/share/elasticsearch/data
       - ./elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
     environment:
-      ES_JAVA_OPTS: "-Xms32m -Xmx256m"
+      ES_JAVA_OPTS: "-Xms128m -Xmx256m"
     ulimits:
       memlock:
         soft: -1
@@ -33,8 +33,9 @@ services:
       - ~/manon-nginx-logs/:/manon-nginx/
       - ./logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
       - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
+      - ./logstash/patterns:/opt/logstash/patterns:ro
     environment:
-      LS_JAVA_OPTS: "-Xms32m -Xmx256m"
+      LS_JAVA_OPTS: "-Xms128m -Xmx256m"
     depends_on:
       - elasticsearch
 

config/docker/logstash/patterns/logstash-patterns.txt

@@ -0,0 +1,3 @@
+MSECOND [0-9]{3}
+LOGBACK_DATE %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}.{1}%{MSECOND}
+THREAD_ID [^\]]*

config/docker/logstash/pipeline/logstash-1-inputs.conf

@@ -0,0 +1,26 @@
+input {
+
+    # ------------------------------
+    # ------ Nginx access logs
+    # ------------------------------
+    file {
+        path => "/manon-nginx/access*.log"
+        start_position => "beginning"
+        type => "manon_nginx_access_logs"
+
+        # always (re)parse all existing files, don't use it in production
+        sincedb_path => "/dev/null"
+    }
+
+    # ------------------------------
+    # ------ Manon logs
+    # ------------------------------
+    file {
+        path => "/manon-app/manon*.log"
+        start_position => "beginning"
+        type => "manon_app_logs"
+
+        # always (re)parse all existing files, don't use it in production
+        sincedb_path => "/dev/null"
+    }
+}

config/docker/logstash/pipeline/logstash-2-filters.conf

@@ -0,0 +1,42 @@
+# see https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns
+
+filter {
+
+    # ------------------------------
+    # ------ Nginx access logs
+    # ------------------------------
+    if [type] == "manon_nginx_access_logs" {
+        grok {
+    		match => [ "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}"]
+            overwrite => [ "message" ]
+    	}
+    	mutate {
+           convert => ["response", "integer"]
+           convert => ["bytes", "integer"]
+           convert => ["responsetime", "float"]
+        }
+        date {
+            match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
+            remove_field => [ "timestamp" ]
+        }
+        useragent {
+            source => "agent"
+        }
+    }
+
+    # ------------------------------
+    # ------ Manon logs
+    # ------------------------------
+    if [type] == "manon_app_logs" {
+        grok {
+            match => { "message" =>"%{LOGBACK_DATE:timestamp} \[%{THREAD_ID:thread_id}\] %{NOTSPACE:log_lvl}\s+%{NOTSPACE:j_class} L\.%{NOTSPACE:j_line} \[%{DATA:j_mdc}\] %{GREEDYDATA:content}" }
+        }
+        date {
+            match => [ "timestamp" , "YYYY-MM-dd HH:mm:ss,SSS", "YYYY-MM-dd HH:mm:ss.SSS" ]
+        }
+        mutate {
+            remove_field => [ "timestamp", "message" ]
+            # strip => ["content"]
+        }
+    }
+}

config/docker/logstash/pipeline/logstash-3-outputs.conf

@@ -0,0 +1,34 @@
+output {
+
+    # ------------------------------
+    # ------ Nginx access logs
+    # ------------------------------
+    if [type] == "manon_nginx_access_logs" {
+	    elasticsearch {
+            hosts => ["elasticsearch:9200"]
+            index => "manon-nginx-access-%{+YYYY.MM.dd}"
+            document_type => "manon_nginx_access_logs"
+        }
+
+        ## uncomment to debug, don't use it in production
+        #stdout {
+        #    codec => "rubydebug"
+        #}
+    }
+
+    # ------------------------------
+    # ------ Manon logs
+    # ------------------------------
+    if [type] == "manon_app_logs" {
+        elasticsearch {
+            hosts => ["elasticsearch:9200"]
+            index => "manon-app-%{+YYYY.MM.dd}"
+            document_type => "manon_app_logs"
+        }
+
+        ## uncomment to debug, don't use it in production
+        #stdout {
+        #    codec => "rubydebug"
+        #}
+    }
+}

src/main/resources/application.yml

@@ -6,7 +6,7 @@ spring:
   batch:
     initialize-schema: always
     job.enabled: false
-  main.banner-mode: log
+  main.banner-mode: console
   datasource.driver-class-name: org.mariadb.jdbc.Driver
 
 server:
@@ -32,7 +32,6 @@ management:
     web:
       base-path: /actuator
       exposure.include: ["configprops", "env", "health", "info", "metrics", "scheduledtasks"]
-
 manon:
   admin:
     default-admin: