feat: update examples, filters

meabed · Oct 7, 2018 · f8aa14e · f8aa14e
1 parent 54cb3fb
commit f8aa14e
Show file tree

Hide file tree

Showing 13 changed files with 176 additions and 29 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,6 +4,8 @@ services:
   logstash_tdd:
     image: docker.elastic.co/logstash/logstash:${TAG:-6.4.1}
     container_name: logstash_tdd
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
     volumes:
     - ./files/:/files
     - ./files/logstash.yml:/usr/share/logstash/config/logstash.yml

diff --git a/files/configuration/20-heruko-filter.conf b/files/configuration/20-heruko-filter.conf
@@ -4,7 +4,7 @@ filter {
 
     grok {
       match => { "message"=>".*" }
-      add_tag => [ "test_tag", "heroku_log"]
+      add_tag => [ "heroku_log" ]
     }
 
     # convert the message to key value

diff --git a/files/configuration/21-nginx-filter.conf b/files/configuration/21-nginx-filter.conf
diff --git a/files/configuration/22-apache-filter.conf b/files/configuration/22-apache-filter.conf
@@ -0,0 +1,31 @@
+filter {
+
+  grok {
+    match => [ "message" , "%{COMBINEDAPACHELOG}"]
+    overwrite => [ "message" ]
+    add_tag => [ "apache_log" ]
+  }
+
+  if "apache_log" in [tags] {
+    geoip {
+      source => "clientip"
+      target => "geoip"
+      add_tag => [ "apache_geoip" ]
+    }
+
+    date {
+      match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
+      remove_field => [ "timestamp" ]
+    }
+
+    useragent {
+      source => "agent"
+    }
+
+    mutate {
+      convert => ["response", "integer"]
+      convert => ["bytes", "integer"]
+      convert => ["responsetime", "float"]
+    }
+  }
+}
diff --git a/go-server/public/app.js b/go-server/public/app.js
@@ -51,11 +51,17 @@ app.controller('mainlCtrl', function ($scope, $http, $timeout, $location, $windo
   $scope.showUseful = 0;
   $scope.currentTab = 1;
 
-  $scope.logstashMessage = '';
   $scope.messagesFromLS = [];
   var storage = $scope.storage = $localStorage;
 
   storage.messageHistory = storage.messageHistory || [];
+  var messageHistoryLen = storage.messageHistory.length;
+
+  $scope.logstashMessage = '';
+
+  if (storage.messageHistory[messageHistoryLen - 1]) {
+    $scope.logstashMessage = storage.messageHistory[messageHistoryLen - 1].msg || '';
+  }
 
   socket.on('message:received', function (message) {
     var jsonMsg = JSON.parse(message);
@@ -65,6 +71,19 @@ app.controller('mainlCtrl', function ($scope, $http, $timeout, $location, $windo
     // console.log($scope.messages);
   });
 
+  $scope.tryExample = function (idx) {
+    $scope.logstashMessage = $scope.logExamples[idx].text;
+    $scope.sendToLogstash($scope.logstashMessage);
+  };
+
+  $scope.toggleExampleFilter = function (idx) {
+    console.log(idx)
+  };
+
+  $scope.copyExample = function (idx) {
+    console.log(idx)
+  };
+
   $scope.clearMessagesHistory = function () {
     storage.messageHistory = [];
   };

diff --git a/go-server/public/index.html b/go-server/public/index.html
@@ -36,21 +36,8 @@
     <div class="container" id="page">
         <ng-include src="'parts/useful-links.html'"></ng-include>
         <ng-include src="'parts/home-main-tabs.html'"></ng-include>
+        <ng-include src="'parts/msg-from-logstash.html'"></ng-include>
 
-
-        <div class="container row no-gutters">
-            <div class="pt-1 col-12">
-                <div class="d-inline-block">Messages from Logstash: {{ messagesFromLS.length }}</div>
-                <div ng-if="messagesFromLS.length > 0" class="d-inline-block float-right">
-                    <a href="#" ng-click="clearMessagesFromLS()" class="btn btn-outline-primary btn-sm">Clear</a>
-                </div>
-            </div>
-            <div class="container col-12">
-                <div class="card card-body ls-card" ng-repeat="obj in messagesFromLS | orderBy:'-inc' track by obj.inc">
-                    <div hljs hljs-source="'{{ obj.msg | prettyJSON }}'"></div>
-                </div>
-            </div>
-        </div>
     </div>
 </main>
 

diff --git a/go-server/public/log-examples.js b/go-server/public/log-examples.js
@@ -1,4 +1,22 @@
 window.logExamples = [
+  {
+    type: 'Apache Access Log',
+    text: '104.194.203.69 - - [01/Apr/2017:16:21:15 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"',
+    note: '',
+    filter_file: 'apache_access_1'
+  },
+  {
+    type: 'Nginx Log',
+    text: '104.194.203.69 - - [10/Nov/2017:15:17:20 +0000] "GET /favicon.ico HTTP/1.1" 200 199 "http://192.168.1.19/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"',
+    note: '',
+    filter_file: 'nginx_access_1'
+  },
+  {
+    type: 'Nginx Log',
+    text: '66.249.65.159 - - [06/Nov/2014:19:10:38 +0600] "GET /news/53f8d72920ba2744fe873ebc.html HTTP/1.1" 404 177 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"',
+    note: '',
+    filter_file: 'nginx_access_1'
+  },
   {
     type: 'heroku route',
     text: '2018-10-01T11:02:34.305524+00:00 heroku[router]: at=info method=GET path= "/convert?from= EUR&to= GBP&amount= 100&exchanger=yahoo" host=go-swap-server.herokuapp.com request_id=7cbfb25f-c86e-4201-ba21-7aa86085a6db fwd="63.143.42.249" dyno=web.1 connect=0ms service=96ms status=200 bytes=368 protocol=https'
@@ -22,5 +40,7 @@ window.logExamples = [
     type: 'AWS ELB SSL',
     text: '2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.001065 0.000015 0.000023 - - 57 502 "- - - " "-" ECDHE-ECDSA-AES128-GCM-SHA256 TLSv1.2',
     note: 'from: https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html'
-  }
+  },
 ];
+
+
diff --git a/go-server/public/parts/examples.html b/go-server/public/parts/examples.html
@@ -1,19 +1,20 @@
 <div ng-if="currentTab == 3" class="bg-lightx d-block">
     <div><h4>Example Log Messages</h4></div>
     <table class="table table-striped table-sm">
-        <tr class="d-flex" ng-repeat="(key, example) in this.logExamples track by $index">
-            <td class="col-2">{{ example.type }} </td>
-            <td class="col-10">{{ example.text }} </td>
+        <tr class="d-flex">
+            <td class="col-2">Type</td>
+            <td class="col-8">Message Text</td>
+            <td class="col-2">Action</td>
         </tr>
-    </table>
-    <br>
-    <div><h4>Example Logstash Filters</h4></div>
-    <div>Coming soon...</div>
-    <table class="table table-striped table-sm">
-        <tr class="d-flex" ng-repeat="(key, example) in this.logFilters track by $index">
+
+        <tr class="d-flex" ng-repeat="(key, example) in this.logExamples track by $index">
             <td class="col-2">{{ example.type }} </td>
-            <td class="col-10">{{ example.text }} </td>
+            <td class="col-8">{{ example.text }} </td>
+            <td class="col-2">
+                <a href="#" ng-click="copyExample($index)" class="btn btn-outline-primary btn-sm">Copy</a>
+                <a href="#" ng-click="tryExample($index)" class="btn btn-outline-primary btn-sm">Try</a>
+                <a href="#" ng-click="toggleExampleFilter($index)" class="btn btn-outline-primary btn-sm">Filter</a>
+            </td>
         </tr>
     </table>
-
 </div>
diff --git a/go-server/public/parts/filters/apache_access_1.html b/go-server/public/parts/filters/apache_access_1.html
@@ -0,0 +1,30 @@
+filter {
+
+  grok {
+    match => [ "message" , "%{COMBINEDAPACHELOG}"]
+    overwrite => [ "message" ]
+    add_tag => [ "apache_log" ]
+  }
+
+  geoip {
+    source => "clientip"
+    target => "geoip"
+    add_tag => [ "apache_geoip" ]
+  }
+
+  date {
+    match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
+    remove_field => [ "timestamp" ]
+  }
+
+  useragent {
+    source => "agent"
+  }
+
+  mutate {
+    convert => ["response", "integer"]
+    convert => ["bytes", "integer"]
+    convert => ["responsetime", "float"]
+  }
+
+}
diff --git a/go-server/public/parts/filters/nginx_1.html b/go-server/public/parts/filters/nginx_1.html
@@ -0,0 +1,29 @@
+filter {
+
+  grok {
+    match => [ "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}"]
+    overwrite => [ "message" ]
+  }
+
+  mutate {
+    convert => ["response", "integer"]
+    convert => ["bytes", "integer"]
+    convert => ["responsetime", "float"]
+  }
+
+  geoip {
+    source => "clientip"
+    target => "geoip"
+    add_tag => [ "nginx-geoip" ]
+  }
+
+  date {
+    match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
+    remove_field => [ "timestamp" ]
+  }
+
+  useragent {
+    source => "agent"
+  }
+
+}
diff --git a/go-server/public/parts/msg-from-logstash.html b/go-server/public/parts/msg-from-logstash.html
@@ -0,0 +1,13 @@
+<div class="container row no-gutters">
+    <div class="pt-1 col-12">
+        <div class="d-inline-block">Messages from Logstash: {{ messagesFromLS.length }}</div>
+        <div ng-if="messagesFromLS.length > 0" class="d-inline-block float-right">
+            <a href="#" ng-click="clearMessagesFromLS()" class="btn btn-outline-primary btn-sm">Clear</a>
+        </div>
+    </div>
+    <div class="container col-12">
+        <div class="pre-card resizable card card-body ls-card" ng-repeat="obj in messagesFromLS | orderBy:'-inc' track by obj.inc">
+            <div hljs hljs-source="'{{ obj.msg | prettyJSON }}'"></div>
+        </div>
+    </div>
+</div>
diff --git a/go-server/public/parts/useful-links.html b/go-server/public/parts/useful-links.html
@@ -15,6 +15,9 @@ <h4 class="card-title">Useful Links</h4>
                 <li>
                     <a href="https://packetrevolt.com/2017/09/1-billion-logs-per-day-with-elk-stack-hardware-and-scalable-architecture/" target="_blank">https://packetrevolt.com/2017/09/1-billion-logs-per-day-with-elk-stack-hardware-and-scalable-architecture/</a>
                 </li>
+                <li>
+                    <a href="https://krakensystems.co/blog/2018/logstash-nginx-logs-part-1" target="_blank">https://krakensystems.co/blog/2018/logstash-nginx-logs-part-1</a>
+                </li>
             </ul>
         </div>
     </div>

diff --git a/go-server/public/style.css b/go-server/public/style.css
@@ -39,15 +39,27 @@ div#page {
 
 pre {
     margin-bottom: 0;
-    max-height: 300px;
     overflow-x: scroll;
 }
 
 .card-body {
     padding: 0.75rem;
 }
 
+div.pre-card {
+    height: 350px;
+    max-height: 650px;
+}
+
+.resizable {
+    resize: both;
+    overflow: auto;
+}
+
 .footer i.fa {
     color: #77797a;
     font-size: 25px;
 }
+
+.hljs {
+}