structured syslog fields are now parsed

into additional_fields. fixes #SERVER.93, relates #SERVER-92

src/main/java/org/graylog2/messagehandlers/gelf/GELFMessage.java

@@ -229,6 +229,10 @@ public Map<String, Object> getAdditionalData() {
      * @param value
      */
     public void addAdditionalData(String key, Object value) {
+        if (!key.startsWith(GELF.USER_DEFINED_FIELD_PREFIX)) {
+           key = GELF.USER_DEFINED_FIELD_PREFIX + key;
+        }
+
         if (key != null && value != null) {
 
             if (value instanceof Long) {
@@ -245,6 +249,16 @@ public void addAdditionalData(String key, Object value) {
         }
     }
 
+    /**
+     * Add a whole set of additional fields.
+     * @param fields
+     */
+    public void addAdditionalData(Map<String, String> fields) {
+        for (Map.Entry<String, String> field : fields.entrySet()) {
+            addAdditionalData(field.getKey(), field.getValue());
+        }
+    }
+
     /**
      * Set the filterOut
      *

src/main/java/org/graylog2/messagehandlers/syslog/StructuredSyslog.java

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2011 Lennart Koopmann <lennart@socketfeed.com>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.graylog2.messagehandlers.syslog;
+
+import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.log4j.Logger;
+import org.productivity.java.syslog4j.server.impl.event.structured.StructuredSyslogServerEvent;
+
+/**
+ * StructuredSyslog.java: Dec 24, 2011 5:32:06 PM
+ *
+ * Parses structured syslog data.
+ *
+ * @author Lennart Koopmann <lennart@socketfeed.com>
+ */
+public class StructuredSyslog {
+
+    private static final Logger LOG = Logger.getLogger(StructuredSyslog.class);
+
+    public static Map<String, String> extractFields(byte[] rawSyslogMessage) {
+        Map<String, String> fields = new HashMap<String, String>();
+        try {
+            StructuredSyslogServerEvent s = new StructuredSyslogServerEvent(
+                    rawSyslogMessage,
+                    rawSyslogMessage.length,
+                    InetAddress.getLocalHost()
+             );
+
+            Map raw = s.getStructuredMessage().getStructuredData();
+            if (raw != null) {
+                Set ks = raw.keySet();
+                if (ks.size() > 0) {
+                    Object[] fl = raw.keySet().toArray();
+
+                    if (fl != null && fl.length > 0) {
+                        String sdID = (String) fl[0];
+                        fields = (HashMap) raw.get(sdID);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            LOG.debug("Could not extract structured syslog", e);
+            return new HashMap();
+        }
+
+        return fields;
+    }
+
+}

src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java

@@ -20,6 +20,8 @@
 
 package org.graylog2.messagehandlers.syslog;
 
+import java.net.InetAddress;
+import java.util.logging.Level;
 import org.apache.log4j.Logger;
 import org.graylog2.Tools;
 import org.graylog2.messagehandlers.gelf.GELFMessage;
@@ -31,7 +33,10 @@
 import java.net.SocketAddress;
 import java.net.UnknownHostException;
 import java.util.Date;
+import java.util.Map;
 import org.graylog2.Main;
+import org.productivity.java.syslog4j.impl.message.structured.StructuredSyslogMessage;
+import org.productivity.java.syslog4j.server.impl.event.structured.StructuredSyslogServerEvent;
 
 /**
  * SyslogEventHandler.java: May 17, 2010 8:58:18 PM
@@ -84,6 +89,25 @@ public void event(SyslogServerIF syslogServer, SocketAddress socketAddress, Sysl
             }
         }
 
+        // try to parse from event. if that fails or nothing is parsed, tokenize self.
+        // Parse possibly included structured syslog data into additional_fields.
+        Map<String, String> structuredData = StructuredSyslog.extractFields(event.getRaw());
+        if (structuredData.size() > 0) {
+            // We were able to parse structured data from the message. Add as additional fields.
+            LOG.debug("Parsed <" + structuredData.size() + "> structured data pairs."
+                        + " Adding as additional_fields. Not using tokenizer.");
+            gelf.addAdditionalData(structuredData);
+        } else {
+            /*
+             * There was no structured data to be parsed or parsing failed.
+             *
+             * This means that we can safely extract values with the Tokenizer
+             * without interfering with structured data.
+             */
+             LOG.debug("No structured data was parsed from message. Using tokenizer.");
+            // XXX IMPLEMENT
+        }
+
         // Possibly overwrite host with RNDS if configured.
         String host = event.getHost();
         try {

src/main/java/org/graylog2/messagehandlers/syslog/Tokenizer.java

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2011 Lennart Koopmann <lennart@socketfeed.com>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.graylog2.messagehandlers.syslog;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.productivity.java.syslog4j.server.impl.event.SyslogServerEvent;
+
+/**
+ * Tokenizer.java: Dec 24, 2011 4:54:31 PM
+ *
+ * Breaks down syslog messages into additional_fields if they could not
+ * be parsed as structured syslog.
+ *
+ * @author Lennart Koopmann <lennart@socketfeed.com>
+ */
+public class Tokenizer {
+
+    char[] chseparators = { '=' };
+
+    public static Map extractAdditionalFields(SyslogServerEvent msg) {
+        Map extracted = new HashMap();
+
+        return extracted;
+    }
+
+    // No spaces between = and
+
+}

src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java

@@ -20,13 +20,16 @@
 
 package org.graylog2.messagehandlers.gelf;
 
+import java.util.Map;
+import java.lang.Object;
 import com.mongodb.BasicDBList;
 import com.mongodb.BasicDBObject;
 import org.bson.types.ObjectId;
 import org.graylog2.blacklists.Blacklist;
 import org.junit.Test;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
 import static org.junit.Assert.*;
@@ -137,7 +140,7 @@ public void testToOneliner() {
 
         GELFMessage gelfMessage = createGELFMessage();
 
-        String oneLiner = "host.example.com - short message severity=Emergency,facility=local0,file=test.file,line=42,test=test";
+        String oneLiner = "host.example.com - short message severity=Emergency,facility=local0,file=test.file,line=42,_test=test";
         assertEquals(oneLiner, gelfMessage.toOneLiner());
     }
 
@@ -207,4 +210,29 @@ public void testAllRequiredFieldsSet() {
         GELFMessage gelfMessage = createGELFMessage();
         assertTrue(gelfMessage.allRequiredFieldsSet());
     }
+
+    @Test
+    public void testAddAdditionalData() {
+        GELFMessage msg = new GELFMessage();
+        msg.addAdditionalData("_foo", "bar");
+        msg.addAdditionalData("lol", "wat"); // _ should be added automatically.
+
+        Map<String, Object> expected = new HashMap<String, Object>();
+        expected.put("_foo", "bar");
+        expected.put(("_lol"), "wat");
+
+        assertEquals(expected, msg.getAdditionalData());
+    }
+
+    @Test
+    public void testAddAdditionalDataWithMap() {
+        Map<String, String> fields = new HashMap<String, String>();
+        fields.put("_foo", "bar");
+        fields.put("_lol", "wat");
+
+        GELFMessage msg = new GELFMessage();
+        msg.addAdditionalData(fields);
+
+        assertEquals(fields, msg.getAdditionalData());
+    }
 }

src/test/java/org/graylog2/messagehandlers/syslog/StructuredSyslogTest.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2011 Lennart Koopmann <lennart@socketfeed.com>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.graylog2.messagehandlers.syslog;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class StructuredSyslogTest {
+
+    // http://tools.ietf.org/rfc/rfc5424.txt
+    public static String ValidStructuredMessage = "<165>1 2012-12-25T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] BOMAn application event log entry";
+    public static String ValidNonStructuredMessage = "<86>Dec 24 17:05:01 nb-lkoopmann CRON[10049]: pam_unix(cron:session): session closed for user root";
+    public static String MessageLookingLikeStructured = "<133>NOMA101FW01A: NetScreen device_id=NOMA101FW01A [Root]system-notification-00257(traffic): start_time=\"2011-12-23 17:33:43\" duration=0 reason=Creation";
+
+    @Test
+    public void testExtractFields() {
+        Map expected = new HashMap();
+        expected.put("eventSource", "Application");
+        expected.put("eventID", "1011");
+        expected.put("iut", "3");
+
+        Map result = StructuredSyslog.extractFields(ValidStructuredMessage.getBytes());
+        assertEquals(expected, result);
+    }
+
+    @Test
+    public void testExtractFieldsOfNonStructuredMessage() {
+        Map result = StructuredSyslog.extractFields(ValidNonStructuredMessage.getBytes());
+        assertEquals(0, result.size());
+    }
+
+    @Test
+    public void testExtractFieldsOfAMessageThatOnlyLooksStructured() {
+        Map result = StructuredSyslog.extractFields(MessageLookingLikeStructured.getBytes());
+        assertEquals(0, result.size());
+    }
+
+}