Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: kroepke/graylog2-server
...
head fork: kroepke/graylog2-server
compare: f8a352905c
Checking mergeability… Don't worry, you can still create the pull request.
  • 6 commits
  • 11 files changed
  • 0 commit comments
  • 1 contributor
Commits on Dec 23, 2011
@lennartkoopmann lennartkoopmann fix dependency bug in Makefile
fixes #SERVER-89
0642521
@lennartkoopmann lennartkoopmann bumped version to 0.9.6 425b7dd
Commits on Dec 24, 2011
@lennartkoopmann lennartkoopmann bumped version to 0.9.7-dev 584253b
@lennartkoopmann lennartkoopmann fix wrong debug output of syslog messages
fixes #SERVER-91
d6b8de9
@lennartkoopmann lennartkoopmann allowe to override date of syslog messages to NOW
this takes place if the date could not be parsed. configurable via graylog2.conf.
fixes #SERVER-90
00e964e
Commits on Dec 25, 2011
@lennartkoopmann lennartkoopmann structured syslog fields are now parsed
into additional_fields. fixes #SERVER.93, relates #SERVER-92
f8a3529
View
8 Makefile
@@ -2,8 +2,8 @@ NAME=graylog2-server
PREFIX=/usr
DESTDIR=
-SERVER_W_DEP=target/graylog2-server-0.9.6-SNAPSHOT-jar-with-dependencies.jar
-SERVER=target/graylog2-server-0.9.6-SNAPSHOT.jar
+SERVER_W_DEP=target/graylog2-server-0.9.6-jar-with-dependencies.jar
+SERVER=target/graylog2-server-0.9.6.jar
SYSLOG4J=lib/syslog4j-0.9.46-bin.jar
INITD=contrib/distro/generic/graylog2-server.init.d
CONF=misc/graylog2.conf
@@ -11,11 +11,15 @@ CONF=misc/graylog2.conf
MVN_REPO="/tmp/$(NAME)-build-${USER}"
MVN_OPTS=-Dmaven.repo.local=${MVN_REPO}
+all: $(SERVER) $(SERVER_W_DEP) prepare
all: $(SERVER) $(SERVER_W_DEP) test
$(SERVER) $(SERVER_W_DEP):
mvn $(MVN_OPTS) assembly:assembly
+prepare:
+ mvn install:install-file $(MVN_OPTS) -DgroupId=org.syslog4j -DartifactId=syslog4j -Dversion=0.9.46 -Dpackaging=jar -Dfile=lib/syslog4j-0.9.46-bin.jar
+
test:
mvn $(MVN_OPTS) test
View
2  misc/graylog2.conf
@@ -8,6 +8,8 @@ elasticsearch_index_name = graylog2
# Always try a reverse DNS lookup instead of parsing hostname from syslog message?
force_syslog_rdns = false
+# Set time to NOW if parsing date/time from syslog message failed instead of rejecting it?
+allow_override_syslog_date = true
# MongoDB Configuration
mongodb_useauth = true
View
2  pom.xml
@@ -6,7 +6,7 @@
<groupId>org.graylog2</groupId>
<artifactId>graylog2-server</artifactId>
- <version>0.9.6-SNAPSHOT</version>
+ <version>0.9.7-dev</version>
<packaging>jar</packaging>
<properties>
View
7 src/main/java/org/graylog2/Configuration.java
@@ -59,6 +59,9 @@
@Parameter(value = "mongodb_useauth", required = true)
private boolean mongoUseAuth = false;
+ @Parameter(value = "allow_override_syslog_date", required = true)
+ private boolean allowOverrideSyslogDate = true;
+
@Parameter(value = "elasticsearch_url", required = true)
private String elasticsearchUrl = "http://localhost:9200/";
@@ -149,6 +152,10 @@ public boolean getForceSyslogRdns() {
return forceSyslogRdns;
}
+ public boolean getAllowOverrideSyslogDate() {
+ return allowOverrideSyslogDate;
+ }
+
public String getElasticSearchUrl() {
String ret = elasticsearchUrl;
View
2  src/main/java/org/graylog2/Main.java
@@ -63,7 +63,7 @@
public final class Main {
private static final Logger LOG = Logger.getLogger(Main.class);
- private static final String GRAYLOG2_VERSION = "0.9.6-beta";
+ private static final String GRAYLOG2_VERSION = "0.9.7-dev";
public static RulesEngine drools = null;
private static final int SCHEDULED_THREADS_POOL_SIZE = 7;
View
14 src/main/java/org/graylog2/messagehandlers/gelf/GELFMessage.java
@@ -229,6 +229,10 @@ public void setFacility(String facility) {
* @param value
*/
public void addAdditionalData(String key, Object value) {
+ if (!key.startsWith(GELF.USER_DEFINED_FIELD_PREFIX)) {
+ key = GELF.USER_DEFINED_FIELD_PREFIX + key;
+ }
+
if (key != null && value != null) {
if (value instanceof Long) {
@@ -246,6 +250,16 @@ public void addAdditionalData(String key, Object value) {
}
/**
+ * Add a whole set of additional fields.
+ * @param fields
+ */
+ public void addAdditionalData(Map<String, String> fields) {
+ for (Map.Entry<String, String> field : fields.entrySet()) {
+ addAdditionalData(field.getKey(), field.getValue());
+ }
+ }
+
+ /**
* Set the filterOut
*
* @param filterOut
View
70 src/main/java/org/graylog2/messagehandlers/syslog/StructuredSyslog.java
@@ -0,0 +1,70 @@
+/**
+ * Copyright 2011 Lennart Koopmann <lennart@socketfeed.com>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.graylog2.messagehandlers.syslog;
+
+import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.log4j.Logger;
+import org.productivity.java.syslog4j.server.impl.event.structured.StructuredSyslogServerEvent;
+
+/**
+ * StructuredSyslog.java: Dec 24, 2011 5:32:06 PM
+ *
+ * Parses structured syslog data.
+ *
+ * @author Lennart Koopmann <lennart@socketfeed.com>
+ */
+public class StructuredSyslog {
+
+ private static final Logger LOG = Logger.getLogger(StructuredSyslog.class);
+
+ public static Map<String, String> extractFields(byte[] rawSyslogMessage) {
+ Map<String, String> fields = new HashMap<String, String>();
+ try {
+ StructuredSyslogServerEvent s = new StructuredSyslogServerEvent(
+ rawSyslogMessage,
+ rawSyslogMessage.length,
+ InetAddress.getLocalHost()
+ );
+
+ Map raw = s.getStructuredMessage().getStructuredData();
+ if (raw != null) {
+ Set ks = raw.keySet();
+ if (ks.size() > 0) {
+ Object[] fl = raw.keySet().toArray();
+
+ if (fl != null && fl.length > 0) {
+ String sdID = (String) fl[0];
+ fields = (HashMap) raw.get(sdID);
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.debug("Could not extract structured syslog", e);
+ return new HashMap();
+ }
+
+ return fields;
+ }
+
+}
View
38 src/main/java/org/graylog2/messagehandlers/syslog/SyslogEventHandler.java
@@ -20,6 +20,8 @@
package org.graylog2.messagehandlers.syslog;
+import java.net.InetAddress;
+import java.util.logging.Level;
import org.apache.log4j.Logger;
import org.graylog2.Tools;
import org.graylog2.messagehandlers.gelf.GELFMessage;
@@ -30,7 +32,11 @@
import java.net.SocketAddress;
import java.net.UnknownHostException;
+import java.util.Date;
+import java.util.Map;
import org.graylog2.Main;
+import org.productivity.java.syslog4j.impl.message.structured.StructuredSyslogMessage;
+import org.productivity.java.syslog4j.server.impl.event.structured.StructuredSyslogServerEvent;
/**
* SyslogEventHandler.java: May 17, 2010 8:58:18 PM
@@ -70,10 +76,36 @@ public void event(SyslogServerIF syslogServer, SocketAddress socketAddress, Sysl
LOG.debug("Raw: " + new String(event.getRaw()));
LOG.debug("Host stripped from message? " + event.isHostStrippedFromMessage());
- // Manually check for provided date because it's necessary to parse the GELF message. Second check for completness later.
+ // Check if date could be parsed.
if (event.getDate() == null) {
- LOG.info("Syslog message is missing date or could not be parsed. Not further handling. Message was: " + event.getRaw());
- return;
+ if (Main.configuration.getAllowOverrideSyslogDate()) {
+ // empty Date constructor allocates a Date object and initializes it so that it represents the time at which it was allocated.
+ event.setDate(new Date());
+ LOG.info("Date could not be parsed. Was set to NOW because allow_override_syslog_date is true.");
+ } else {
+ LOG.info("Syslog message is missing date or date could not be parsed. (Possibly set allow_override_syslog_date to true) "
+ + "Not further handling. Message was: " + new String(event.getRaw()));
+ return;
+ }
+ }
+
+ // try to parse from event. if that fails or nothing is parsed, tokenize self.
+ // Parse possibly included structured syslog data into additional_fields.
+ Map<String, String> structuredData = StructuredSyslog.extractFields(event.getRaw());
+ if (structuredData.size() > 0) {
+ // We were able to parse structured data from the message. Add as additional fields.
+ LOG.debug("Parsed <" + structuredData.size() + "> structured data pairs."
+ + " Adding as additional_fields. Not using tokenizer.");
+ gelf.addAdditionalData(structuredData);
+ } else {
+ /*
+ * There was no structured data to be parsed or parsing failed.
+ *
+ * This means that we can safely extract values with the Tokenizer
+ * without interfering with structured data.
+ */
+ LOG.debug("No structured data was parsed from message. Using tokenizer.");
+ // XXX IMPLEMENT
}
// Possibly overwrite host with RNDS if configured.
View
47 src/main/java/org/graylog2/messagehandlers/syslog/Tokenizer.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright 2011 Lennart Koopmann <lennart@socketfeed.com>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.graylog2.messagehandlers.syslog;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.productivity.java.syslog4j.server.impl.event.SyslogServerEvent;
+
+/**
+ * Tokenizer.java: Dec 24, 2011 4:54:31 PM
+ *
+ * Breaks down syslog messages into additional_fields if they could not
+ * be parsed as structured syslog.
+ *
+ * @author Lennart Koopmann <lennart@socketfeed.com>
+ */
+public class Tokenizer {
+
+ char[] chseparators = { '=' };
+
+ public static Map extractAdditionalFields(SyslogServerEvent msg) {
+ Map extracted = new HashMap();
+
+ return extracted;
+ }
+
+ // No spaces between = and
+
+}
View
30 src/test/java/org/graylog2/messagehandlers/gelf/GELFMessageTest.java
@@ -20,6 +20,8 @@
package org.graylog2.messagehandlers.gelf;
+import java.util.Map;
+import java.lang.Object;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import org.bson.types.ObjectId;
@@ -27,6 +29,7 @@
import org.junit.Test;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.*;
@@ -137,7 +140,7 @@ public void testToOneliner() {
GELFMessage gelfMessage = createGELFMessage();
- String oneLiner = "host.example.com - short message severity=Emergency,facility=local0,file=test.file,line=42,test=test";
+ String oneLiner = "host.example.com - short message severity=Emergency,facility=local0,file=test.file,line=42,_test=test";
assertEquals(oneLiner, gelfMessage.toOneLiner());
}
@@ -207,4 +210,29 @@ public void testAllRequiredFieldsSet() {
GELFMessage gelfMessage = createGELFMessage();
assertTrue(gelfMessage.allRequiredFieldsSet());
}
+
+ @Test
+ public void testAddAdditionalData() {
+ GELFMessage msg = new GELFMessage();
+ msg.addAdditionalData("_foo", "bar");
+ msg.addAdditionalData("lol", "wat"); // _ should be added automatically.
+
+ Map<String, Object> expected = new HashMap<String, Object>();
+ expected.put("_foo", "bar");
+ expected.put(("_lol"), "wat");
+
+ assertEquals(expected, msg.getAdditionalData());
+ }
+
+ @Test
+ public void testAddAdditionalDataWithMap() {
+ Map<String, String> fields = new HashMap<String, String>();
+ fields.put("_foo", "bar");
+ fields.put("_lol", "wat");
+
+ GELFMessage msg = new GELFMessage();
+ msg.addAdditionalData(fields);
+
+ assertEquals(fields, msg.getAdditionalData());
+ }
}
View
58 src/test/java/org/graylog2/messagehandlers/syslog/StructuredSyslogTest.java
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2011 Lennart Koopmann <lennart@socketfeed.com>
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package org.graylog2.messagehandlers.syslog;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class StructuredSyslogTest {
+
+ // http://tools.ietf.org/rfc/rfc5424.txt
+ public static String ValidStructuredMessage = "<165>1 2012-12-25T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] BOMAn application event log entry";
+ public static String ValidNonStructuredMessage = "<86>Dec 24 17:05:01 nb-lkoopmann CRON[10049]: pam_unix(cron:session): session closed for user root";
+ public static String MessageLookingLikeStructured = "<133>NOMA101FW01A: NetScreen device_id=NOMA101FW01A [Root]system-notification-00257(traffic): start_time=\"2011-12-23 17:33:43\" duration=0 reason=Creation";
+
+ @Test
+ public void testExtractFields() {
+ Map expected = new HashMap();
+ expected.put("eventSource", "Application");
+ expected.put("eventID", "1011");
+ expected.put("iut", "3");
+
+ Map result = StructuredSyslog.extractFields(ValidStructuredMessage.getBytes());
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void testExtractFieldsOfNonStructuredMessage() {
+ Map result = StructuredSyslog.extractFields(ValidNonStructuredMessage.getBytes());
+ assertEquals(0, result.size());
+ }
+
+ @Test
+ public void testExtractFieldsOfAMessageThatOnlyLooksStructured() {
+ Map result = StructuredSyslog.extractFields(MessageLookingLikeStructured.getBytes());
+ assertEquals(0, result.size());
+ }
+
+}

No commit comments for this range

Something went wrong with that request. Please try again.