Skip to content

Commit

Permalink
fix grok tester resource
Browse files Browse the repository at this point in the history 
remove obsolete pattern string

#377
  • Loading branch information
@kroepke
kroepke committed Jan 8, 2015
1 parent 4351507 commit f273e1a
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 97 deletions.
94 changes: 0 additions & 94 deletions graylog2-server/src/main/java/org/graylog2/inputs/extractors/GrokExtractor.java
View file
Expand Up @@ -35,100 +35,6 @@

public class GrokExtractor extends Extractor {
private static final Logger log = LoggerFactory.getLogger(GrokExtractor.class);
public static final String PATTERNS = "USERNAME [a-zA-Z0-9._-]+\n" +
"USER %{USERNAME}\n" +
"INT (?:[+-]?(?:[0-9]+))\n" +
"BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))\n" +
"NUMBER (?:%{BASE10NUM})\n" +
"BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))\n" +
"BASE16FLOAT \\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\\.[0-9A-Fa-f]*)?)|(?:\\.[0-9A-Fa-f]+)))\\b\n" +
"\n" +
"POSINT \\b(?:[1-9][0-9]*)\\b\n" +
"NONNEGINT \\b(?:[0-9]+)\\b\n" +
"WORD \\b\\w+\\b\n" +
"NOTSPACE \\S+\n" +
"SPACE \\s*\n" +
"DATA .*?\n" +
"GREEDYDATA .*\n" +
"QUOTEDSTRING (?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))\n" +
"UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}\n" +
"\n" +
"# Networking\n" +
"MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})\n" +
"CISCOMAC (?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})\n" +
"WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})\n" +
"COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})\n" +
"IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\n" +
"IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])\n" +
"IP (?:%{IPV6}|%{IPV4})\n" +
"HOSTNAME \\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)\n" +
"HOST %{HOSTNAME}\n" +
"IPORHOST (?:%{HOSTNAME}|%{IP})\n" +
"HOSTPORT %{IPORHOST}:%{POSINT}\n" +
"\n" +
"# paths\n" +
"PATH (?:%{UNIXPATH}|%{WINPATH})\n" +
"UNIXPATH (?>/(?>[\\w_%!$@:.,~-]+|\\\\.)*)+\n" +
"TTY (?:/dev/(pts|tty([pq])?)(\\w+)?/?(?:[0-9]+))\n" +
"WINPATH (?>[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+\n" +
"URIPROTO [A-Za-z]+(\\+[A-Za-z+]+)?\n" +
"URIHOST %{IPORHOST}(?::%{POSINT:port})?\n" +
"# uripath comes loosely from RFC1738, but mostly from what Firefox\n" +
"# doesn't turn into %XX\n" +
"URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\\-]*)+\n" +
"#URIPARAM \\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?\n" +
"URIPARAM \\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\\-\\[\\]]*\n" +
"URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?\n" +
"URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?\n" +
"\n" +
"# Months: January, Feb, 3, 03, 12, December\n" +
"MONTH \\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b\n" +
"MONTHNUM (?:0?[1-9]|1[0-2])\n" +
"MONTHNUM2 (?:0[1-9]|1[0-2])\n" +
"MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])\n" +
"\n" +
"# Days: Monday, Tue, Thu, etc...\n" +
"DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)\n" +
"\n" +
"# Years?\n" +
"YEAR (?>\\d\\d){1,2}\n" +
"HOUR (?:2[0123]|[01]?[0-9])\n" +
"MINUTE (?:[0-5][0-9])\n" +
"# '60' is a leap second in most time standards and thus is valid.\n" +
"SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)\n" +
"TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])\n" +
"# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)\n" +
"DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}\n" +
"DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}\n" +
"ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))\n" +
"ISO8601_SECOND (?:%{SECOND}|60)\n" +
"TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?\n" +
"DATE %{DATE_US}|%{DATE_EU}\n" +
"DATESTAMP %{DATE}[- ]%{TIME}\n" +
"TZ (?:[PMCE][SD]T|UTC)\n" +
"DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}\n" +
"DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}\n" +
"DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}\n" +
"DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}\n" +
"\n" +
"# Syslog Dates: Month Day HH:MM:SS\n" +
"SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}\n" +
"PROG (?:[\\w._/%-]+)\n" +
"SYSLOGPROG %{PROG:program}(?:\\[%{POSINT:pid}\\])?\n" +
"SYSLOGHOST %{IPORHOST}\n" +
"SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>\n" +
"HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}\n" +
"\n" +
"# Shortcuts\n" +
"QS %{QUOTEDSTRING}\n" +
"\n" +
"# Log formats\n" +
"SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:\n" +
"COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)\n" +
"COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}\n" +
"\n" +
"# Log Levels\n" +
"LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)";

private final Grok grok = new Grok();

Expand Down
20 changes: 17 additions & 3 deletions graylog2-server/src/main/java/org/graylog2/rest/resources/tools/GrokTesterResource.java
View file
Expand Up @@ -22,33 +22,47 @@
import oi.thekraken.grok.api.Match;
import oi.thekraken.grok.api.exception.GrokException;
import org.apache.shiro.authz.annotation.RequiresAuthentication;
import org.graylog2.inputs.extractors.GrokExtractor;
import org.graylog2.grok.GrokPattern;
import org.graylog2.grok.GrokPatternService;
import org.graylog2.rest.resources.tools.responses.GrokTesterResponse;
import org.graylog2.shared.rest.resources.RestResource;
import org.hibernate.validator.constraints.NotEmpty;

import javax.inject.Inject;
import javax.validation.constraints.NotNull;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import java.io.StringReader;
import java.util.List;
import java.util.Map;
import java.util.Set;

@RequiresAuthentication
@Path("/tools/grok_tester")
@Produces(MediaType.APPLICATION_JSON)
public class GrokTesterResource extends RestResource {

private final GrokPatternService grokPatternService;

@Inject
public GrokTesterResource(GrokPatternService grokPatternService) {
this.grokPatternService = grokPatternService;
}

@GET
@Timed
public Object grokTest(@QueryParam("pattern") @NotEmpty String pattern,
@QueryParam("string") @NotNull String string) throws GrokException {

final Set<GrokPattern> grokPatterns = grokPatternService.loadAll();

final Grok grok = new Grok();
grok.addPatternFromReader(new StringReader(GrokExtractor.PATTERNS));
for (GrokPattern grokPattern : grokPatterns) {
grok.addPattern(grokPattern.name, grokPattern.pattern);
}

grok.compile(pattern);
final Match match = grok.match(string);
match.captures();
Expand Down

0 comments on commit f273e1a

Please sign in to comment.