grok pattern import failure

[jschlyter]

Problem description

Graylog failes to parse some grokpatterns when reading from file.

Steps to reproduce the problem

Try to import haproxy-test.txt
via /system/grokpatterns "Import Pattern File". Pattern HAPROXYHTTP and HAPROXYTCP are truncated.

Adding each pattern manually works.

Environment

• Graylog Version: 2.0.1
• Elasticsearch Version: 2.3.2
• MongoDB Version: 2.6.9
• Operating System: CentOS 7.2
• Browser version: Chrome 50.0.2661.94

[Nayar]

Happened to me too. I guess this causes it

\[

[jalogisch]

if you try to import https://github.com/whyscream/postfix-grok-patterns/blob/master/postfix.grok this also not working.

all pattern manual do work without a problem!

[joschi]

The problem seems to be that the web interface is cutting off the patterns on upload (captured JSON request body, see POSTFIX_SMTP_STAGE for example):

{
   "patterns" : [
      {
         "name" : "POSTFIX_QUEUEID",
         "pattern" : "([0-9A-F]{6,}|[0-9a-zA-Z]{15,})"
      },
      {
         "name" : "POSTFIX_CLIENT_INFO",
         "pattern" : "%{HOSTNAME:postfix_client_hostname}?\\[%{IP:postfix_client_ip}\\](:%{INT:postfix_client_port})?"
      },
      {
         "name" : "POSTFIX_RELAY_INFO",
         "pattern" : "%{HOSTNAME:postfix_relay_hostname}?\\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service}"
      },
      {
         "pattern" : "(CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL(",
         "name" : "POSTFIX_SMTP_STAGE"
      },
      {
         "pattern" : "(accept|defer|discard|filter|header-redirect|reject)",
         "name" : "POSTFIX_ACTION"
      },
      {
         "pattern" : "\\d{3}",
         "name" : "POSTFIX_STATUS_CODE"
      },
      {
         "name" : "POSTFIX_STATUS_CODE_ENHANCED",
         "pattern" : "\\d\\.\\d\\.\\d"
      },
      {
         "pattern" : "Service",
         "name" : "POSTFIX_DNSBL_MESSAGE"
      },
      {
         "name" : "POSTFIX_PS_ACCESS_ACTION",
         "pattern" : "(DISCONNECT|BLACKLISTED|WHITELISTED|WHITELIST"
      },
      {
         "name" : "POSTFIX_PS_VIOLATION",
         "pattern" : "(BARE"
      },
      {
         "pattern" : "%{NUMBER}[smhd]",
         "name" : "POSTFIX_TIME_UNIT"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_KEYVALUE"
      },
      {
         "pattern" : "(%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_WARNING"
      },
      {
         "name" : "POSTFIX_TLSCONN",
         "pattern" : "(Anonymous|Trusted|Untrusted|Verified)"
      },
      {
         "pattern" : "%{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission}",
         "name" : "POSTFIX_DELAYS"
      },
      {
         "name" : "POSTFIX_LOSTCONN",
         "pattern" : "(lost"
      },
      {
         "name" : "POSTFIX_PROXY_MESSAGE",
         "pattern" : "(%{POSTFIX_STATUS_CODE:postfix_proxy_status_code}"
      },
      {
         "pattern" : "[^:]*",
         "name" : "GREEDYDATA_NO_COLON"
      },
      {
         "pattern" : "[^;]*",
         "name" : "GREEDYDATA_NO_SEMICOLON"
      },
      {
         "pattern" : "connect",
         "name" : "POSTFIX_SMTPD_CONNECT"
      },
      {
         "name" : "POSTFIX_SMTPD_DISCONNECT",
         "pattern" : "disconnect"
      },
      {
         "name" : "POSTFIX_SMTPD_LOSTCONN",
         "pattern" : "%{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}("
      },
      {
         "name" : "POSTFIX_SMTPD_NOQUEUE",
         "pattern" : "NOQUEUE:"
      },
      {
         "pattern" : "improper",
         "name" : "POSTFIX_SMTPD_PIPELINING"
      },
      {
         "name" : "POSTFIX_SMTPD_PROXY",
         "pattern" : "proxy-%{POSTFIX_ACTION:postfix_proxy_result}:"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_CLEANUP_MILTER"
      },
      {
         "name" : "POSTFIX_QMGR_REMOVED",
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:"
      },
      {
         "name" : "POSTFIX_QMGR_ACTIVE",
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_QMGR_EXPIRED"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_PIPE_ANY"
      },
      {
         "name" : "POSTFIX_PS_CONNECT",
         "pattern" : "CONNECT"
      },
      {
         "name" : "POSTFIX_PS_ACCESS",
         "pattern" : "%{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access}"
      },
      {
         "pattern" : "%{POSTFIX_SMTPD_NOQUEUE}",
         "name" : "POSTFIX_PS_NOQUEUE"
      },
      {
         "pattern" : "NOQUEUE:",
         "name" : "POSTFIX_PS_TOOBUSY"
      },
      {
         "name" : "POSTFIX_PS_DNSBL",
         "pattern" : "%{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}"
      },
      {
         "pattern" : "cache",
         "name" : "POSTFIX_PS_CACHE"
      },
      {
         "pattern" : "%{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}(",
         "name" : "POSTFIX_PS_VIOLATIONS"
      },
      {
         "name" : "POSTFIX_DNSBLOG_LISTING",
         "pattern" : "addr"
      },
      {
         "pattern" : "(DIS)?CONNECT(",
         "name" : "POSTFIX_TLSPROXY_CONN"
      },
      {
         "name" : "POSTFIX_ANVIL_CONN_RATE",
         "pattern" : "statistics:"
      },
      {
         "pattern" : "statistics:",
         "name" : "POSTFIX_ANVIL_CONN_CACHE"
      },
      {
         "name" : "POSTFIX_ANVIL_CONN_COUNT",
         "pattern" : "statistics:"
      },
      {
         "name" : "POSTFIX_SMTP_DELIVERY",
         "pattern" : "%{POSTFIX_KEYVALUE}"
      },
      {
         "pattern" : "connect",
         "name" : "POSTFIX_SMTP_CONNERR"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_SMTP_LOSTCONN"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_SMTP_TIMEOUT"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_SMTP_RELAYERR"
      },
      {
         "pattern" : "(daemon",
         "name" : "POSTFIX_MASTER_START"
      },
      {
         "pattern" : "terminating",
         "name" : "POSTFIX_MASTER_EXIT"
      },
      {
         "pattern" : "%{POSTFIX_QUEUEID:postfix_queueid}:",
         "name" : "POSTFIX_BOUNCE_NOTIFICATION"
      },
      {
         "pattern" : "statistics:",
         "name" : "POSTFIX_SCACHE_LOOKUPS"
      },
      {
         "name" : "POSTFIX_SCACHE_SIMULTANEOUS",
         "pattern" : "statistics:"
      },
      {
         "pattern" : "statistics:",
         "name" : "POSTFIX_SCACHE_TIMESTAMP"
      },
      {
         "name" : "POSTFIX_SMTPD",
         "pattern" : "%{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE}"
      },
      {
         "pattern" : "%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}",
         "name" : "POSTFIX_CLEANUP"
      },
      {
         "name" : "POSTFIX_QMGR",
         "pattern" : "%{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}|%{POSTFIX_WARNING}"
      },
      {
         "name" : "POSTFIX_PIPE",
         "pattern" : "%{POSTFIX_PIPE_ANY}"
      },
      {
         "pattern" : "%{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}",
         "name" : "POSTFIX_POSTSCREEN"
      },
      {
         "pattern" : "%{POSTFIX_DNSBLOG_LISTING}|%{POSTFIX_WARNING}",
         "name" : "POSTFIX_DNSBLOG"
      },
      {
         "pattern" : "%{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT}",
         "name" : "POSTFIX_ANVIL"
      },
      {
         "name" : "POSTFIX_SMTP",
         "pattern" : "%{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}"
      },
      {
         "name" : "POSTFIX_DISCARD",
         "pattern" : "%{POSTFIX_KEYVALUE}"
      },
      {
         "name" : "POSTFIX_LMTP",
         "pattern" : "%{POSTFIX_SMTP}"
      },
      {
         "name" : "POSTFIX_PICKUP",
         "pattern" : "%{POSTFIX_KEYVALUE}"
      },
      {
         "name" : "POSTFIX_TLSPROXY",
         "pattern" : "%{POSTFIX_TLSPROXY_CONN}|%{POSTFIX_WARNING}"
      },
      {
         "name" : "POSTFIX_MASTER",
         "pattern" : "%{POSTFIX_MASTER_START}|%{POSTFIX_MASTER_EXIT}|%{POSTFIX_WARNING}"
      },
      {
         "name" : "POSTFIX_BOUNCE",
         "pattern" : "%{POSTFIX_BOUNCE_NOTIFICATION}"
      },
      {
         "pattern" : "%{POSTFIX_WARNING}",
         "name" : "POSTFIX_SENDMAIL"
      },
      {
         "pattern" : "%{POSTFIX_WARNING}",
         "name" : "POSTFIX_POSTDROP"
      },
      {
         "pattern" : "%{POSTFIX_SCACHE_LOOKUPS}|%{POSTFIX_SCACHE_SIMULTANEOUS}|%{POSTFIX_SCACHE_TIMESTAMP}",
         "name" : "POSTFIX_SCACHE"
      },
      {
         "name" : "POSTFIX_TRIVIAL_REWRITE",
         "pattern" : "%{POSTFIX_WARNING}"
      },
      {
         "pattern" : "%{POSTFIX_WARNING}",
         "name" : "POSTFIX_TLSMGR"
      },
      {
         "name" : "POSTFIX_LOCAL",
         "pattern" : "%{POSTFIX_KEYVALUE}"
      },
      {
         "name" : "POSTFIX_VIRTUAL",
         "pattern" : "%{POSTFIX_SMTP_DELIVERY}"
      }
   ]
}