Merge pull request #1583 from sebres/_0.10/fix-datedetector-grave-fix-v2

 0.10/datedetector grave fix

ChangeLog

@@ -13,6 +13,15 @@ TODO: implementing of options resp. other tasks from PR #1346
 
 ### Fixes
 * [Grave] memory leak's fixed (gh-1277, gh-1234)
+* [Grave] Misleading date patterns defined more precisely (using extended syntax
+  `%Ex[mdHMS]` for exact two-digit match or e. g. `%ExY` as more precise year 
+  pattern, within same century of last year and the next 3 years)
+* [Grave] extends date detector template with distance (position of match in 
+  log-line), to prevent grave collision using (re)ordered template list (e.g.
+  find-spot of wrong date-match inside foreign input, misleading date patterns
+  by ambiguous formats, etc.)
+* Distance collision check always prefers template with shortest distance
+  (left for right) if date pattern is not anchored
 * Tricky bug fix: last position of log file will be never retrieved (gh-795),
   because of CASCADE all log entries will be deleted from logs table together with jail, 
   if used "INSERT OR REPLACE" statement
@@ -39,6 +48,11 @@ TODO: implementing of options resp. other tasks from PR #1346
   - if fail2ban running as systemd-service, for logging to the systemd-journal, 
     the `logtarget` could be set to STDOUT
   - value `logtarget` for system targets allowed also in lowercase (stdout, stderr, syslog, etc.)
+* Fixed UTC/GMT named time zone, using `%Z` and `%z` patterns 
+  (special case with 0 zone offset, see gh-1575)
+* `filter.d/freeswitch.conf`
+    - Optional prefixes (server, daemon, dual time) if systemd daemon logs used (gh-1548)
+    - User part rewritten to accept IPv6 resp. domain after "@" (gh-1548)
 
 ### New Features
 * IPv6 support:
@@ -135,6 +149,22 @@ fail2ban-client set loglevel INFO
   nevertheless, as long as one jail was successful configured (gh-1619)
   Message about wrong jail configuration logged in client log (stdout, systemd
   journal etc.) and in server log with error level
+* More precise date template handling (WARNING: theoretically possible incompatibilities):
+  - datedetector rewritten more strict as earlier;
+  - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`);
+  - more as one date pattern can be specified using option `datepattern` now 
+    (new-line separated);
+  - some default options like `datepattern` can be specified directly in 
+    section `[Definition]`, that avoids contrary usage of unnecessarily `[Init]`
+    section, because of performance (each extra section costs time);
+  - option `datepattern` can be specified in jail also (e. g. jails without filters 
+    or custom log-format, new-line separated for multiple patterns);
+  - if first unnamed group specified in pattern, only this will be cut out from
+    search log-line (e. g.: `^date:[({DATE})]` will cut out only datetime match 
+    pattern, and leaves `date:[] ...` for searching in filter);
+  - faster match and fewer searching of appropriate templates
+    (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
+  - several standard filters extended with exact prefixed or anchored date templates;
 * fail2ban-testcases:
   - `assertLogged` extended with parameter wait (to wait up to specified timeout,
     before we throw assert exception) + test cases rewritten using that

config/filter.d/3proxy.conf

@@ -9,6 +9,8 @@ failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ <HOST>:\d+ [\d.]+:\d+ \d+ \d+ \d+\s
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 # http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are
 # all authentication problems (%E field)

config/filter.d/apache-badbots.conf

@@ -14,6 +14,9 @@ failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
+
 # DEV Notes:
 # List of bad bots fetched from http://www.user-agents.org
 # Generated on Thu Nov  7 14:23:35 PST 2013 by files/gen_badbots.

config/filter.d/apache-common.conf

@@ -10,6 +10,8 @@ after = apache-common.local
 
 _apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
 
+datepattern = {^LN-BEG}
+
 # Common prefix for [error] apache messages which also would include <HOST>
 # Depending on the version it could be
 # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]

config/filter.d/apache-fakegooglebot.conf

@@ -6,6 +6,8 @@ failregex = ^<HOST> .*Googlebot.*$
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # DEV Notes:
 #

config/filter.d/apache-pass.conf

@@ -3,16 +3,15 @@
 #
 # The knocking request must have a referer.
 
-[INCLUDES]
-
-before = apache-common.conf
-
 [Definition]
 
 failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
+
 [Init]
 
 knocking_url = /knocking/

config/filter.d/assp.conf

@@ -20,6 +20,9 @@ failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}%%b-%%d-%%Exy %%H:%%M:%%S
+              {^LN-BEG}
+
 # DEV Notes:
 # V1 Examples matches:
 #   Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);

config/filter.d/asterisk.conf

@@ -31,6 +31,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
 
 ignoreregex =
 
+datepattern = {^LN-BEG}
 
 # Author: Xavier Devlamynck / Daniel Black
 #

config/filter.d/common.conf

@@ -61,4 +61,7 @@ __prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostn
 # pam_ldap
 __pam_auth = pam_unix
 
+# standardly all formats using prefix have line-begin anchored date:
+datepattern = {^LN-BEG}
+
 # Author: Yaroslav Halchenko

config/filter.d/counter-strike.conf

@@ -8,8 +8,6 @@ failregex = ^: Bad Rcon: "rcon \d+ "\S+"  sv_contact ".*?"" from "<HOST>:\d+"$
 
 ignoreregex =
 
-[Init]
-
 datepattern = ^L %%d/%%m/%%Y - %%H:%%M:%%S
 
 

config/filter.d/courier-auth.conf

@@ -15,5 +15,7 @@ failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[<HOST>\]$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # Author: Christoph Haas
 # Modified by: Cyril Jaquier

config/filter.d/directadmin.conf

@@ -13,7 +13,6 @@ failregex = ^: \'<HOST>\' \d{1,3} failed login attempt(s)?. \s*
 
 ignoreregex = 
 
-[Init]
 datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S
 
 #

config/filter.d/dovecot.conf

@@ -17,10 +17,11 @@ failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentica
 
 ignoreregex = 
 
-[Init]
-
 journalmatch = _SYSTEMD_UNIT=dovecot.service
 
+datepattern = {^LN-BEG}TAI64N
+              {^LN-BEG}
+
 # DEV Notes:
 # * the first regex is essentially a copy of pam-generic.conf
 # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016)

config/filter.d/ejabberd-auth.conf

@@ -25,8 +25,6 @@ failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\
 #
 ignoreregex = 
 
-[Init]
-
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
 
@@ -35,3 +33,8 @@ maxlines = 2
 # Values:  TEXT
 #
 journalmatch = 
+
+#datepattern = ^(?:=[^=]+={3,} )?({DATE})
+# explicit time format using prefix =...==== and no date in second string begins with I(...)... 
+datepattern = ^(?:=[^=]+={3,} )?(%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?)
+              ^I\(()**

config/filter.d/freeswitch.conf

@@ -8,13 +8,26 @@
 # IP addresses on your LAN.
 #
 
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
 [Definition]
 
-failregex = ^\.\d+ \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[.*\] from ip <HOST>$
-            ^\.\d+ \[WARNING\] sofia_reg\.c:\d+ Can't find user \[\d+@\d+\.\d+\.\d+\.\d+\] from <HOST>$
+_daemon = freeswitch
+
+# Prefix contains common prefix line (server, daemon, etc.) and 2 datetimes if used systemd backend
+_pref_line = ^%(__prefix_line)s(?:\d+-\d+-\d+ \d+:\d+:\d+\.\d+)?
+
+failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[[^\]]*\] from ip <HOST>$
+            %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ Can't find user \[[^@]+@[^\]]+\] from <HOST>$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}
+
 # Author: Rupa SChomaker, soapee01, Daniel Black
 # https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban
 # Thanks to Jim on mailing list of samples and guidance

config/filter.d/guacamole.conf

@@ -17,6 +17,9 @@ failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" fa
 #
 ignoreregex = 
 
-[Init]
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
+
+datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p
+              ^WARNING:()**
+              {^LN-BEG}

config/filter.d/kerio.conf

@@ -9,8 +9,6 @@ failregex = ^ SMTP Spam attack detected from <HOST>,
 
 ignoreregex =
 
-[Init]
-
 datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
 
 # DEV NOTES:

config/filter.d/monit.conf

@@ -13,7 +13,7 @@ before = common.conf
 _daemon = monit
 
 # Regexp for previous (accessing monit httpd) and new (access denied) versions
-failregex = ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
+failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
             ^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$
 
 # Ignore login with empty user (first connect, no user specified)

config/filter.d/murmur.conf

@@ -15,13 +15,14 @@ _daemon = murmurd
 #      variable in your server config file (murmur.ini / mumble-server.ini).
 _usernameregex = [^>]+
 
-_prefix = <W>[\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
+_prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
 
 failregex = ^%(_prefix)s Invalid server password$
             ^%(_prefix)s Wrong certificate or password for existing user$
 
 ignoreregex =
 
+datepattern = ^<W>{DATE}
 
 # DEV Notes:
 #

config/filter.d/nginx-botsearch.conf

@@ -13,6 +13,9 @@ failregex = ^<HOST> \- \S+ \[\] \"(GET|POST|HEAD) \/<block> \S+\" 404 .+$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # DEV Notes:
 # Based on apache-botsearch filter

config/filter.d/nginx-http-auth.conf

@@ -8,6 +8,8 @@ failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not f
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV NOTES:
 # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files
 # Extensive search of all nginx auth failures not done yet.

config/filter.d/nginx-limit-req.conf

@@ -43,3 +43,4 @@ failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by z
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}

config/filter.d/nsd.conf

@@ -26,3 +26,6 @@ failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
              ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
 
 ignoreregex =
+
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}

config/filter.d/openhab.conf

@@ -9,7 +9,6 @@
 [Definition] 
 failregex = ^<HOST>\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$
 
-[Init]
 datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z
 
 

config/filter.d/oracleims.conf

@@ -52,10 +52,12 @@ before = common.conf
 # Note that you MUST have LOG_FORMAT=4 for this to work!
 #
 
-failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
+failregex = tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
 # Values:  TEXT
 #
 ignoreregex =
+
+datepattern = ^<co ts="{DATE}"\s+

config/filter.d/php-url-fopen.conf

@@ -18,3 +18,6 @@ ignoreregex =
 # http://blogs.buanzo.com.ar/2009/04/fail2ban-filter-for-php-injection-attacks.html#comment-1489
 #
 # Author: Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
+
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}

config/filter.d/portsentry.conf

@@ -8,5 +8,8 @@ failregex = \/<HOST> Port\: [0-9]+ (TCP|UDP) Blocked$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
+
 # Author: Pacop <pacoparu@gmail.com>
 

config/filter.d/selinux-common.conf

@@ -18,4 +18,6 @@ failregex = ^type=%(_type)s msg=audit\(:\d+\): (user )?pid=\d+ uid=%(_uid)s auid
 
 ignoreregex =
 
+datepattern = EPOCH
+
 # Author: Daniel Black

config/filter.d/sogo-auth.conf

@@ -6,7 +6,12 @@
 
 failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d*  grace: -?\d*  expire: -?\d*  bound: -?\d*)?\s*$
 
-ignoreregex = 
+ignoreregex = "^<ADDR>"
+
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)?
+              ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # 
 # DEV Notes:

config/filter.d/squid.conf

@@ -9,5 +9,8 @@ failregex = ^\s+\d\s<HOST>\s+[A-Z_]+_DENIED/403 .*$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
+
 # Author: Daniel Black
 

config/filter.d/squirrelmail.conf

@@ -5,8 +5,6 @@ failregex = ^ \[LOGIN_ERROR\].*from <HOST>: Unknown user or password incorrect\.
 
 ignoreregex =
 
-[Init]
-
 datepattern = ^%%m/%%d/%%Y %%H:%%M:%%S
 
 # DEV NOTES:

config/filter.d/sshd.conf

@@ -38,13 +38,13 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro
 
 ignoreregex = 
 
-[Init]
-
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 10
 
 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 #
 #   "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because