Merge remote-tracking branch 0.10 into _0.10/fix-datedetector-grave-f…

…ix-v2
fail2ban · Nov 28, 2016 · 40cbe96 · 40cbe96
2 parents a2cf34a + 389ad10
commit 40cbe96
Show file tree

Hide file tree

Showing 40 changed files with 475 additions and 142 deletions.
diff --git a/ChangeLog b/ChangeLog
@@ -37,6 +37,17 @@ TODO: implementing of options resp. other tasks from PR #1346
 * Pyinotify-backend: stability fix for sporadically errors in multi-threaded
   environment (without lock)
 * Fixed sporadically error in testCymruInfoNxdomain, because of unsorted values
+* Misleading errors logged from ignorecommand in success case on retcode 1 (gh-1194)
+* fail2ban.service - systemd service updated (gh-1618):
+  - starting service in normal mode (without forking)
+  - does not restart if service exited normally (exit-code 0, e.g. stopped via fail2ban-client)
+  - does not restart if service can not start (exit-code 255, e.g. wrong configuration, etc.)
+  - service can be additionally started/stopped with commands (fail2ban-client, fail2ban-server)
+  - automatically creates `/var/run/fail2ban` directory before start fail2ban 
+    (systems with virtual resp. memory-based FS for `/var/run`), see gh-1531
+  - if fail2ban running as systemd-service, for logging to the systemd-journal, 
+    the `logtarget` could be set to STDOUT
+  - value `logtarget` for system targets allowed also in lowercase (stdout, stderr, syslog, etc.)
 * Fixed UTC/GMT named time zone, using `%Z` and `%z` patterns 
   (special case with 0 zone offset, see gh-1575)
 * `filter.d/freeswitch.conf`
@@ -68,6 +79,8 @@ TODO: implementing of options resp. other tasks from PR #1346
     banned in this jail, if option `--unban` specified
   - `unban --all` - unbans all IP addresses (in all jails and database)
   - `unban <IP> ... <IP>` - unbans \<IP\> (in all jails and database) (see gh-1388)
+  - introduced new option `-t` or `--test` to test configuration resp. start server only 
+    if configuration is clean (fails by wrong configured jails if option `-t` specified)
 * New command action parameter `actionrepair` - command executed in order to restore
   sane environment in error case of `actioncheck`.
 
@@ -132,6 +145,10 @@ fail2ban-client set loglevel INFO
   - new replacement for `<ADDR>` in opposition to `<HOST>`, for separate
     usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6`
     together, without host (dns)
+* Misconfigured jails don't prevent fail2ban from starting, server starts 
+  nevertheless, as long as one jail was successful configured (gh-1619)
+  Message about wrong jail configuration logged in client log (stdout, systemd
+  journal etc.) and in server log with error level
 * More precise date template handling (WARNING: theoretically possible incompatibilities):
   - datedetector rewritten more strict as earlier;
   - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`);
@@ -185,6 +202,12 @@ releases.
     - Allow for having no trailing space after 'failed:' (gh-1497)
 * `filter.d/vsftpd.conf`
     - Optional reason part in message after FAIL LOGIN (gh-1543)
+* `filter.d/sendmail-reject.conf`
+    - removed mandatory double space (if dns-host available, gh-1579)
+* filter.d/sshd.conf
+    - recognized "Failed publickey for" (gh-1477);
+    - optimized failregex to match all of "Failed any-method for ... from <HOST>" (gh-1479)
+    - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479)
 
 
 ### New Features

diff --git a/FILTERS b/FILTERS
@@ -227,7 +227,7 @@ Regular expressions (failregex, ignoreregex) assume that the date/time has been
 removed from the log line (this is just how fail2ban works internally ATM).
 
 If the format is like '<date...> error 1.2.3.4 is evil' then you need to match
-the < at the start so regex should be similar to '^<> <HOST> is evil$' using
+the <> at the start so regex should be similar to '^<> error <HOST> is evil$' using
 <HOST> where the IP/domain name appears in the log line.
 
 The following general rules apply to regular expressions:

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
@@ -28,6 +28,10 @@
 #
 
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -54,10 +58,16 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
-	    IP=<ip>
+actionban = oifs=${IFS}; 
+              IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
+              IFS=,; ADDRESSES=$(echo $ADDRESSES)
+            IFS=${oifs}
+            IP=<ip>
             if [ ! -z "$ADDRESSES" ]; then
-                (printf %%b "<message>\n"; date '+Note: Local timezone is %%z (%%Z)'; grep -E '(^|[^0-9])<ip>([^0-9]|$)' <logpath>) | <mailcmd> "Abuse from <ip>" <mailargs> ${ADDRESSES//,/\" \"}
+                ( printf %%b "<message>\n"; date '+Note: Local timezone is %%z (%%Z)'; 
+                  printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n";
+                  %(_grep_logs)s;
+                ) | <mailcmd> "Abuse from <ip>" <mailargs> $ADDRESSES
             fi
 
 # Option:  actionunban
@@ -92,3 +102,7 @@ mailcmd = mail -s
 #
 mailargs =
 
+# Number of log lines to include in the email
+#
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
@@ -0,0 +1,13 @@
+[DEFAULT]
+
+# Usage:
+#   _grep_logs_args = 'test'
+#   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
+#
+_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
+_grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
+
+[Init]
+greplimit = tail -n <grepmax>
+grepmax = 1000
+grepopts = -m <grepmax>
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -17,7 +18,7 @@ before = mail-whois-common.conf
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -26,7 +27,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -40,15 +41,18 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+
+_ban_mail_content = ( printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
-            Here is more information about <ip> :\n
-            `%(_whois_command)s`\n\n
-            Lines containing IP:<ip> in <logpath>\n
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            Here is more information about <ip> :\n"
+            %(_whois_command)s;
+            printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n";
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+            Fail2Ban" )
+actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -60,6 +64,12 @@ actionunban =
 
 [Init]
 
+# Option:  mailcmd
+# Notes.:  Your system mail command. Is passed 2 args: subject and recipient
+# Values:  CMD
+#
+mailcmd = mail -s
+
 # Default name of the chain
 #
 name = default
@@ -74,4 +84,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -19,7 +20,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
@@ -33,10 +34,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "<ip>" | cut -d':' -f2-`
             AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "<ip>" | cut -d':' -f2-`
             hostname: `host -t A <ip> 2>&1`\n\n
-            Lines containing IP:<ip> in <logpath>\n
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            Lines containing failures of <ip>\n";
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+            Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest>
 
 [Init]
 
@@ -50,4 +52,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
@@ -25,10 +26,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
             `/usr/bin/whois <ip> || echo missing whois program`\n\n
-            Lines containing IP:<ip> in <logpath>\n
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            Lines containing failures of <ip>\n";
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+            Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest>
 
 [Init]
 
@@ -42,4 +44,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>
diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf
@@ -23,7 +23,7 @@ _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 
 failregex = ^%(__prefix_line)s\w{14}: ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[<HOST>\]( \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
             ^%(__prefix_line)sruleset=check_relay, arg1=(?P<dom>\S+), arg2=<HOST>, relay=((?P=dom) )?\[(\d+\.){3}\d+\]( \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^%(__prefix_line)s\w{14}: rejecting commands from  (\S+ )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
+            ^%(__prefix_line)s\w{14}: rejecting commands from (\S* )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
             ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]: ((?i)expn|vrfy) \S+ \[rejected\]$
             ^(?P<__prefix>%(__prefix_line)s\w+: )<[^@]+@[^>]+>\.\.\. No such user here<SKIPLINES>(?P=__prefix)from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[<HOST>\]$
 

diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf
@@ -20,7 +20,7 @@ _daemon = sshd
 
 failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*$
             ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from <HOST>\s*$
-            ^%(__prefix_line)sFailed \S+ for .*? from <HOST>(?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$
+            ^%(__prefix_line)sFailed \S+ for (?P<cond_inv>invalid user )?(?P<user>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from <HOST>(?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$)
             ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM <HOST>\s*$
             ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from <HOST>\s*$
             ^%(__prefix_line)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*$

diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py
@@ -29,7 +29,7 @@
 import sys
 from ..helpers import getLogger
 
-if sys.version_info >= (3,2): # pragma: no cover
+if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, \

diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
@@ -28,13 +28,25 @@
 import os
 from ConfigParser import NoOptionError, NoSectionError
 
-from .configparserinc import SafeConfigParserWithIncludes, logLevel
+from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
 
+# if sys.version_info >= (3,5):
+# 	def _merge_dicts(x, y):
+# 		return {**x, **y}
+# else:
+def _merge_dicts(x, y):
+	r = x
+	if y:
+		r = x.copy()
+		r.update(y)
+	return r
+
+
 class ConfigReader():
 	"""Generic config reader class.
 
@@ -127,9 +139,9 @@ def options(self, *args):
 			return self._cfg.options(*args)
 		return {}
 
-	def get(self, sec, opt):
+	def get(self, sec, opt, raw=False, vars={}):
 		if self._cfg is not None:
-			return self._cfg.get(sec, opt)
+			return self._cfg.get(sec, opt, raw=raw, vars=vars)
 		return None
 
 	def getOptions(self, *args, **kwargs):
@@ -210,6 +222,8 @@ def read(self, filename):
 
 	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
 		values = dict()
+		if pOptions is None:
+			pOptions = {}
 		for optname in options:
 			if isinstance(options, (list,tuple)):
 				if len(optname) > 2:
@@ -218,15 +232,15 @@ def getOptions(self, sec, options, pOptions=None, shouldExist=False):
 					(opttype, optname), optvalue = optname, None
 			else:
 				opttype, optvalue = options[optname]
+			if optname in pOptions:
+				continue
 			try:
 				if opttype == "bool":
 					v = self.getboolean(sec, optname)
 				elif opttype == "int":
 					v = self.getint(sec, optname)
 				else:
-					v = self.get(sec, optname)
-				if not pOptions is None and optname in pOptions:
-					continue
+					v = self.get(sec, optname, vars=pOptions)
 				values[optname] = v
 			except NoSectionError as e:
 				if shouldExist:
@@ -289,6 +303,12 @@ def readexplicit(self):
 		return SafeConfigParserWithIncludes.read(self._cfg, self._file)
 
 	def getOptions(self, pOpts):
+		# overwrite static definition options with init values, supplied as
+		# direct parameters from jail-config via action[xtra1="...", xtra2=...]:
+		if self._initOpts:
+			if not pOpts:
+				pOpts = dict()
+			pOpts = _merge_dicts(pOpts, self._initOpts)
 		self._opts = ConfigReader.getOptions(
 			self, "Definition", self._configOpts, pOpts)
 

diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py
@@ -72,9 +72,9 @@ def readAll(self):
 	def getEarlyOptions(self):
 		return self.__fail2ban.getEarlyOptions()
 
-	def getOptions(self, jail=None, updateMainOpt=None):
+	def getOptions(self, jail=None, updateMainOpt=None, ignoreWrong=True):
 		self.__fail2ban.getOptions(updateMainOpt)
-		return self.__jails.getOptions(jail)
+		return self.__jails.getOptions(jail, ignoreWrong=ignoreWrong)
 
 	def convertToProtocol(self):
 		self.__streams["general"] = self.__fail2ban.convert()

diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py
@@ -125,7 +125,7 @@ def __processCmd(self, cmd, showRet=True, timeout=-1):
 			if client:
 				try :
 					client.close()
-				except Exception as e:
+				except Exception as e: # pragma: no cover
 					if showRet or self._conf["verbose"] > 1:
 						logSys.debug(e)
 			if showRet or c[0] == 'echo':