- performance of fail2ban optimized

   -- cache dnsToIp, ipToName to prevent long wait during retrieving of ip/name for wrong dns or lazy dns-system;
   -- instead of simple "sleep" used conditional wait "wait_for", that internal increases sleep interval up to sleeptime;
   -- ticket / banmanager / failmanager modules are performance optimized;
- performance of test cases optimized:
   -- added option "--fast" to decrease wait intervals, avoid passive waiting, and skip few very slow test cases;
- code review after partially cherry pick of branch 'ban-time-incr' (see gh-716)
   -- ticket module prepared to easy merge with newest version of 'ban-time-incr', now additionally holds banTime, banCount and json-data;
   -- executeCmd partially moved from action to new module utils, etc.
   -- python 2.6 compatibility;

MANIFEST

@@ -180,7 +180,6 @@ fail2ban/server/banmanager.py
 fail2ban/server/database.py
 fail2ban/server/datedetector.py
 fail2ban/server/datetemplate.py
-fail2ban/server/faildata.py
 fail2ban/server/failmanager.py
 fail2ban/server/failregex.py
 fail2ban/server/filter.py
@@ -197,6 +196,7 @@ fail2ban/server/server.py
 fail2ban/server/strptime.py
 fail2ban/server/ticket.py
 fail2ban/server/transmitter.py
+fail2ban/server/utils.py
 fail2ban/tests/__init__.py
 fail2ban/tests/action_d/__init__.py
 fail2ban/tests/action_d/test_badips.py

bin/fail2ban-testcases

@@ -58,6 +58,9 @@ def get_opt_parser():
 		Option('-n', "--no-network", action="store_true",
 			   dest="no_network",
 			   help="Do not run tests that require the network"),
+		Option('-f', "--fast", action="store_true",
+			   dest="fast",
+			   help="Try to increase speed of the tests, decreasing of wait intervals, memory database"),
 		Option("-t", "--log-traceback", action='store_true',
 			   help="Enrich log-messages with compressed tracebacks"),
 		Option("--full-traceback", action='store_true',
@@ -120,7 +123,7 @@ if not opts.log_level or opts.log_level != 'critical': # pragma: no cover
 	print("Fail2ban %s test suite. Python %s. Please wait..." \
 	    % (version, str(sys.version).replace('\n', '')))
 
-tests = gatherTests(regexps, opts.no_network)
+tests = gatherTests(regexps, opts)
 #
 # Run the tests
 #

doc/fail2ban.server.rst

@@ -10,7 +10,6 @@ fail2ban.server package
    fail2ban.server.database
    fail2ban.server.datedetector
    fail2ban.server.datetemplate
-   fail2ban.server.faildata
    fail2ban.server.failmanager
    fail2ban.server.failregex
    fail2ban.server.filter
@@ -26,3 +25,4 @@ fail2ban.server package
    fail2ban.server.strptime
    fail2ban.server.ticket
    fail2ban.server.transmitter
+   fail2ban.server.utils

doc/fail2ban.server.faildata.rst → doc/fail2ban.server.utils.rst

@@ -1,7 +1,7 @@
-fail2ban.server.faildata module
+fail2ban.server.utils module
 ===============================
 
-.. automodule:: fail2ban.server.faildata
+.. automodule:: fail2ban.server.utils
     :members:
     :undoc-members:
     :show-inheritance:

fail2ban/server/action.py

@@ -32,6 +32,7 @@
 from abc import ABCMeta
 from collections import MutableMapping
 
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -40,21 +41,6 @@
 # Create a lock for running system commands
 _cmd_lock = threading.Lock()
 
-# Some hints on common abnormal exit codes
-_RETCODE_HINTS = {
-	127: '"Command not found".  Make sure that all commands in %(realCmd)r '
-			'are in the PATH of fail2ban-server process '
-			'(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). '
-			'You may want to start '
-			'"fail2ban-server -f" separately, initiate it with '
-			'"fail2ban-client reload" in another shell session and observe if '
-			'additional informative error messages appear in the terminals.'
-	}
-
-# Dictionary to lookup signal name from number
-signame = dict((num, name)
-	for name, num in signal.__dict__.iteritems() if name.startswith("SIG"))
-
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -561,56 +547,6 @@ def executeCmd(realCmd, timeout=60):
 
 		_cmd_lock.acquire()
 		try: # Try wrapped within another try needed for python version < 2.5
-			stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_")
-			stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_")
-			try:
-				popen = subprocess.Popen(
-					realCmd, stdout=stdout, stderr=stderr, shell=True)
-				stime = time.time()
-				retcode = popen.poll()
-				while time.time() - stime <= timeout and retcode is None:
-					time.sleep(0.1)
-					retcode = popen.poll()
-				if retcode is None:
-					logSys.error("%s -- timed out after %i seconds." %
-						(realCmd, timeout))
-					os.kill(popen.pid, signal.SIGTERM) # Terminate the process
-					time.sleep(0.1)
-					retcode = popen.poll()
-					if retcode is None: # Still going...
-						os.kill(popen.pid, signal.SIGKILL) # Kill the process
-						time.sleep(0.1)
-						retcode = popen.poll()
-			except OSError, e:
-				logSys.error("%s -- failed with %s" % (realCmd, e))
+			return Utils.executeCmd(realCmd, timeout, shell=True, output=False)
 		finally:
 			_cmd_lock.release()
-
-		std_level = retcode == 0 and logging.DEBUG or logging.ERROR
-		if std_level >= logSys.getEffectiveLevel():
-			stdout.seek(0); msg = stdout.read()
-			if msg != '':
-				logSys.log(std_level, "%s -- stdout: %r", realCmd, msg)
-			stderr.seek(0); msg = stderr.read()
-			if msg != '':
-				logSys.log(std_level, "%s -- stderr: %r", realCmd, msg)
-		stdout.close()
-		stderr.close()
-
-		if retcode == 0:
-			logSys.debug("%s -- returned successfully" % realCmd)
-			return True
-		elif retcode is None:
-			logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid))
-		elif retcode < 0:
-			logSys.error("%s -- killed with %s" %
-				(realCmd, signame.get(-retcode, "signal %i" % -retcode)))
-		else:
-			msg = _RETCODE_HINTS.get(retcode, None)
-			logSys.error("%s -- returned %i" % (realCmd, retcode))
-			if msg:
-				logSys.info("HINT on %i: %s"
-							% (retcode, msg % locals()))
-			return False
-		raise RuntimeError("Command execution failed: %s" % realCmd)
-

fail2ban/server/actions.py

@@ -42,6 +42,7 @@
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -225,14 +226,11 @@ def run(self):
 					self._jail.name, name, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
-			if not self.idle:
-				#logSys.debug(self._jail.name + ": action")
-				ret = self.__checkBan()
-				if not ret:
-					self.__checkUnBan()
-					time.sleep(self.sleeptime)
-			else:
+			if self.idle:
 				time.sleep(self.sleeptime)
+				continue
+			if not Utils.wait_for(self.__checkBan, self.sleeptime):
+				self.__checkUnBan()
 		self.__flushBan()
 
 		actions = self._actions.items()

fail2ban/server/banmanager.py

@@ -247,12 +247,10 @@ def geBanListExtendedRIR(self, cymru_info):
 
 	@staticmethod
 	def createBanTicket(ticket):
-		ip = ticket.getIP()
-		#lastTime = ticket.getTime()
-		lastTime = MyTime.time()
-		banTicket = BanTicket(ip, lastTime, ticket.getMatches())
-		banTicket.setAttempt(ticket.getAttempt())
-		return banTicket
+		# we should always use correct time to calculate correct end time (ban time is variable now, 
+		# + possible double banning by restore from database and from log file)
+		# so use as lastTime always time from ticket.
+		return BanTicket(ticket=ticket)
 
 	##
 	# Add a ban ticket.
@@ -264,11 +262,25 @@ def createBanTicket(ticket):
 	def addBanTicket(self, ticket):
 		try:
 			self.__lock.acquire()
-			if not self._inBanList(ticket):
-				self.__banList.append(ticket)
-				self.__banTotal += 1
-				return True
-			return False
+			# check already banned
+			for i in self.__banList:
+				if ticket.getIP() == i.getIP():
+					# if already permanent
+					btorg, torg = i.getBanTime(self.__banTime), i.getTime()
+					if btorg == -1:
+						return False
+					# if given time is less than already banned time
+					btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
+					if btnew != -1 and tnew + btnew <= torg + btorg:
+						return False
+					# we have longest ban - set new (increment) ban time
+					i.setTime(tnew)
+					i.setBanTime(btnew)
+					return False
+			# not yet banned - add new
+			self.__banList.append(ticket)
+			self.__banTotal += 1
+			return True
 		finally:
 			self.__lock.release()
 
@@ -313,8 +325,7 @@ def unBanList(self, time):
 				return list()
 
 			# Gets the list of ticket to remove.
-			unBanList = [ticket for ticket in self.__banList
-						 if ticket.getTime() < time - self.__banTime]
+			unBanList = [ticket for ticket in self.__banList if ticket.isTimedOut(time, self.__banTime)]
 
 			# Removes tickets.
 			self.__banList = [ticket for ticket in self.__banList

fail2ban/server/database.py

@@ -418,8 +418,7 @@ def addBan(self, cur, jail, ticket):
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
 			(jail.name, ticket.getIP(), int(round(ticket.getTime())),
-				{"matches": ticket.getMatches(),
-				 "failures": ticket.getAttempt()}))
+				ticket.getData()))
 
 	@commitandrollback
 	def delBan(self, cur, jail, ip):
@@ -477,8 +476,8 @@ def getBans(self, **kwargs):
 		tickets = []
 		for ip, timeofban, data in self._getBans(**kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
-			tickets.append(FailTicket(ip, timeofban, data.get('matches')))
-			tickets[-1].setAttempt(data.get('failures', 1))
+			tickets.append(FailTicket(ip, timeofban))
+			tickets[-1].setData(data)
 		return tickets
 
 	def getBansMerged(self, ip=None, jail=None, bantime=None):
@@ -520,6 +519,7 @@ def getBansMerged(self, ip=None, jail=None, bantime=None):
 				prev_banip = results[0][0]
 				matches = []
 				failures = 0
+				tickdata = {}
 				for banip, timeofban, data in results:
 					#TODO: Implement data parts once arbitrary match keys completed
 					if banip != prev_banip:
@@ -530,11 +530,14 @@ def getBansMerged(self, ip=None, jail=None, bantime=None):
 						prev_banip = banip
 						matches = []
 						failures = 0
-					matches.extend(data.get('matches', []))
+						tickdata = {}
+					matches.extend(data.get('matches', ()))
 					failures += data.get('failures', 1)
+					tickdata.update(data.get('data', {}))
 					prev_timeofban = timeofban
 				ticket = FailTicket(banip, prev_timeofban, matches)
 				ticket.setAttempt(failures)
+				ticket.setData(**tickdata)
 				tickets.append(ticket)
 
 			if cacheKey: