Merge 77aa523 into 74e76e0

.travis.yml

@@ -5,10 +5,14 @@ python:
   - "2.5"
   - "2.6"
   - "2.7"
+  - "3.2"
+  - "3.3"
 install:
   - pip install pyinotify
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then pip install -q coveralls; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]] || [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then pip install -q coveralls; fi
+before_script:
+    - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi
 script:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coverage run --rcfile=.travis_coveragerc fail2ban-testcases; else python ./fail2ban-testcases; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]] || [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then coverage run --rcfile=.travis_coveragerc fail2ban-testcases; else python ./fail2ban-testcases; fi
 after_script:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coveralls; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]] || [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then coveralls; fi

MANIFEST

@@ -9,6 +9,7 @@ fail2ban-client
 fail2ban-server
 fail2ban-testcases
 fail2ban-regex
+fail2ban-2to3
 client/configreader.py
 client/configparserinc.py
 client/jailreader.py

client/beautifier.py

@@ -110,6 +110,9 @@ def beautify(self, response):
 					for path in response[:-1]:
 						msg = msg + "|- " + path + "\n"
 					msg = msg + "`- " + response[len(response)-1]
+			elif inC[2] == "logencoding":
+				msg = "Current log encoding is set to:\n"
+				msg = msg + response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"

client/csocket.py

@@ -29,11 +29,14 @@
 
 #from cPickle import dumps, loads, HIGHEST_PROTOCOL
 from pickle import dumps, loads, HIGHEST_PROTOCOL
-import socket
+import socket, sys
 
 class CSocket:
 
-	END_STRING = "<F2B_END_COMMAND>"
+	if sys.version_info >= (3,):
+		END_STRING = bytes("<F2B_END_COMMAND>", encoding='ascii')
+	else:
+		END_STRING = "<F2B_END_COMMAND>"
 
 	def __init__(self, sock = "/var/run/fail2ban/fail2ban.sock"):
 		# Create an INET, STREAMing socket
@@ -52,7 +55,10 @@ def send(self, msg):
 
 	#@staticmethod
 	def receive(sock):
-		msg = ''
+		if sys.version_info >= (3,):
+			msg = bytes("", encoding='ascii')
+		else:
+			msg = ''
 		while msg.rfind(CSocket.END_STRING) == -1:
 			chunk = sock.recv(6)
 			if chunk == '':

client/jailreader.py

@@ -62,6 +62,7 @@ def isEnabled(self):
 	def getOptions(self):
 		opts = [["bool", "enabled", "false"],
 				["string", "logpath", "/var/log/messages"],
+				["string", "logencoding", "auto"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
 				["int", "findtime", 600],
@@ -116,6 +117,8 @@ def convert(self):
 						logSys.error("No file found for " + path)
 					for p in pathList:
 						stream.append(["set", self.__name, "addlogpath", p])
+			elif opt == "logencoding":
+				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
 			elif opt == "backend":
 				backend = self.__opts[opt]
 			elif opt == "maxretry":

common/protocol.py

@@ -57,6 +57,7 @@
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
 ["set <JAIL> addlogpath <FILE>", "adds <FILE> to the monitoring list of <JAIL>"], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
+["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
@@ -78,6 +79,7 @@
 ["set <JAIL> actionunban <ACT> <CMD>", "sets the unban command <CMD> of the action <ACT> for <JAIL>"], 
 ['', "JAIL INFORMATION", ""],
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
+["get <JAIL> logencoding <ENCODING>", "gets the <ENCODING> of the log files for <JAIL>"],
 ["get <JAIL> ignoreip", "gets the list of ignored IP addresses for <JAIL>"],
 ["get <JAIL> failregex", "gets the list of regular expressions which matches the failures for <JAIL>"],
 ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"],

config/jail.conf

@@ -55,6 +55,13 @@ backend = auto
 #        but it will be logged as info.
 usedns = warn
 
+# "logencoding" specifies the encoding of the log files handled by the jail
+#   This is used to decode the lines from the log file.
+#   Typical examples:  "ascii", "utf-8"
+#
+#   auto:   will use the system locale setting
+logencoding = auto
+
 
 # This jail corresponds to the standard configuration in Fail2ban 0.6.
 # The mail-whois action send a notification e-mail with a whois request

fail2ban-2to3

@@ -0,0 +1,17 @@
+#!/bin/bash
+# This script carries out conversion of fail2ban to python3
+# A backup of any converted files are created with ".bak"
+# extension
+
+set -eu
+
+nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases"
+
+find . \
+    -name "*.py" \
+    -exec 2to3 -w --no-diffs $nonPyFiles {} + \
+    || (echo "Fail!" >&2 && exit 1)
+
+echo "Success!" >&2
+
+exit 0

fail2ban-regex

@@ -22,7 +22,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko"
 __license__ = "GPL"
 
-import getopt, sys, time, logging, os
+import getopt, sys, time, logging, os, locale
 
 # Inserts our own modules path first in the list
 # fix for bug #343821
@@ -77,6 +77,7 @@ class Fail2banRegex:
 		self.__ignoreregex = list()
 		self.__failregex = list()
 		self.__verbose = False
+		self.encoding = locale.getpreferredencoding()
 		# Setup logging
 		logging.getLogger("fail2ban").handlers = []
 		self.__hdlr = logging.StreamHandler(Fail2banRegex.test)
@@ -110,6 +111,8 @@ class Fail2banRegex:
 		print "This tools can test regular expressions for \"fail2ban\"."
 		print
 		print "Options:"
+		print "    -e ENCODING, --encoding=ENCODING"
+		print "                            set the file encoding. default:system locale"
 		print "    -h, --help              display this help message"
 		print "    -V, --version           print the version"
 		print "    -v, --verbose           verbose output"
@@ -141,6 +144,8 @@ class Fail2banRegex:
 				sys.exit(0)
 			elif opt[0] in ["-v", "--verbose"]:
 				self.__verbose = True
+			elif opt[0] in ["-e", "--encoding"]:
+				self.encoding = opt[1]
 
 	#@staticmethod
 	def logIsFile(value):
@@ -318,8 +323,8 @@ if __name__ == "__main__":
 	fail2banRegex = Fail2banRegex()
 	# Reads the command line options.
 	try:
-		cmdOpts = 'hVcv'
-		cmdLongOpts = ['help', 'version', 'verbose']
+		cmdOpts = 'e:hVcv'
+		cmdLongOpts = ['encoding=', 'help', 'version', 'verbose']
 		optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts)
 	except getopt.GetoptError:
 		fail2banRegex.dispUsage()
@@ -346,10 +351,16 @@ if __name__ == "__main__":
 
 		if fail2banRegex.logIsFile(cmd_log):
 			try:
-				hdlr = open(cmd_log)
+				hdlr = open(cmd_log, 'rb')
 				print "Use log file   : " + cmd_log
+				print "Use encoding   : " + fail2banRegex.encoding
 				print
 				for line in hdlr:
+					try:
+						line = line.decode(fail2banRegex.encoding, 'strict')
+					except UnicodeDecodeError:
+						if sys.version_info >= (3,): # Python 3 must be decoded
+							line = line.decode(fail2banRegex.encoding, 'ignore')
 					fail2banRegex.testIgnoreRegex(line)
 					fail2banRegex.testRegex(line)
 			except IOError, e:

fail2ban-server

@@ -104,10 +104,10 @@ class Fail2banServer:
 			if opt[0] == "-x":
 				self.__conf["force"] = True
 			if opt[0] in ["-h", "--help"]:
-	 			self.dispUsage()
+				self.dispUsage()
 				sys.exit(0)
 			if opt[0] in ["-V", "--version"]:
-	 			self.dispVersion()
+				self.dispVersion()
 				sys.exit(0)
 
 	def start(self, argv):

fail2ban-testcases-all-python3

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Simple helper script to exercise unittests using all available
+# (under /usr/bin and /usr/local/bin python3.*)
+
+set -eu
+
+failed=
+for python in /usr/{,local/}bin/python3.[0-9]{,.*}{,-dbg}
+do
+	[ -e "$python" ] || continue
+	echo "Testing using $python"
+	$python ./fail2ban-testcases "$@" || failed+=" $python"
+done
+
+if [ ! -z "$failed" ]; then
+	echo "E: Failed with $failed"
+	exit 1
+fi

server/action.py

@@ -342,7 +342,7 @@ def executeCmd(realCmd):
 					return True
 				else:
 					msg = _RETCODE_HINTS.get(retcode, None)
- 					logSys.error("%s returned %x" % (realCmd, retcode))
+					logSys.error("%s returned %x" % (realCmd, retcode))
 					if msg:
 						logSys.info("HINT on %x: %s"
 									% (retcode, msg % locals()))

server/asyncserver.py

@@ -42,7 +42,10 @@
 
 class RequestHandler(asynchat.async_chat):
 
-	END_STRING = "<F2B_END_COMMAND>"
+	if sys.version_info >= (3,):
+		END_STRING = bytes("<F2B_END_COMMAND>", encoding="ascii")
+	else:
+		END_STRING = "<F2B_END_COMMAND>"
 
 	def __init__(self, conn, transmitter):
 		asynchat.async_chat.__init__(self, conn)
@@ -62,7 +65,10 @@ def collect_incoming_data(self, data):
 
 	def found_terminator(self):
 		# Joins the buffer items.
-		message = loads("".join(self.__buffer))
+		if sys.version_info >= (3,):
+			message = loads(bytes("", encoding="ascii").join(self.__buffer))
+		else:
+			message = loads("".join(self.__buffer))
 		# Gives the message to the transmitter.
 		message = self.__transmitter.proceed(message)
 		# Serializes the response.

server/datedetector.py

@@ -194,7 +194,7 @@ def getUnixTime(self, line):
 		if date == None:
 			return None
 		else:
-			return time.mktime(date)
+			return time.mktime(tuple(date))
 
 	##
 	# Sort the template lists using the hits score. This method is not called
@@ -204,7 +204,7 @@ def sortTemplate(self):
 		self.__lock.acquire()
 		try:
 			logSys.debug("Sorting the template list")
-			self.__templates.sort(lambda x, y: cmp(x.getHits(), y.getHits()), reverse=True)
+			self.__templates.sort(key=lambda x: x.getHits(), reverse=True)
 			t = self.__templates[0]
 			logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits()))
 		finally:

server/datetemplate.py

@@ -166,10 +166,10 @@ def getDate(self, line):
 				# Bug fix for #1241756
 				# If the date is greater than the current time, we suppose
 				# that the log is not from this year but from the year before
-				if time.mktime(date) > MyTime.time():
+				if time.mktime(tuple(date)) > MyTime.time():
 					logSys.debug(
 						u"Correcting deduced year from %d to %d since %f > %f" %
-						(date[0], date[0]-1, time.mktime(date), MyTime.time()))
+						(date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time()))
 					# NOTE: Possibly makes week/year day incorrect
 					date[0] -= 1
 				elif date[1] == 1 and date[2] == 1: