Merge pull request #86 from modoboa/refactor/python3

Refactor/python3

.travis.yml

@@ -2,6 +2,7 @@ language: python
 cache: pip
 python:
   - "2.7"
+  - "3.4"
 
 env:
   - DB="POSTGRESQL"
@@ -17,7 +18,7 @@ before_install:
   - if [[ $DB = 'MYSQL' ]]; then pip install -q mysqlclient; fi
 
 install:
-  - pip install -e git+https://github.com/tonioo/modoboa.git#egg=modoboa
+  - pip install -e git+https://github.com/tonioo/modoboa@refactor/python3_fixes#egg=modoboa
   - pip install -q factory-boy testfixtures
   - python setup.py -q develop
 

modoboa_webmail/forms.py

@@ -2,12 +2,14 @@
 
 """Webmail forms."""
 
+from __future__ import unicode_literals
+
 from email.header import Header
 from email.mime.image import MIMEImage
 import os
 import pkg_resources
-import urllib
-from urlparse import urlparse
+
+from six.moves.urllib.parse import urlparse, unquote
 
 import lxml.html
 
@@ -68,7 +70,7 @@ def make_body_images_inline(body):
         if src is None:
             continue
         o = urlparse(src)
-        path = urllib.unquote(os.path.join(settings.BASE_DIR, o.path[1:]))
+        path = unquote(os.path.join(settings.BASE_DIR, o.path[1:]))
         if not os.path.exists(path):
             continue
         fname = os.path.basename(path)
@@ -82,7 +84,7 @@ def make_body_images_inline(body):
         )
         part["Content-Disposition"] = "inline"
         parts.append(part)
-    return lxml.html.tostring(html), parts
+    return lxml.html.tostring(html, encoding="unicode"), parts
 
 
 class ComposeMailForm(forms.Form):
@@ -199,7 +201,7 @@ def _format_sender_address(self, user, address):
         """Format address before message is sent."""
         if user.first_name != "" or user.last_name != "":
             return '"{}" <{}>'.format(
-                Header(user.fullname, "utf8").encode(), address)
+                Header(user.fullname, "utf8"), address)
         return address
 
     def _build_msg(self, request):

modoboa_webmail/lib/attachments.py

@@ -1,6 +1,11 @@
+from __future__ import unicode_literals
+
+from email import encoders
+from email.mime.base import MIMEBase
 import os
 
 from rfc6266 import build_header
+import six
 
 from django.conf import settings
 from django.core.files.uploadhandler import FileUploadHandler, SkipFile
@@ -71,9 +76,6 @@ def create_mail_attachment(attdef, payload=None):
     :param attdef: a dictionary containing the attachment definition
     :return: a MIMEBase object
     """
-    from email import Encoders
-    from email.mime.base import MIMEBase
-
     if "content-type" in attdef:
         maintype, subtype = attdef["content-type"].split("/")
     elif "Content-Type" in attdef:
@@ -88,10 +90,10 @@ def create_mail_attachment(attdef, payload=None):
             res.set_payload(fp.read())
     else:
         res.set_payload(payload)
-    Encoders.encode_base64(res)
-    if isinstance(attdef['fname'], str):
-        attdef['fname'] = attdef['fname'].decode('utf-8')
-    res['Content-Disposition'] = build_header(attdef['fname'])
+    encoders.encode_base64(res)
+    if isinstance(attdef["fname"], six.binary_type):
+        attdef["fname"] = attdef["fname"].decode("utf-8")
+    res["Content-Disposition"] = build_header(attdef["fname"])
     return res
 
 

modoboa_webmail/lib/fetch_parser.py

@@ -1,15 +1,20 @@
 # coding: utf-8
-"""
-:mod:`fetch_parser` --- Simple parser for FETCH responses
----------------------------------------------------------
+"""Simple parser for FETCH responses.
 
 The ``imaplib`` module doesn't parse IMAP responses, it returns raw
-values. This is pretty annoying when one FETCH command is issued to
-retrieve multiple attributes of a message.
+values. Since Modoboa relies on BODYSTRUCTURE attributes to display
+messages (we don't want to overload the server), a parser is required.
+
+Python 2/3 compatibility note: this parser excepts bytes objects when
+run with Python3 and str (not unicode) ones with Python2.
 
-This simple module tries to fix that *problem*.
 """
 
+# from __future__ import unicode_literals
+
+import chardet
+import six
+
 
 class ParseError(Exception):
     pass
@@ -119,9 +124,36 @@ def parse_bodystructure(buf, depth=0, prefix=""):
         "End of buffer reached while looking for a BODY/BODYSTRUCTURE end")
 
 
+def convert_to_str(chunk):
+    """Convert a list of bytes to str and guess encoding."""
+    buf = ""
+    if not isinstance(chunk, (list, tuple)):
+        chunk = (chunk, )
+    for part in chunk:
+        condition = (
+            six.PY2 and isinstance(part, six.text_type) or
+            six.PY3 and isinstance(part, six.binary_type)
+        )
+        if not condition:
+            buf += part
+            continue
+        try:
+            buf += part.decode("utf-8")
+        except UnicodeDecodeError:
+            pass
+        else:
+            continue
+        try:
+            result = chardet.detect(part)
+        except UnicodeDecodeError:
+            raise RuntimeError("Can't find string encoding")
+        buf += part.decode(result["encoding"])
+    return buf
+
+
 def parse_response_chunk(chunk):
     """Parse a piece of response..."""
-    buf = "".join(chunk)
+    buf = convert_to_str(chunk)
     parts = buf.split(" ", 3)
     result = {}
     response = parts[3]
@@ -165,7 +197,7 @@ def parse_response_chunk(chunk):
         result[cmdname] = response[:end]
         response = response[end + 1:]
         try:
-            func = globals()["parse_%s" % cmdname.lower()]
+            func = globals()["parse_{}".format(cmdname.lower())]
             result[cmdname] = func(result[cmdname])[0]
         except KeyError:
             pass
@@ -189,8 +221,8 @@ def parse_fetch_response(data):
     else:
         while cpt < len(data):
             content = ()
-            while cpt < len(data) and data[cpt] != ")":
-                if isinstance(data[cpt], str):
+            while cpt < len(data) and data[cpt] != b")":
+                if type(data[cpt]) in [bytes, str]:
                     # FIXME : probably an unsolicited response
                     cpt += 1
                     continue
@@ -204,38 +236,45 @@ def parse_fetch_response(data):
 
 def dump_bodystructure(bs, depth=0):
     if depth:
-        print " " * (depth * 4),
+        print(" " * (depth * 4), )
     if isinstance(bs[0], dict):
-        print "%s :" % bs[0]["partnum"],
+        print("%s :" % bs[0]["partnum"], )
         struct = bs[0]["struct"]
     else:
         struct = bs
 
     if isinstance(struct[0], list):
-        print "multipart/%s" % struct[1]
+        print("multipart/%s" % struct[1])
         for part in struct[0]:
             dump_bodystructure(part, depth + 1)
     else:
-        print "%s/%s" % (struct[0], struct[1])
+        print("%s/%s" % (struct[0], struct[1]))
 
 
 if __name__ == "__main__":
-#    resp = [('855 (UID 46931 BODYSTRUCTURE ((("text" "plain" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 886 32 NIL NIL NIL NIL)("text" "html" ("charset" "us-ascii") NIL NIL "quoted-printable" 1208 16 NIL NIL NIL NIL) "alternative" ("boundary" "----=_NextPart_001_0003_01CCC564.B2F64FF0") NIL NIL NIL)("application" "octet-stream" ("name" "Carte Verte_2.pdf") NIL NIL "base64" 285610 NIL ("attachment" ("filename" "Carte Verte_2.pdf")) NIL NIL) "mixed" ("boundary" "----=_NextPart_000_0002_01CCC564.B2F64FF0") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {153}', 'From: <Service.client10@maaf.fr>\r\nTo: <TONIO@NGYN.ORG>\r\nCc: \r\nSubject: Notre contact du 28/12/2011 - 192175092\r\nDate: Wed, 28 Dec 2011 13:29:17 +0100\r\n\r\n'), ')']
+    # resp = [(b'855 (UID 46931 BODYSTRUCTURE ((("text" "plain" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 886 32 NIL NIL NIL NIL)("text" "html" ("charset" "us-ascii") NIL NIL "quoted-printable" 1208 16 NIL NIL NIL NIL) "alternative" ("boundary" "----=_NextPart_001_0003_01CCC564.B2F64FF0") NIL NIL NIL)("application" "octet-stream" ("name" "Carte Verte_2.pdf") NIL NIL "base64" 285610 NIL ("attachment" ("filename" "Carte Verte_2.pdf")) NIL NIL) "mixed" ("boundary" "----=_NextPart_000_0002_01CCC564.B2F64FF0") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {153}', b'From: <Service.client10@maaf.fr>\r\nTo: <TONIO@NGYN.ORG>\r\nCc: \r\nSubject: Notre contact du 28/12/2011 - 192175092\r\nDate: Wed, 28 Dec 2011 13:29:17 +0100\r\n\r\n'), b')']
 
 #    resp = [('856 (UID 46936 BODYSTRUCTURE (("text" "plain" ("charset" "ISO-8859-1") NIL NIL "quoted-printable" 724 22 NIL NIL NIL NIL)("text" "html" ("charset" "ISO-8859-1") NIL NIL "quoted-printable" 2662 48 NIL NIL NIL NIL) "alternative" ("boundary" "----=_Part_1326887_254624357.1325083973970") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {258}', 'Date: Wed, 28 Dec 2011 15:52:53 +0100 (CET)\r\nFrom: =?ISO-8859-1?Q?Malakoff_M=E9d=E9ric?= <communication@communication.malakoffmederic.com>\r\nTo: Antoine Nguyen <tonio@ngyn.org>\r\nSubject: =?ISO-8859-1?Q?Votre_inscription_au_grand_Jeu_Malakoff_M=E9d=E9ric?=\r\n\r\n'), ')']
 
-    resp = [
-        ('856 (UID 11111 BODYSTRUCTURE ((("text" "plain" ("charset" "UTF-8") NIL NIL "7bit" 0 0 NIL NIL NIL NIL) "mixed" ("boundary" "----=_Part_407172_3159001.1321948277321") NIL NIL NIL)("application" "octet-stream" ("name" "26274308.pdf") NIL NIL "base64" 14906 NIL ("attachment" ("filename" "26274308.pdf")) NIL NIL) "mixed" ("boundary" "----=_Part_407171_9686991.1321948277321") NIL NIL NIL)',
-         ),
-        ')']
-
-#    resp = [('19 (UID 19 FLAGS (\\Seen) BODYSTRUCTURE (("text" "plain" ("charset" "ISO-8859-1" "format" "flowed") NIL NIL "7bit" 2 1 NIL NIL NIL NIL)("message" "rfc822" ("name*" "ISO-8859-1\'\'%5B%49%4E%53%43%52%49%50%54%49%4F%4E%5D%20%52%E9%63%E9%70%74%69%6F%6E%20%64%65%20%76%6F%74%72%65%20%64%6F%73%73%69%65%72%20%64%27%69%6E%73%63%72%69%70%74%69%6F%6E%20%46%72%65%65%20%48%61%75%74%20%44%E9%62%69%74") NIL NIL "8bit" 3632 ("Wed, 13 Dec 2006 20:30:02 +0100" {70}',
-#  "[INSCRIPTION] R\xe9c\xe9ption de votre dossier d'inscription Free Haut D\xe9bit"),
-#            (' (("Free Haut Debit" NIL "inscription" "freetelecom.fr")) (("Free Haut Debit" NIL "inscription" "freetelecom.fr")) ((NIL NIL "hautdebit" "freetelecom.fr")) ((NIL NIL "nguyen.antoine" "wanadoo.fr")) NIL NIL NIL "<20061213193125.9DA0919AC@dgroup2-2.proxad.net>") ("text" "plain" ("charset" "iso-8859-1") NIL NIL "8bit" 1428 38 NIL ("inline" NIL) NIL NIL) 76 NIL ("inline" ("filename*" "ISO-8859-1\'\'%5B%49%4E%53%43%52%49%50%54%49%4F%4E%5D%20%52%E9%63%E9%70%74%69%6F%6E%20%64%65%20%76%6F%74%72%65%20%64%6F%73%73%69%65%72%20%64%27%69%6E%73%63%72%69%70%74%69%6F%6E%20%46%72%65%65%20%48%61%75%74%20%44%E9%62%69%74")) NIL NIL) "mixed" ("boundary" "------------040706080908000209030901") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {266}',
-#             'Date: Tue, 19 Dec 2006 19:50:13 +0100\r\nFrom: Antoine Nguyen <nguyen.antoine@wanadoo.fr>\r\nTo: Antoine Nguyen <tonio@koalabs.org>\r\nSubject: [Fwd: [INSCRIPTION] =?ISO-8859-1?Q?R=E9c=E9ption_de_votre_?=\r\n =?ISO-8859-1?Q?dossier_d=27inscription_Free_Haut_D=E9bit=5D?=\r\n\r\n'), ')']
-
-    # resp = [('123 (UID 3 BODYSTRUCTURE (((("text" "plain" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 1266 30 NIL NIL NIL NIL)("text" "html" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 8830 227 NIL NIL NIL NIL) "alternative" ("boundary" "_000_152AC7ECD1F8AB43A9AD95DBDDCA3118082C09GKIMA24cmcicfr_") NIL NIL NIL)("image" "png" ("name" "image005.png") "<image005.png@01CC6CAA.4FADC490>" "image005.png" "base64" 7464 NIL ("inline" ("filename" "image005.png" "size" "5453" "creation-date" "Tue, 06 Sep 2011 13:33:49 GMT" "modification-date" "Tue, 06 Sep 2011 13:33:49 GMT")) NIL NIL)("image" "jpeg" ("name" "image006.jpg") "<image006.jpg@01CC6CAA.4FADC490>" "image006.jpg" "base64" 2492 NIL ("inline" ("filename" "image006.jpg" "size" "1819" "creation-date" "Tue, 06 Sep 2011 13:33:49 GMT" "modification-date" "Tue, 06 Sep 2011 13:33:49 GMT")) NIL NIL) "related" ("boundary" "_006_152AC7ECD1F8AB43A9AD95DBDDCA3118082C09GKIMA24cmcicfr_" "type" "multipart/alternative") NIL NIL NIL)("application" "pdf" ("name" "bilan assurance CIC.PDF") NIL "bilan assurance CIC.PDF" "base64" 459532 NIL ("attachment" ("filename" "bilan assurance CIC.PDF" "size" "335811" "creation-date" "Fri, 16 Sep 2011 12:45:23 GMT" "modification-date" "Fri, 16 Sep 2011 12:45:23 GMT")) NIL NIL)(("text" "plain" ("charset" "utf-8") NIL NIL "quoted-printable" 1389 29 NIL NIL NIL NIL)("text" "html" ("charset" "utf-8") NIL NIL "quoted-printable" 1457 27 NIL NIL NIL NIL) "alternative" ("boundary" "===============0775904800==") ("inline" NIL) NIL NIL) "mixed" ("boundary" "_007_152AC7ECD1F8AB43A9AD95DBDDCA3118082C09GKIMA24cmcicfr_") NIL ("fr-FR") NIL)',), ')']
+    # resp = [
+    #     (b'856 (UID 11111 BODYSTRUCTURE ((("text" "plain" ("charset" "UTF-8") NIL NIL "7bit" 0 0 NIL NIL NIL NIL) "mixed" ("boundary" "----=_Part_407172_3159001.1321948277321") NIL NIL NIL)("application" "octet-stream" ("name" "26274308.pdf") NIL NIL "base64" 14906 NIL ("attachment" ("filename" "26274308.pdf")) NIL NIL) "mixed" ("boundary" "----=_Part_407171_9686991.1321948277321") NIL NIL NIL)',
+    #      ),
+    #     b')']
+
+    # resp = [
+    #     (
+    #         b'19 (UID 19 FLAGS (\\Seen) BODYSTRUCTURE (("text" "plain" ("charset" "ISO-8859-1" "format" "flowed") NIL NIL "7bit" 2 1 NIL NIL NIL NIL)("message" "rfc822" ("name*" "ISO-8859-1\'\'%5B%49%4E%53%43%52%49%50%54%49%4F%4E%5D%20%52%E9%63%E9%70%74%69%6F%6E%20%64%65%20%76%6F%74%72%65%20%64%6F%73%73%69%65%72%20%64%27%69%6E%73%63%72%69%70%74%69%6F%6E%20%46%72%65%65%20%48%61%75%74%20%44%E9%62%69%74") NIL NIL "8bit" 3632 ("Wed, 13 Dec 2006 20:30:02 +0100" {70}',
+    #         b"[INSCRIPTION] R\xe9c\xe9ption de votre dossier d'inscription Free Haut D\xe9bit"
+    #     ),
+    #     (
+    #         b' (("Free Haut Debit" NIL "inscription" "freetelecom.fr")) (("Free Haut Debit" NIL "inscription" "freetelecom.fr")) ((NIL NIL "hautdebit" "freetelecom.fr")) ((NIL NIL "nguyen.antoine" "wanadoo.fr")) NIL NIL NIL "<20061213193125.9DA0919AC@dgroup2-2.proxad.net>") ("text" "plain" ("charset" "iso-8859-1") NIL NIL "8bit" 1428 38 NIL ("inline" NIL) NIL NIL) 76 NIL ("inline" ("filename*" "ISO-8859-1\'\'%5B%49%4E%53%43%52%49%50%54%49%4F%4E%5D%20%52%E9%63%E9%70%74%69%6F%6E%20%64%65%20%76%6F%74%72%65%20%64%6F%73%73%69%65%72%20%64%27%69%6E%73%63%72%69%70%74%69%6F%6E%20%46%72%65%65%20%48%61%75%74%20%44%E9%62%69%74")) NIL NIL) "mixed" ("boundary" "------------040706080908000209030901") NIL NIL NIL) BODY[HEADER.FIELDS (DATE FROM TO CC SUBJECT)] {266}',
+    #         b'Date: Tue, 19 Dec 2006 19:50:13 +0100\r\nFrom: Antoine Nguyen <nguyen.antoine@wanadoo.fr>\r\nTo: Antoine Nguyen <tonio@koalabs.org>\r\nSubject: [Fwd: [INSCRIPTION] =?ISO-8859-1?Q?R=E9c=E9ption_de_votre_?=\r\n =?ISO-8859-1?Q?dossier_d=27inscription_Free_Haut_D=E9bit=5D?=\r\n\r\n'
+    #     ),
+    #     b')'
+    # ]
+
+    resp = [(b'123 (UID 3 BODYSTRUCTURE (((("text" "plain" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 1266 30 NIL NIL NIL NIL)("text" "html" ("charset" "iso-8859-1") NIL NIL "quoted-printable" 8830 227 NIL NIL NIL NIL) "alternative" ("boundary" "_000_152AC7ECD1F8AB43A9AD95DBDDCA3118082C09GKIMA24cmcicfr_") NIL NIL NIL)("image" "png" ("name" "image005.png") "<image005.png@01CC6CAA.4FADC490>" "image005.png" "base64" 7464 NIL ("inline" ("filename" "image005.png" "size" "5453" "creation-date" "Tue, 06 Sep 2011 13:33:49 GMT" "modification-date" "Tue, 06 Sep 2011 13:33:49 GMT")) NIL NIL)("image" "jpeg" ("name" "image006.jpg") "<image006.jpg@01CC6CAA.4FADC490>" "image006.jpg" "base64" 2492 NIL ("inline" ("filename" "image006.jpg" "size" "1819" "creation-date" "Tue, 06 Sep 2011 13:33:49 GMT" "modification-date" "Tue, 06 Sep 2011 13:33:49 GMT")) NIL NIL) "related" ("boundary" "_006_152AC7ECD1F8AB43A9AD95DBDDCA3118082C09GKIMA24cmcicfr_" "type" "multipart/alternative") NIL NIL NIL)("application" "pdf" ("name" "bilan assurance CIC.PDF") NIL "bilan assurance CIC.PDF" "base64" 459532 NIL ("attachment" ("filename" "bilan assurance CIC.PDF" "size" "335811" "creation-date" "Fri, 16 Sep 2011 12:45:23 GMT" "modification-date" "Fri, 16 Sep 2011 12:45:23 GMT")) NIL NIL)(("text" "plain" ("charset" "utf-8") NIL NIL "quoted-printable" 1389 29 NIL NIL NIL NIL)("text" "html" ("charset" "utf-8") NIL NIL "quoted-printable" 1457 27 NIL NIL NIL NIL) "alternative" ("boundary" "===============0775904800==") ("inline" NIL) NIL NIL) "mixed" ("boundary" "_007_152AC7ECD1F8AB43A9AD95DBDDCA3118082C09GKIMA24cmcicfr_") NIL ("fr-FR") NIL)',), b')']
 
     # resp = [('856 (UID 11111 BODYSTRUCTURE ((("text" "plain" ("charset" "UTF-8") NIL NIL "7bit" 0 0 NIL NIL NIL NIL) "mixed" ("boundary" "----=_Part_407172_3159001.1321948277321") NIL NIL NIL)("application" "octet-stream" ("name" "26274308.pdf") NIL NIL "base64" 14906 NIL ("attachment" ("filename" "(26274308.pdf")) NIL NIL) "mixed" ("boundary" "----=_Part_407171_9686991.1321948277321") NIL NIL NIL)',), ')']
 
-    print parse_fetch_response(resp)
+    print(parse_fetch_response(resp))

modoboa_webmail/lib/imapemail.py

@@ -7,6 +7,7 @@
 
 import chardet
 from rfc6266 import parse_headers
+import six
 
 from django.conf import settings
 from django.core.files.base import ContentFile
@@ -141,13 +142,14 @@ def body(self):
                 content = decode_payload(
                     part["encoding"], data[int(self.mailid)]["BODY[%s]" % pnum]
                 )
-                charset = self._find_content_charset(part)
-                if charset is not None:
-                    try:
-                        content = content.decode(charset)
-                    except (UnicodeDecodeError, LookupError):
-                        result = chardet.detect(content)
-                        content = content.decode(result["encoding"])
+                if not isinstance(content, six.text_type):
+                    charset = self._find_content_charset(part)
+                    if charset is not None:
+                        try:
+                            content = content.decode(charset)
+                        except (UnicodeDecodeError, LookupError):
+                            result = chardet.detect(content)
+                            content = content.decode(result["encoding"])
                 bodyc += content
             self._fetch_inlines()
             self._body = getattr(self, "viewmail_%s" % self.mformat)(
@@ -194,7 +196,7 @@ def _find_attachments(self):
 
     def _fetch_inlines(self):
         """Store inline images on filesystem to display them."""
-        for cid, params in self.bs.inlines.iteritems():
+        for cid, params in list(self.bs.inlines.items()):
             if re.search(r"\.\.", cid):
                 continue
             fname = "modoboa_webmail/%s_%s" % (self.mailid, cid)

modoboa_webmail/lib/imapheader.py

@@ -2,10 +2,13 @@
 Set of functions used to parse and transform email headers.
 """
 
+from __future__ import unicode_literals
+
 import datetime
 import email
 
 import chardet
+import six
 
 from django.utils import timezone
 from django.utils.formats import date_format
@@ -40,7 +43,10 @@
 
 def to_unicode(value):
     """Try to convert a string to unicode."""
-    if value is None or isinstance(value, unicode):
+    condition = (
+        value is None or isinstance(value, six.text_type)
+    )
+    if condition:
         return value
     try:
         value = value.decode("utf-8")