Merge pull request #2246 from lfdversluis/issue-2236-torrents-decode-…

…chardet

Introduced chardet, the fullpath is now decoded with chardet

Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py

@@ -11,11 +11,11 @@
 from twisted.internet import defer
 
 from Tribler.Core import NoDispersyRLock
-from Tribler.Core.APIImplementation import maketorrent
 from Tribler.Core.DownloadConfig import DownloadStartupConfig, DownloadConfigInterface
 from Tribler.Core.DownloadState import DownloadState
 from Tribler.Core.Libtorrent import checkHandleAndSynchronize, waitForHandleAndSynchronize
 from Tribler.Core.TorrentDef import TorrentDefNoMetainfo, TorrentDef
+from Tribler.Core.Utilities import maketorrent
 from Tribler.Core.Utilities.torrent_utils import get_info_from_handle
 from Tribler.Core.osutils import fix_filebasename
 from Tribler.Core.simpledefs import (DLSTATUS_WAITING4HASHCHECK, DLSTATUS_HASHCHECKING, DLSTATUS_METADATA,

Tribler/Core/TorrentDef.py

@@ -13,7 +13,7 @@
 from Tribler.Core.simpledefs import INFOHASH_LENGTH
 from Tribler.Core.defaults import TDEF_DEFAULTS
 from Tribler.Core.exceptions import TorrentDefNotFinalizedException, NotYetImplementedException
-import Tribler.Core.APIImplementation.maketorrent as maketorrent
+import Tribler.Core.Utilities.maketorrent as maketorrent
 
 from Tribler.Core.Utilities.utilities import validTorrentFile, isValidURL, parse_magnetlink
 from Tribler.Core.Utilities.unicode import dunno2unicode

Tribler/Core/APIImplementation/maketorrent.py → Tribler/Core/Utilities/maketorrent.py

@@ -12,6 +12,8 @@
 from types import LongType
 from libtorrent import bencode
 
+import chardet
+
 from Tribler.Core.Utilities.unicode import bin2unicode
 from Tribler.Core.APIImplementation.miscutils import offset2piece
 from Tribler.Core.osutils import fix_filebasename
@@ -243,7 +245,12 @@ def pathlist2filename(pathlist):
     fullpath = ''
     for elem in pathlist:
         fullpath = os.path.join(fullpath, elem)
-    return fullpath.decode('utf-8')
+
+    try:
+        return fullpath.decode('utf-8')
+    except UnicodeDecodeError:
+        charenc = chardet.detect(fullpath)['encoding']
+        return fullpath.decode(charenc)
 
 
 def pathlist2savefilename(pathlist, encoding):

Tribler/Test/Core/Utilities/__init__.py

@@ -0,0 +1,3 @@
+"""
+This package contains tests for the Utilities module in Tribler/Core.
+"""

Tribler/Test/Core/Utilities/test_maketorrent.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+import os
+
+from Tribler.Core.Utilities.maketorrent import pathlist2filename
+from Tribler.Test.test_as_server import BaseTestCase
+
+
+class TestMakeTorrent(BaseTestCase):
+
+    def test_pathlist2filename_utf8(self):
+        path_list = [u"test", u"path"]
+        path = pathlist2filename(path_list)
+        self.assertEqual(path, os.path.join(u"test", u"path"))
+
+    def test_pathlist2filename_not_utf8(self):
+        part = u'\xb0\xe7'.encode("latin-1")
+        path_list = ["test", part]
+        path = pathlist2filename(path_list)
+        self.assertEqual(path, os.path.join(u"test", u"\xb0\xe7"))