Fix tab expansion for Python 3.

sqlparse/lexer.py

@@ -227,10 +227,15 @@ def add_filter(self, filter_, **options):
             filter_ = filter_(**options)
         self.filters.append(filter_)
 
+    def _expandtabs(self, text):
+        if self.tabsize > 0:
+            text = text.expandtabs(self.tabsize)
+        return text
+
     def _decode(self, text):
         if sys.version_info[0] == 3:
             if isinstance(text, str):
-                return text
+                return self._expandtabs(text)
         if self.encoding == 'guess':
             try:
                 text = text.decode('utf-8')
@@ -243,10 +248,7 @@ def _decode(self, text):
                 text = text.decode(self.encoding)
             except UnicodeDecodeError:
                 text = text.decode('unicode-escape')
-
-        if self.tabsize > 0:
-            text = text.expandtabs(self.tabsize)
-        return text
+        return self._expandtabs(text)
 
     def get_tokens(self, text, unfiltered=False):
         """

tests/test_tokenize.py

@@ -73,8 +73,6 @@ def test_negative_numbers(self):
         self.assertEqual(tokens[2][0], Number.Integer)
         self.assertEqual(tokens[2][1], '-1')
 
-    # Somehow this test fails on Python 3.2
-    @pytest.mark.skipif('sys.version_info >= (3,0)')
     def test_tab_expansion(self):
         s = "\t"
         lex = lexer.Lexer()