specify input file encoding manually

textract/parsers/__init__.py

@@ -22,13 +22,13 @@
 # default encoding that is returned by the process method. specify it
 # here so the default is used on both the process function and also by
 # the command line interface
-DEFAULT_ENCODING = 'utf_8'
+DEFAULT_OUTPUT_ENCODING = 'utf_8'
 
 # filename format
 _FILENAME_SUFFIX = '_parser'
 
 
-def process(filename, encoding=DEFAULT_ENCODING, extension=None, **kwargs):
+def process(filename, input_encoding=None, output_encoding=DEFAULT_OUTPUT_ENCODING, extension=None, **kwargs):
     """This is the core function used for extracting text. It routes the
     ``filename`` to the appropriate parser and returns the extracted
     text as a byte-string encoded with ``encoding``.
@@ -74,7 +74,7 @@ def process(filename, encoding=DEFAULT_ENCODING, extension=None, **kwargs):
     # do the extraction
 
     parser = filetype_module.Parser()
-    return parser.process(filename, encoding, **kwargs)
+    return parser.process(filename, input_encoding, output_encoding, **kwargs)
 
 
 def _get_available_extensions():

textract/parsers/utils.py

@@ -32,7 +32,7 @@ def encode(self, text, encoding):
         """
         return text.encode(encoding, 'ignore')
 
-    def process(self, filename, encoding, **kwargs):
+    def process(self, filename, input_encoding, output_encoding="utf8", **kwargs):
         """Process ``filename`` and encode byte-string with ``encoding``. This
         method is called by :func:`textract.parsers.process` and wraps
         the :meth:`.BaseParser.extract` method in `a delicious unicode
@@ -44,10 +44,10 @@ def process(self, filename, encoding, **kwargs):
         # output encoding
         # http://nedbatchelder.com/text/unipain/unipain.html#35
         byte_string = self.extract(filename, **kwargs)
-        unicode_string = self.decode(byte_string)
-        return self.encode(unicode_string, encoding)
+        unicode_string = self.decode(byte_string, input_encoding)
+        return self.encode(unicode_string, output_encoding)
 
-    def decode(self, text):
+    def decode(self, text, input_encoding=None):
         """Decode ``text`` using the `chardet
         <https://github.com/chardet/chardet>`_ package.
         """
@@ -60,7 +60,11 @@ def decode(self, text):
         if not text:
             return u''
 
-        # use chardet to automatically detect the encoding text
+        # use the provided encoding
+        if input_encoding:
+            return text.decode(input_encoding)
+
+        # use chardet to automatically detect the encoding text if no encoding is provided
         result = chardet.detect(text)
         return text.decode(result['encoding'])