fix python 2/3 unicode inconsistency (part 2)

gransk/core/document.py

@@ -168,7 +168,7 @@ def get_document(path, parent=None):
   doc.docid = digest.hexdigest()
 
   _, ext = os.path.splitext(doc.path)
-  doc.ext = ext.lower() or 'none'
+  doc.ext = ext.lstrip('.').lower() or 'none'
 
   doc.parent = parent
 

gransk/core/magic.py

@@ -19,7 +19,7 @@ class Subscriber(abstract_subscriber.Subscriber):
 
   def setup(self, _):
     """Compile file headers for all magic extractors into a regex pattern."""
-    self.pattern = re.compile('|'.join(list(self.pipeline.magic.keys())))
+    self.pattern = re.compile(b'|'.join(list(self.pipeline.magic.keys())))
 
   def consume(self, doc, payload):
     """

gransk/plugins/extractors/tests/tika_extractor_test.py

@@ -27,8 +27,8 @@ def test_simple(self):
     extractor = tika_extractor.Subscriber(mock_pipeline)
 
     expected = (
-        'This is an unstructured document containing the \nidentifier '
-        '"193.34.2.1" (ip address), stored as a PDF document.')
+        b'This is an unstructured document containing the \nidentifier '
+        b'"193.34.2.1" (ip address), stored as a PDF document.')
 
     with open('config.yml') as inp:
       config = yaml.load(inp.read())
@@ -48,7 +48,7 @@ def test_simple(self):
 
     actual = doc.text
 
-    self.assertEqual(expected, actual)
+    self.assertEqual(expected.decode('utf-8'), actual)
 
 
 if __name__ == '__main__':

gransk/plugins/storage/store_text.py

@@ -5,6 +5,7 @@
 
 import os
 import logging
+import io
 
 from werkzeug import secure_filename
 
@@ -54,7 +55,7 @@ def consume(self, doc, payload):
 
     new_path = os.path.join(self.root, new_filename)
 
-    with open(new_path, 'wb') as out:
+    with io.open(new_path, 'w', encoding='utf-8') as out:
       out.write(doc.text)
 
     doc.meta['text_file'] = new_path

utils/helper_usage.py

@@ -1,3 +1,7 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
 import os
 import re
 import sys
@@ -26,10 +30,10 @@
                     direct[m].update([p])
 
 for m in sorted(use.keys()):
-    #print "%s = '%s' # %s" % (m, m.lower(), ', '.join(use[m]))
-    print "%s = '%s'" % (m, m.lower())
+    #print("%s = '%s' # %s" % (m, m.lower(), ', '.join(use[m])))
+    print("%s = '%s'" % (m, m.lower()))
 
-print "\n\n__DIRECT__"
+print("\n\n__DIRECT__")
 
 for m in sorted(direct.keys()):
-    print "%s = '%s' # %s" % (m.upper(), m, ', '.join(direct[m]))
+    print("%s = '%s' # %s" % (m.upper(), m, ', '.join(direct[m])))