Cleanups

src/zope/app/locales/extract.py

@@ -19,6 +19,7 @@
 from __future__ import print_function
 __docformat__ = 'restructuredtext'
 
+from collections import defaultdict
 import os
 import sys
 import fnmatch
@@ -33,11 +34,11 @@
 from zope.i18nmessageid import Message
 from zope.app.locales.interfaces import IPOTEntry, IPOTMaker, ITokenEater
 
-if not hasattr(tokenize, 'generate_tokens'):
+try:
+    from tokenize import generate_tokens
+except ImportError:
     # The function was renamed in Python 3
-    generate_tokens = tokenize.tokenize
-else:
-    generate_tokens = tokenize.generate_tokens
+    from tokenize import tokenize as generate_tokens
 
 try:
     text_type = unicode
@@ -51,7 +52,7 @@
 pot_header = u'''\
 ##############################################################################
 #
-# Copyright (c) 2003-2004 Zope Foundation and Contributors.
+# Copyright (c) 2003-2017 Zope Foundation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -129,7 +130,9 @@ class POTEntry(object):
     ``unicode`` on its value and default value; on Python 3, ``unicode`` is str, so if
     we pass a bytes literal, we wind up doing str(bytes), which produces awkward
     data: "b'\\xd6"; this doesn't work at all when the C extension is not available
-    on Python 2 because the ``unicode`` constructor raises an error):
+    on Python 2 because the ``unicode`` constructor raises an error).
+    See https://github.com/zopefoundation/zope.i18nmessageid/issues/5 and
+    https://github.com/zopefoundation/zope.i18nmessageid/issues/4:
 
     >>> from zope.i18nmessageid.message import pyMessage
     >>> if pyMessage is not Message and str is bytes:
@@ -275,13 +278,13 @@ class TokenEater(object):
     ...     u"_('k, bye', ''); "
     ...     u"_('kthxbye')"
     ...     )
-    >>> for t in generate_tokens(file.readline): eater(*t)
+    >>> for t in generate_tokens(file.readline):
+    ...     eater(*t)
 
     The catalog of collected message ids contains our example
 
     >>> catalog = eater.getCatalog()
-    >>> items = list(catalog.items())
-    >>> items.sort()
+    >>> items = sorted(catalog.items())
     >>> items
     [(u'hello ${name}', [(None, 1)]),
      (u'hi ${name}', [(None, 1)]),
@@ -316,7 +319,6 @@ class TokenEater(object):
     Note that everything gets converted to unicode.
     """
 
-
     def __init__(self):
         self.__messages = {}
         self.__state = self.__waiting
@@ -406,11 +408,10 @@ def getCatalog(self):
         catalog = {}
         # Sort the entries.  First sort each particular entry's keys, then
         # sort all the entries by their first item.
-        reverse = {}
+        reverse = defaultdict(list)
         for k, v in self.__messages.items():
-            keys = list(v.keys())
-            keys.sort()
-            reverse.setdefault(tuple(keys), []).append((k, v))
+            reverse[tuple(sorted(v.keys()))].append((k, v))
+
         rkeys = reverse.keys()
         for rkey in sorted(rkeys):
             rentries = reverse[rkey]
@@ -428,12 +429,6 @@ def getCatalog(self):
 def find_files(dir, pattern, exclude=()):
     files = []
 
-    # def visit(files, dirname, names):
-    #     names[:] = filter(lambda x:x not in exclude, names)
-    #     files += [os.path.join(dirname, name)
-    #               for name in fnmatch.filter(names, pattern)
-    #               if name not in exclude]
-
     for dirpath, _dirnames, filenames in os.walk(dir):
         files += [os.path.join(dirpath, name)
                   for name in fnmatch.filter(filenames, pattern)

src/zope/app/locales/pygettext.py

@@ -146,11 +146,11 @@
 import tokenize
 
 
-if not hasattr(tokenize, 'generate_tokens'):
+try:
+    from tokenize import generate_tokens
+except ImportError:
     # The function was renamed in Python 3
-    generate_tokens = tokenize.tokenize
-else:
-    generate_tokens = tokenize.generate_tokens
+    from tokenize import tokenize as generate_tokens
 
 
 # for selftesting
@@ -264,9 +264,6 @@ def __init__(self, options):
 
     def __call__(self, ttype, tstring, stup, etup, line):
         # dispatch
-##        import token
-##        print(\, file=sys.stderr, 'ttype:', token.tok_name[ttype])
-##              'tstring:', tstring
         self.__state(ttype, tstring, stup[0])
 
     def __waiting(self, ttype, tstring, lineno):
@@ -363,8 +360,8 @@ def write(self, fp):
                 # k is the message string, v is a dictionary-set of (filename,
                 # lineno) tuples.  We want to sort the entries in v first by
                 # file name and then by line number.
-                v = list(v.keys())
-                v.sort()
+                v = sorted(v.keys())
+
                 if not options.writelocations:
                     pass
                 # location comments are different b/w Solaris and GNU:
@@ -476,15 +473,12 @@ class Options(object):
         elif opt in ('-x', '--exclude-file'):
             options.excludefilename = arg
         elif opt in ('-X', '--no-docstrings'):
-            fp = open(arg)
-            try:
+            with open(arg) as fp:
                 while 1:
                     line = fp.readline()
                     if not line:
                         break
                     options.nodocstrings[line[:-1]] = 1
-            finally:
-                fp.close()
 
     # calculate escapes
     make_escapes(options.escape)
@@ -495,9 +489,8 @@ class Options(object):
     # initialize list of strings to exclude
     if options.excludefilename:
         try:
-            fp = open(options.excludefilename)
-            options.toexclude = fp.readlines()
-            fp.close()
+            with open(options.excludefilename) as fp:
+                options.toexclude = fp.readlines()
         except IOError:
             print(_(
                 "Can't read --exclude-file: %s") % options.excludefilename,

src/zope/app/locales/tests.py

@@ -203,7 +203,7 @@ def doctest_POTMaker_write():
         >>> print(pot)
         ##############################################################################
         #
-        # Copyright (c) 2003-2004 Zope Foundation and Contributors.
+        # Copyright (c) 2003-2017 Zope Foundation and Contributors.
         # All Rights Reserved.
         #
         # This software is subject to the provisions of the Zope Public License,
@@ -313,8 +313,11 @@ def main(self, argv):
         main(argv)
 
     def test_extract(self):
+        me = __file__
+        if me.endswith(".pyc"):
+            me = me[:-1]
         out, _err = self.run_patched(['-d', 'TESTDOMAIN',
-                                      '-v', '-a', '-D', '-o', '-', __file__])
+                                      '-v', '-a', '-D', '-o', '-', me])
         self.assertIn('POT-Creation-Date', out.getvalue())