bpo-43014: Improve performance of tokenize.tokenize by 20-30%

python · Jan 24, 2021 · 15bd9ef · 15bd9ef
1 parent bf9239b
commit 15bd9ef
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
@@ -27,6 +27,7 @@
 from builtins import open as _builtin_open
 from codecs import lookup, BOM_UTF8
 import collections
+import functools
 from io import TextIOWrapper
 import itertools as _itertools
 import re
@@ -95,6 +96,7 @@ def _all_string_prefixes():
                 result.add(''.join(u))
     return result
 
+@functools.lru_cache
 def _compile(expr):
     return re.compile(expr, re.UNICODE)
 

diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
@@ -0,0 +1 @@
+Improve performance of :mod:`tokenize` by 20-30%.  Patch by Anthony Sottile.