Move some global variables in parser into thread local (#264)

thriftpy2/parser/__init__.py

@@ -13,7 +13,7 @@
 import sys
 import types
 
-from .parser import parse, parse_fp, incomplete_type, _cast
+from .parser import parse, parse_fp, threadlocal, _cast
 from .exc import ThriftParserError
 from ..thrift import TPayloadMeta
 
@@ -35,7 +35,7 @@ def load(path,
     real_module = bool(module_name)
     thrift = parse(path, module_name, include_dirs=include_dirs,
                    include_dir=include_dir, encoding=encoding)
-    if incomplete_type:
+    if threadlocal.incomplete_type:
         fill_incomplete_ttype(thrift, thrift)
 
     # add sub modules to sys.modules recursively
@@ -58,18 +58,18 @@ def fill_incomplete_ttype(tmodule, definition):
         # construct const value
         if definition[0] == 'UNKNOWN_CONST':
             ttype = get_definition(
-                tmodule, incomplete_type[definition[1]][0], definition[3])
+                tmodule, threadlocal.incomplete_type[definition[1]][0], definition[3])
             return _cast(ttype)(definition[2])
         # construct incomplete alias type
-        elif definition[1] in incomplete_type:
+        elif definition[1] in threadlocal.incomplete_type:
             return (
                 definition[0],
-                get_definition(tmodule, *incomplete_type[definition[1]])
+                get_definition(tmodule, *threadlocal.incomplete_type[definition[1]])
             )
         # construct incomplete type which is contained in service method's args
-        elif definition[0] in incomplete_type:
+        elif definition[0] in threadlocal.incomplete_type:
             real_type = get_definition(
-                tmodule, *incomplete_type[definition[0]]
+                tmodule, *threadlocal.incomplete_type[definition[0]]
             )
             return (real_type[0], definition[1], real_type[1], definition[2])
         # construct incomplete compound type
@@ -88,10 +88,10 @@ def fill_incomplete_ttype(tmodule, definition):
     elif isinstance(definition, TPayloadMeta):
         for index, value in definition.thrift_spec.items():
             # if the ttype of the field is a single type and it is incompleted
-            if value[0] in incomplete_type:
+            if value[0] in threadlocal.incomplete_type:
                 real_type = fill_incomplete_ttype(
                     tmodule, get_definition(
-                        tmodule, *incomplete_type[value[0]]
+                        tmodule, *threadlocal.incomplete_type[value[0]]
                     )
                 )
                 # if the incomplete ttype is a compound type
@@ -107,19 +107,19 @@ def fill_incomplete_ttype(tmodule, definition):
                     definition.thrift_spec[index] = (
                         fill_incomplete_ttype(
                             tmodule, get_definition(
-                                tmodule, *incomplete_type[value[0]]
+                                tmodule, *threadlocal.incomplete_type[value[0]]
                             )
                         ),
                     ) + tuple(value[1:])
             # if the field's ttype is a compound type
             # and it contains incomplete types
-            elif value[2] in incomplete_type:
+            elif value[2] in threadlocal.incomplete_type:
                 definition.thrift_spec[index] = (
                     value[0],
                     value[1],
                     fill_incomplete_ttype(
                         tmodule, get_definition(
-                            tmodule, *incomplete_type[value[2]]
+                            tmodule, *threadlocal.incomplete_type[value[2]]
                         )
                     ),
                     value[3])
@@ -129,8 +129,8 @@ def fill_incomplete_ttype(tmodule, definition):
                 def walk(part):
                     if isinstance(part, tuple):
                         return tuple(walk(x) for x in part)
-                    if part in incomplete_type:
-                        return get_definition(tmodule, *incomplete_type[part])
+                    if part in threadlocal.incomplete_type:
+                        return get_definition(tmodule, *threadlocal.incomplete_type[part])
                     return part
                 definition.thrift_spec[index] = (
                     value[0],
@@ -158,7 +158,7 @@ def get_definition(thrift, name, lineno):
                                     (name, lineno))
         if isinstance(ref_type, int) and ref_type < 0:
             raise ThriftParserError('No type found: %r, at line %d' %
-                                    incomplete_type[ref_type])
+                                    threadlocal.incomplete_type[ref_type])
         if hasattr(ref_type, '_ttype'):
             return (getattr(ref_type, '_ttype'), ref_type)
         else:

thriftpy2/parser/parser.py

@@ -9,6 +9,7 @@
 
 import collections
 import os
+import threading
 import types
 from urllib.parse import urlparse
 from urllib.request import urlopen
@@ -20,6 +21,9 @@
 from .lexer import *  # noqa
 
 
+threadlocal = threading.local()
+
+
 def p_error(p):
     if p is None:
         raise ThriftGrammarError('Grammar error at EOF')
@@ -49,12 +53,12 @@ def p_header_unit(p):
 
 def p_include(p):
     '''include : INCLUDE LITERAL'''
-    thrift = thrift_stack[-1]
+    thrift = threadlocal.thrift_stack[-1]
     if thrift.__thrift_file__ is None:
         raise ThriftParserError('Unexpected include statement while loading '
                                 'from file like object.')
     replace_include_dirs = [os.path.dirname(thrift.__thrift_file__)] \
-        + include_dirs_
+        + threadlocal.include_dirs_
     for include_dir in replace_include_dirs:
         path = os.path.join(include_dir, p[2])
         if os.path.exists(path):
@@ -74,7 +78,7 @@ def p_namespace(p):
     '''namespace : NAMESPACE namespace_scope IDENTIFIER'''
     # namespace is useless in thriftpy2
     # if p[2] == 'py' or p[2] == '*':
-    #     setattr(thrift_stack[-1], '__name__', p[3])
+    #     setattr(threadlocal.thrift_stack[-1], '__name__', p[3])
 
 
 def p_namespace_scope(p):
@@ -114,7 +118,7 @@ def p_const(p):
     except AssertionError:
         raise ThriftParserError('Type error for constant %s at line %d' %
                                 (p[3], p.lineno(3)))
-    setattr(thrift_stack[-1], p[3], val)
+    setattr(threadlocal.thrift_stack[-1], p[3], val)
     _add_thrift_meta('consts', val)
 
 
@@ -160,7 +164,7 @@ def p_const_map_item(p):
 
 def p_const_ref(p):
     '''const_ref : IDENTIFIER'''
-    child = thrift_stack[-1]
+    child = threadlocal.thrift_stack[-1]
     for name in p[1].split('.'):
         father = child
         child = getattr(child, name, None)
@@ -187,13 +191,13 @@ def p_ttype(p):
 
 def p_typedef(p):
     '''typedef : TYPEDEF field_type IDENTIFIER type_annotations'''
-    setattr(thrift_stack[-1], p[3], p[2])
+    setattr(threadlocal.thrift_stack[-1], p[3], p[2])
 
 
 def p_enum(p):  # noqa
     '''enum : ENUM IDENTIFIER '{' enum_seq '}' type_annotations'''
     val = _make_enum(p[2], p[4])
-    setattr(thrift_stack[-1], p[2], val)
+    setattr(threadlocal.thrift_stack[-1], p[2], val)
     _add_thrift_meta('enums', val)
 
 
@@ -223,7 +227,7 @@ def p_struct(p):
 def p_seen_struct(p):
     '''seen_struct : STRUCT IDENTIFIER '''
     val = _make_empty_struct(p[2])
-    setattr(thrift_stack[-1], p[2], val)
+    setattr(threadlocal.thrift_stack[-1], p[2], val)
     p[0] = val
 
 
@@ -236,22 +240,22 @@ def p_union(p):
 def p_seen_union(p):
     '''seen_union : UNION IDENTIFIER '''
     val = _make_empty_struct(p[2])
-    setattr(thrift_stack[-1], p[2], val)
+    setattr(threadlocal.thrift_stack[-1], p[2], val)
     p[0] = val
 
 
 def p_exception(p):
     '''exception : EXCEPTION IDENTIFIER '{' field_seq '}' type_annotations '''
     val = _make_struct(p[2], p[4], base_cls=TException)
-    setattr(thrift_stack[-1], p[2], val)
+    setattr(threadlocal.thrift_stack[-1], p[2], val)
     _add_thrift_meta('exceptions', val)
 
 
 def p_simple_service(p):
     '''simple_service : SERVICE IDENTIFIER '{' function_seq '}'
                 | SERVICE IDENTIFIER EXTENDS IDENTIFIER '{' function_seq '}'
     '''
-    thrift = thrift_stack[-1]
+    thrift = threadlocal.thrift_stack[-1]
 
     if len(p) == 8:
         extends = thrift
@@ -386,12 +390,12 @@ def set_info(self, info):
         return self.index + 1
 
 
-incomplete_type = CurrentIncompleteType()
+threadlocal.incomplete_type = CurrentIncompleteType()
 
 
 def p_ref_type(p):
     '''ref_type : IDENTIFIER'''
-    ref_type = thrift_stack[-1]
+    ref_type = threadlocal.thrift_stack[-1]
 
     for attr in dir(ref_type):
         if attr in {'__doc__', '__loader__', '__name__', '__package__',
@@ -411,7 +415,7 @@ def p_ref_type(p):
                 if index != len(p[1].split('.')) - 1:
                     raise ThriftParserError('No type found: %r, at line %d' %
                                             (p[1], p.lineno(1)))
-                p[0] = incomplete_type.set_info((p[1], p.lineno(1)))
+                p[0] = threadlocal.incomplete_type.set_info((p[1], p.lineno(1)))
                 return
 
     if hasattr(ref_type, '_ttype'):
@@ -511,9 +515,9 @@ def p_type_annotation(p):
         p[0] = p[1], None  # Without Value
 
 
-thrift_stack = []
-include_dirs_ = ['.']
-thrift_cache = {}
+threadlocal.thrift_stack = []
+threadlocal.include_dirs_ = ['.']
+threadlocal.thrift_cache = {}
 
 
 def parse(path, module_name=None, include_dirs=None, include_dir=None,
@@ -540,29 +544,25 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None,
                          is provided, use it as cache key, else use the `path`.
     """
     # dead include checking on current stack
-    for thrift in thrift_stack:
+    for thrift in threadlocal.thrift_stack:
         if thrift.__thrift_file__ is not None and \
                 os.path.samefile(path, thrift.__thrift_file__):
             raise ThriftParserError('Dead including on %s' % path)
 
-    global thrift_cache
-
     cache_key = module_name or os.path.normpath(path)
 
-    if enable_cache and cache_key in thrift_cache:
-        return thrift_cache[cache_key]
+    if enable_cache and cache_key in threadlocal.thrift_cache:
+        return threadlocal.thrift_cache[cache_key]
 
     if lexer is None:
         lexer = lex.lex()
     if parser is None:
         parser = yacc.yacc(debug=False, write_tables=0)
 
-    global include_dirs_
-
     if include_dirs is not None:
-        include_dirs_ = include_dirs
+        threadlocal.include_dirs_ = include_dirs
     if include_dir is not None:
-        include_dirs_.append(include_dir)
+        threadlocal.include_dirs_.append(include_dir)
 
     if not path.endswith('.thrift'):
         raise ThriftParserError('Path should end with .thrift')
@@ -594,13 +594,13 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None,
 
     thrift = types.ModuleType(module_name)
     setattr(thrift, '__thrift_file__', path)
-    thrift_stack.append(thrift)
+    threadlocal.thrift_stack.append(thrift)
     lexer.lineno = 1
     parser.parse(data)
-    thrift_stack.pop()
+    threadlocal.thrift_stack.pop()
 
     if enable_cache:
-        thrift_cache[cache_key] = thrift
+        threadlocal.thrift_cache[cache_key] = thrift
     return thrift
 
 
@@ -624,8 +624,8 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True):
         raise ThriftParserError('thriftpy2 can only generate module with '
                                 '\'_thrift\' suffix')
 
-    if enable_cache and module_name in thrift_cache:
-        return thrift_cache[module_name]
+    if enable_cache and module_name in threadlocal.thrift_cache:
+        return threadlocal.thrift_cache[module_name]
 
     if not hasattr(source, 'read'):
         raise ThriftParserError('Expected `source` to be a file-like object '
@@ -640,18 +640,18 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True):
 
     thrift = types.ModuleType(module_name)
     setattr(thrift, '__thrift_file__', None)
-    thrift_stack.append(thrift)
+    threadlocal.thrift_stack.append(thrift)
     lexer.lineno = 1
     parser.parse(data)
-    thrift_stack.pop()
+    threadlocal.thrift_stack.pop()
 
     if enable_cache:
-        thrift_cache[module_name] = thrift
+        threadlocal.thrift_cache[module_name] = thrift
     return thrift
 
 
 def _add_thrift_meta(key, val):
-    thrift = thrift_stack[-1]
+    thrift = threadlocal.thrift_stack[-1]
 
     if not hasattr(thrift, '__thrift_meta__'):
         meta = collections.defaultdict(list)
@@ -827,7 +827,10 @@ def __cast_struct(v):
 
 
 def _make_enum(name, kvs):
-    attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': TType.I32}
+    attrs = {
+        '__module__': threadlocal.thrift_stack[-1].__name__,
+        '_ttype': TType.I32
+    }
     cls = type(name, (object, ), attrs)
 
     _values_to_names = {}
@@ -851,7 +854,10 @@ def _make_enum(name, kvs):
 
 
 def _make_empty_struct(name, ttype=TType.STRUCT, base_cls=TPayload):
-    attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': ttype}
+    attrs = {
+        '__module__': threadlocal.thrift_stack[-1].__name__,
+        '_ttype': ttype
+    }
     return type(name, (base_cls, ), attrs)
 
 
@@ -887,7 +893,7 @@ def _make_service(name, funcs, extends):
     if extends is None:
         extends = object
 
-    attrs = {'__module__': thrift_stack[-1].__name__}
+    attrs = {'__module__': threadlocal.thrift_stack[-1].__name__}
     cls = type(name, (extends, ), attrs)
     thrift_services = []