Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move some global variables in parser into thread local #264

Merged
merged 1 commit into from
May 7, 2024
Merged

Move some global variables in parser into thread local #264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions thriftpy2/parser/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import sys
import types

from .parser import parse, parse_fp, incomplete_type, _cast
from .parser import parse, parse_fp, threadlocal, _cast
from .exc import ThriftParserError
from ..thrift import TPayloadMeta

Expand All @@ -35,7 +35,7 @@ def load(path,
real_module = bool(module_name)
thrift = parse(path, module_name, include_dirs=include_dirs,
include_dir=include_dir, encoding=encoding)
if incomplete_type:
if threadlocal.incomplete_type:
fill_incomplete_ttype(thrift, thrift)

# add sub modules to sys.modules recursively
Expand All @@ -58,18 +58,18 @@ def fill_incomplete_ttype(tmodule, definition):
# construct const value
if definition[0] == 'UNKNOWN_CONST':
ttype = get_definition(
tmodule, incomplete_type[definition[1]][0], definition[3])
tmodule, threadlocal.incomplete_type[definition[1]][0], definition[3])
return _cast(ttype)(definition[2])
# construct incomplete alias type
elif definition[1] in incomplete_type:
elif definition[1] in threadlocal.incomplete_type:
return (
definition[0],
get_definition(tmodule, *incomplete_type[definition[1]])
get_definition(tmodule, *threadlocal.incomplete_type[definition[1]])
)
# construct incomplete type which is contained in service method's args
elif definition[0] in incomplete_type:
elif definition[0] in threadlocal.incomplete_type:
real_type = get_definition(
tmodule, *incomplete_type[definition[0]]
tmodule, *threadlocal.incomplete_type[definition[0]]
)
return (real_type[0], definition[1], real_type[1], definition[2])
# construct incomplete compound type
Expand All @@ -88,10 +88,10 @@ def fill_incomplete_ttype(tmodule, definition):
elif isinstance(definition, TPayloadMeta):
for index, value in definition.thrift_spec.items():
# if the ttype of the field is a single type and it is incompleted
if value[0] in incomplete_type:
if value[0] in threadlocal.incomplete_type:
real_type = fill_incomplete_ttype(
tmodule, get_definition(
tmodule, *incomplete_type[value[0]]
tmodule, *threadlocal.incomplete_type[value[0]]
)
)
# if the incomplete ttype is a compound type
Expand All @@ -107,19 +107,19 @@ def fill_incomplete_ttype(tmodule, definition):
definition.thrift_spec[index] = (
fill_incomplete_ttype(
tmodule, get_definition(
tmodule, *incomplete_type[value[0]]
tmodule, *threadlocal.incomplete_type[value[0]]
)
),
) + tuple(value[1:])
# if the field's ttype is a compound type
# and it contains incomplete types
elif value[2] in incomplete_type:
elif value[2] in threadlocal.incomplete_type:
definition.thrift_spec[index] = (
value[0],
value[1],
fill_incomplete_ttype(
tmodule, get_definition(
tmodule, *incomplete_type[value[2]]
tmodule, *threadlocal.incomplete_type[value[2]]
)
),
value[3])
Expand All @@ -129,8 +129,8 @@ def fill_incomplete_ttype(tmodule, definition):
def walk(part):
if isinstance(part, tuple):
return tuple(walk(x) for x in part)
if part in incomplete_type:
return get_definition(tmodule, *incomplete_type[part])
if part in threadlocal.incomplete_type:
return get_definition(tmodule, *threadlocal.incomplete_type[part])
return part
definition.thrift_spec[index] = (
value[0],
Expand Down Expand Up @@ -158,7 +158,7 @@ def get_definition(thrift, name, lineno):
(name, lineno))
if isinstance(ref_type, int) and ref_type < 0:
raise ThriftParserError('No type found: %r, at line %d' %
incomplete_type[ref_type])
threadlocal.incomplete_type[ref_type])
if hasattr(ref_type, '_ttype'):
return (getattr(ref_type, '_ttype'), ref_type)
else:
Expand Down
82 changes: 44 additions & 38 deletions thriftpy2/parser/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import collections
import os
import threading
import types
from urllib.parse import urlparse
from urllib.request import urlopen
Expand All @@ -20,6 +21,9 @@
from .lexer import * # noqa


threadlocal = threading.local()


def p_error(p):
if p is None:
raise ThriftGrammarError('Grammar error at EOF')
Expand Down Expand Up @@ -49,12 +53,12 @@ def p_header_unit(p):

def p_include(p):
'''include : INCLUDE LITERAL'''
thrift = thrift_stack[-1]
thrift = threadlocal.thrift_stack[-1]
if thrift.__thrift_file__ is None:
raise ThriftParserError('Unexpected include statement while loading '
'from file like object.')
replace_include_dirs = [os.path.dirname(thrift.__thrift_file__)] \
+ include_dirs_
+ threadlocal.include_dirs_
for include_dir in replace_include_dirs:
path = os.path.join(include_dir, p[2])
if os.path.exists(path):
Expand All @@ -74,7 +78,7 @@ def p_namespace(p):
'''namespace : NAMESPACE namespace_scope IDENTIFIER'''
# namespace is useless in thriftpy2
# if p[2] == 'py' or p[2] == '*':
# setattr(thrift_stack[-1], '__name__', p[3])
# setattr(threadlocal.thrift_stack[-1], '__name__', p[3])


def p_namespace_scope(p):
Expand Down Expand Up @@ -114,7 +118,7 @@ def p_const(p):
except AssertionError:
raise ThriftParserError('Type error for constant %s at line %d' %
(p[3], p.lineno(3)))
setattr(thrift_stack[-1], p[3], val)
setattr(threadlocal.thrift_stack[-1], p[3], val)
_add_thrift_meta('consts', val)


Expand Down Expand Up @@ -160,7 +164,7 @@ def p_const_map_item(p):

def p_const_ref(p):
'''const_ref : IDENTIFIER'''
child = thrift_stack[-1]
child = threadlocal.thrift_stack[-1]
for name in p[1].split('.'):
father = child
child = getattr(child, name, None)
Expand All @@ -187,13 +191,13 @@ def p_ttype(p):

def p_typedef(p):
'''typedef : TYPEDEF field_type IDENTIFIER type_annotations'''
setattr(thrift_stack[-1], p[3], p[2])
setattr(threadlocal.thrift_stack[-1], p[3], p[2])


def p_enum(p): # noqa
'''enum : ENUM IDENTIFIER '{' enum_seq '}' type_annotations'''
val = _make_enum(p[2], p[4])
setattr(thrift_stack[-1], p[2], val)
setattr(threadlocal.thrift_stack[-1], p[2], val)
_add_thrift_meta('enums', val)


Expand Down Expand Up @@ -223,7 +227,7 @@ def p_struct(p):
def p_seen_struct(p):
'''seen_struct : STRUCT IDENTIFIER '''
val = _make_empty_struct(p[2])
setattr(thrift_stack[-1], p[2], val)
setattr(threadlocal.thrift_stack[-1], p[2], val)
p[0] = val


Expand All @@ -236,22 +240,22 @@ def p_union(p):
def p_seen_union(p):
'''seen_union : UNION IDENTIFIER '''
val = _make_empty_struct(p[2])
setattr(thrift_stack[-1], p[2], val)
setattr(threadlocal.thrift_stack[-1], p[2], val)
p[0] = val


def p_exception(p):
'''exception : EXCEPTION IDENTIFIER '{' field_seq '}' type_annotations '''
val = _make_struct(p[2], p[4], base_cls=TException)
setattr(thrift_stack[-1], p[2], val)
setattr(threadlocal.thrift_stack[-1], p[2], val)
_add_thrift_meta('exceptions', val)


def p_simple_service(p):
'''simple_service : SERVICE IDENTIFIER '{' function_seq '}'
| SERVICE IDENTIFIER EXTENDS IDENTIFIER '{' function_seq '}'
'''
thrift = thrift_stack[-1]
thrift = threadlocal.thrift_stack[-1]

if len(p) == 8:
extends = thrift
Expand Down Expand Up @@ -386,12 +390,12 @@ def set_info(self, info):
return self.index + 1


incomplete_type = CurrentIncompleteType()
threadlocal.incomplete_type = CurrentIncompleteType()


def p_ref_type(p):
'''ref_type : IDENTIFIER'''
ref_type = thrift_stack[-1]
ref_type = threadlocal.thrift_stack[-1]

for attr in dir(ref_type):
if attr in {'__doc__', '__loader__', '__name__', '__package__',
Expand All @@ -411,7 +415,7 @@ def p_ref_type(p):
if index != len(p[1].split('.')) - 1:
raise ThriftParserError('No type found: %r, at line %d' %
(p[1], p.lineno(1)))
p[0] = incomplete_type.set_info((p[1], p.lineno(1)))
p[0] = threadlocal.incomplete_type.set_info((p[1], p.lineno(1)))
return

if hasattr(ref_type, '_ttype'):
Expand Down Expand Up @@ -511,9 +515,9 @@ def p_type_annotation(p):
p[0] = p[1], None # Without Value


thrift_stack = []
include_dirs_ = ['.']
thrift_cache = {}
threadlocal.thrift_stack = []
threadlocal.include_dirs_ = ['.']
threadlocal.thrift_cache = {}


def parse(path, module_name=None, include_dirs=None, include_dir=None,
Expand All @@ -540,29 +544,25 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None,
is provided, use it as cache key, else use the `path`.
"""
# dead include checking on current stack
for thrift in thrift_stack:
for thrift in threadlocal.thrift_stack:
if thrift.__thrift_file__ is not None and \
os.path.samefile(path, thrift.__thrift_file__):
raise ThriftParserError('Dead including on %s' % path)

global thrift_cache

cache_key = module_name or os.path.normpath(path)

if enable_cache and cache_key in thrift_cache:
return thrift_cache[cache_key]
if enable_cache and cache_key in threadlocal.thrift_cache:
return threadlocal.thrift_cache[cache_key]

if lexer is None:
lexer = lex.lex()
if parser is None:
parser = yacc.yacc(debug=False, write_tables=0)

global include_dirs_

if include_dirs is not None:
include_dirs_ = include_dirs
threadlocal.include_dirs_ = include_dirs
if include_dir is not None:
include_dirs_.append(include_dir)
threadlocal.include_dirs_.append(include_dir)

if not path.endswith('.thrift'):
raise ThriftParserError('Path should end with .thrift')
Expand Down Expand Up @@ -594,13 +594,13 @@ def parse(path, module_name=None, include_dirs=None, include_dir=None,

thrift = types.ModuleType(module_name)
setattr(thrift, '__thrift_file__', path)
thrift_stack.append(thrift)
threadlocal.thrift_stack.append(thrift)
lexer.lineno = 1
parser.parse(data)
thrift_stack.pop()
threadlocal.thrift_stack.pop()

if enable_cache:
thrift_cache[cache_key] = thrift
threadlocal.thrift_cache[cache_key] = thrift
return thrift


Expand All @@ -624,8 +624,8 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True):
raise ThriftParserError('thriftpy2 can only generate module with '
'\'_thrift\' suffix')

if enable_cache and module_name in thrift_cache:
return thrift_cache[module_name]
if enable_cache and module_name in threadlocal.thrift_cache:
return threadlocal.thrift_cache[module_name]

if not hasattr(source, 'read'):
raise ThriftParserError('Expected `source` to be a file-like object '
Expand All @@ -640,18 +640,18 @@ def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True):

thrift = types.ModuleType(module_name)
setattr(thrift, '__thrift_file__', None)
thrift_stack.append(thrift)
threadlocal.thrift_stack.append(thrift)
lexer.lineno = 1
parser.parse(data)
thrift_stack.pop()
threadlocal.thrift_stack.pop()

if enable_cache:
thrift_cache[module_name] = thrift
threadlocal.thrift_cache[module_name] = thrift
return thrift


def _add_thrift_meta(key, val):
thrift = thrift_stack[-1]
thrift = threadlocal.thrift_stack[-1]

if not hasattr(thrift, '__thrift_meta__'):
meta = collections.defaultdict(list)
Expand Down Expand Up @@ -827,7 +827,10 @@ def __cast_struct(v):


def _make_enum(name, kvs):
attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': TType.I32}
attrs = {
'__module__': threadlocal.thrift_stack[-1].__name__,
'_ttype': TType.I32
}
cls = type(name, (object, ), attrs)

_values_to_names = {}
Expand All @@ -851,7 +854,10 @@ def _make_enum(name, kvs):


def _make_empty_struct(name, ttype=TType.STRUCT, base_cls=TPayload):
attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': ttype}
attrs = {
'__module__': threadlocal.thrift_stack[-1].__name__,
'_ttype': ttype
}
return type(name, (base_cls, ), attrs)


Expand Down Expand Up @@ -887,7 +893,7 @@ def _make_service(name, funcs, extends):
if extends is None:
extends = object

attrs = {'__module__': thrift_stack[-1].__name__}
attrs = {'__module__': threadlocal.thrift_stack[-1].__name__}
cls = type(name, (extends, ), attrs)
thrift_services = []

Expand Down
Loading