From eac2b826a127e80617b09d761e31750b0f383297 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Sat, 4 Nov 2017 21:28:46 -0700 Subject: [PATCH 1/6] Add a test for lists, switch div in trivial_inference --- .../python/apache_beam/typehints/trivial_inference.py | 9 +++++++-- .../apache_beam/typehints/trivial_inference_test.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index a68bd18b1c3e..e6634084ab4e 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -21,8 +21,13 @@ """ from __future__ import absolute_import from __future__ import print_function +from __future__ import division -import __builtin__ +from builtins import zip +from builtins import str +from past.utils import old_div +from builtins import object +import builtins import collections import dis import pprint @@ -344,7 +349,7 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): opname = dis.opname[op] jmp = jmp_state = None if opname.startswith('CALL_FUNCTION'): - standard_args = (arg & 0xF) + (arg & 0xF0) / 8 + standard_args = (arg & 0xF) + old_div((arg & 0xF0), 8) var_args = 'VAR' in opname kw_args = 'KW' in opname pop_count = standard_args + var_args + kw_args + 1 diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index 37b22584723a..9e2df8325b1b 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -72,6 +72,17 @@ def func(a): return None self.assertReturnType(typehints.Union[int, type(None)], func, [int]) + def testSimpleList(self): + self.assertReturnType( + typehints.List[int], + lambda xs: list([1, 2]), + [typehints.Tuple[int, ...]]) + + self.assertReturnType( + typehints.List[int], + lambda xs: list(xs), + [typehints.Tuple[int, ...]]) + def testListComprehension(self): self.assertReturnType( typehints.List[int], From 3b63c6e87e26519b085161a2bf2fbcf31183e56c Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 21 Aug 2017 01:18:57 -0700 Subject: [PATCH 2/6] Try and fix some of the type inferance, list generators are kind of not going to work super well as is. --- .../typehints/trivial_inference.py | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index e6634084ab4e..29f8da94faf9 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -299,8 +299,13 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): yields = set() returns = set() # TODO(robertwb): Default args via inspect module. - local_vars = list(input_types) + [typehints.Union[()]] * (len(co.co_varnames) - - len(input_types)) + try: + input_types_list = list(input_types) + except TypeError: + input_types_list = [input_types] + typehints_union = [typehints.Union[()]] + target_length = len(co.co_varnames) - len(input_types_list) + local_vars = input_types_list + typehints_union * target_length state = FrameState(f, local_vars) states = collections.defaultdict(lambda: None) jumps = collections.defaultdict(int) @@ -308,7 +313,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): last_pc = -1 while pc < end: start = pc - op = ord(code[pc]) + if sys.version_info[0] >= 3: + op = code[pc] + else: + op = ord(code[pc]) if debug: print('-->' if pc == last_pc else ' ', end=' ') @@ -316,7 +324,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): print(dis.opname[op].ljust(20), end=' ') pc += 1 if op >= dis.HAVE_ARGUMENT: - arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg + if sys.version_info[0] >= 3: + arg = code[pc] + code[pc + 1] * 256 + extended_arg + else: + arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg extended_arg = 0 pc += 2 if op == dis.EXTENDED_ARG: @@ -358,6 +369,10 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): elif arg & 0xF0: # TODO(robertwb): Handle this case. return_type = Any + elif (isinstance(state.stack[-pop_count], Const) and + state.stack[-pop_count].value == list): + # TODO(robertwb + holden): Handle this better. + return_type = typehints.List[element_type(state.stack[1])] elif isinstance(state.stack[-pop_count], Const): # TODO(robertwb): Handle this better. if var_args or kw_args: @@ -413,6 +428,11 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): jmp_state = state.copy() jmp_state.stack.pop() state.stack.append(element_type(state.stack[-1])) + elif opname == 'BUILD_LIST': + jmp = pc + arg + jmp_state = state.copy() + jmp_state.stack.pop() + state.stack.append(element_type(state.stack[-1])) else: raise TypeInferenceError('unable to handle %s' % opname) From 141e1a4b46857c98e38c12c8e886051f95be1338 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 21 Nov 2017 13:28:13 -0800 Subject: [PATCH 3/6] Skip list inference test in py2 --- sdks/python/apache_beam/typehints/trivial_inference.py | 4 ++-- sdks/python/apache_beam/typehints/trivial_inference_test.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index 29f8da94faf9..1aa1069c4c61 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -138,8 +138,8 @@ def get_global(self, i): name = self.get_name(i) if name in self.f.__globals__: return Const(self.f.__globals__[name]) - if name in __builtin__.__dict__: - return Const(__builtin__.__dict__[name]) + if name in builtins.__dict__: + return Const(builtins.__dict__[name]) return Any def get_name(self, i): diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index 9e2df8325b1b..98027dcc9b20 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -16,6 +16,7 @@ # """Tests for apache_beam.typehints.trivial_inference.""" +import sys import unittest from apache_beam.typehints import trivial_inference @@ -72,6 +73,7 @@ def func(a): return None self.assertReturnType(typehints.Union[int, type(None)], func, [int]) + @unittest.skipIf(sys.version_info[0] < 3, "List inference test py3 only") def testSimpleList(self): self.assertReturnType( typehints.List[int], From 39b21bfa22bf8fbf5107bdff354252be3ca75cd2 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 28 Nov 2017 01:40:15 -0800 Subject: [PATCH 4/6] Try and fix method lookup for py3 --- sdks/python/apache_beam/typehints/opcodes.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/typehints/opcodes.py b/sdks/python/apache_beam/typehints/opcodes.py index dcca6d026441..76dc25c41c3f 100644 --- a/sdks/python/apache_beam/typehints/opcodes.py +++ b/sdks/python/apache_beam/typehints/opcodes.py @@ -30,6 +30,7 @@ import types from functools import reduce +import inspect from . import typehints from .trivial_inference import BoundMethod @@ -265,9 +266,12 @@ def load_attr(state, arg): name = state.get_name(arg) if isinstance(o, Const) and hasattr(o.value, name): state.stack.append(Const(getattr(o.value, name))) - elif (isinstance(o, type) - and isinstance(getattr(o, name, None), types.MethodType)): - state.stack.append(Const(BoundMethod(getattr(o, name)))) + elif ((isinstance(o, type) or inspect.isclass(o))): + elem = getattr(o, name, None) + if callable(elem): + state.stack.append(Const(BoundMethod(elem))) + else: + state.stack.append(Any) else: state.stack.append(Any) From 385cafdd94694902edf5a989ab9a1040d425691c Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 30 Nov 2017 14:29:29 -0800 Subject: [PATCH 5/6] Works, w/debug --- .../typehints/trivial_inference.py | 43 ++++++++++++++++--- .../typehints/trivial_inference_test.py | 17 +++++--- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index 1aa1069c4c61..d495f01e83e8 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -182,10 +182,14 @@ def union(a, b): def element_type(hint): """Returns the element type of a composite type. """ + print("input hint %s " % hint) hint = Const.unwrap(hint) + print("unwrapped %s " % hint) if isinstance(hint, typehints.SequenceTypeConstraint): + print("sequence type") return hint.inner_type elif isinstance(hint, typehints.TupleHint.TupleConstraint): + print("tuple constraint") return typehints.Union[hint.tuple_types] return Any @@ -220,7 +224,7 @@ def hashable(c): return False -def infer_return_type(c, input_types, debug=False, depth=5): +def infer_return_type(c, input_types, debug=True, depth=5): """Analyses a callable to deduce its return type. Args: @@ -233,27 +237,37 @@ def infer_return_type(c, input_types, debug=False, depth=5): A TypeConstraint that that the return value of this function will (likely) satisfy given the specified inputs. """ + print("\nhi!\n") + print("infering type %s" % c) try: if hashable(c) and c in known_return_types: return known_return_types[c] elif isinstance(c, types.FunctionType): + print("I'm a function!\n") return infer_return_type_func(c, input_types, debug, depth) elif isinstance(c, types.MethodType): if c.__self__ is not None: input_types = [Const(c.__self__)] + input_types return infer_return_type_func(c.__func__, input_types, debug, depth) elif isinstance(c, BoundMethod): - input_types = [c.unbound.__self__.__class__] + input_types + unbound = c.unbound + print("unbound is %s %s %s" % (unbound, type(unbound), dir(unbound))) + # Python 2/3 compatibility. + method_class = getattr(c.unbound, "__self__", c.unbound).__class__ + func = getattr(c.unbound, "__func__", c.unbound) + input_types = [method_class] + input_types return infer_return_type_func( - c.unbound.__func__, input_types, debug, depth) + func, input_types, debug, depth) elif isinstance(c, type): if c in typehints.DISALLOWED_PRIMITIVE_TYPES: + print("Disallowed primitive type %s" % c) return { list: typehints.List[Any], set: typehints.Set[Any], tuple: typehints.Tuple[Any, ...], dict: typehints.Dict[Any, Any] }[c] + print("Allowed primitive time %s" % c) return c else: return Any @@ -267,7 +281,7 @@ def infer_return_type(c, input_types, debug=False, depth=5): return Any -def infer_return_type_func(f, input_types, debug=False, depth=0): +def infer_return_type_func(f, input_types, debug=True, depth=0): """Analyses a function to deduce its return type. Args: @@ -283,6 +297,7 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): Raises: TypeInferenceError: if no type can be inferred. """ + debug = True if debug: print() print(f, id(f), input_types) @@ -359,21 +374,28 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): opname = dis.opname[op] jmp = jmp_state = None + print("Opname %s" % opname) if opname.startswith('CALL_FUNCTION'): + print("Calling function!") standard_args = (arg & 0xF) + old_div((arg & 0xF0), 8) var_args = 'VAR' in opname kw_args = 'KW' in opname pop_count = standard_args + var_args + kw_args + 1 + print("%s, %s, %s, %s" % (standard_args, var_args, kw_args, pop_count)) if depth <= 0: + print("Depth 0") return_type = Any elif arg & 0xF0: + print("idk wtf this is") # TODO(robertwb): Handle this case. return_type = Any elif (isinstance(state.stack[-pop_count], Const) and state.stack[-pop_count].value == list): + print("oook?") # TODO(robertwb + holden): Handle this better. return_type = typehints.List[element_type(state.stack[1])] elif isinstance(state.stack[-pop_count], Const): + print("boople") # TODO(robertwb): Handle this better. if var_args or kw_args: state.stack[-1] = Any @@ -383,7 +405,12 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): debug=debug, depth=depth - 1) else: + print("boop boop") + print("aww yeah? arg is %s and stack minus pop count is %s" % + (arg, state.stack[-pop_count])) + print("v%s" % type(state.stack[-pop_count])) return_type = Any + print("mok?") state.stack[-pop_count:] = [return_type] elif (opname == 'BINARY_SUBSCR' and isinstance(state.stack[1], Const) @@ -400,7 +427,8 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): state.stack.append(Any) elif opname in simple_ops: if debug: - print("Executing simple op " + opname) + print("Executing simple %s op with state %s and arg %s" % + (opname, state, arg)) simple_ops[opname](state, arg) elif opname == 'RETURN_VALUE': returns.add(state.stack[-1]) @@ -420,23 +448,28 @@ def infer_return_type_func(f, input_types, debug=False, depth=0): jmp = arg jmp_state = state.copy() elif opname in ('JUMP_IF_TRUE_OR_POP', 'JUMP_IF_FALSE_OR_POP'): + print("jumping...") jmp = arg jmp_state = state.copy() state.stack.pop() elif opname == 'FOR_ITER': + print("for iter") jmp = pc + arg jmp_state = state.copy() jmp_state.stack.pop() state.stack.append(element_type(state.stack[-1])) elif opname == 'BUILD_LIST': + print("building list") jmp = pc + arg jmp_state = state.copy() jmp_state.stack.pop() state.stack.append(element_type(state.stack[-1])) else: + print("fail") raise TypeInferenceError('unable to handle %s' % opname) if jmp is not None: + print("buttons") # TODO(robertwb): Is this guerenteed to converge? new_state = states[jmp] | jmp_state if jmp < pc and new_state != states[jmp] and jumps[pc] < 5: diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py index 98027dcc9b20..ab031833e40e 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference_test.py +++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py @@ -77,13 +77,9 @@ def func(a): def testSimpleList(self): self.assertReturnType( typehints.List[int], - lambda xs: list([1, 2]), + lambda xs: [1, 2], [typehints.Tuple[int, ...]]) - self.assertReturnType( - typehints.List[int], - lambda xs: list(xs), - [typehints.Tuple[int, ...]]) def testListComprehension(self): self.assertReturnType( @@ -150,6 +146,17 @@ def m(self, x): self.assertReturnType(int, lambda: A().m(3)) self.assertReturnType(float, lambda: A.m(A(), 3.0)) + def testMethod2(self): + + class A(object): + + def m(self, x, y): + return x + + self.assertReturnType(int, lambda: A().m(3, "a")) + self.assertReturnType(float, lambda: A.m(A(), 3.0, "e")) + + def testAlwaysReturnsEarly(self): def some_fn(v): From e91bfb65108426d81946159b9d83ee87c3d1c420 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 30 Nov 2017 16:04:02 -0800 Subject: [PATCH 6/6] Works after removing debugging --- .../typehints/trivial_inference.py | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py index d495f01e83e8..ce3b9647602f 100644 --- a/sdks/python/apache_beam/typehints/trivial_inference.py +++ b/sdks/python/apache_beam/typehints/trivial_inference.py @@ -182,14 +182,10 @@ def union(a, b): def element_type(hint): """Returns the element type of a composite type. """ - print("input hint %s " % hint) hint = Const.unwrap(hint) - print("unwrapped %s " % hint) if isinstance(hint, typehints.SequenceTypeConstraint): - print("sequence type") return hint.inner_type elif isinstance(hint, typehints.TupleHint.TupleConstraint): - print("tuple constraint") return typehints.Union[hint.tuple_types] return Any @@ -224,7 +220,7 @@ def hashable(c): return False -def infer_return_type(c, input_types, debug=True, depth=5): +def infer_return_type(c, input_types, debug=False, depth=5): """Analyses a callable to deduce its return type. Args: @@ -237,13 +233,10 @@ def infer_return_type(c, input_types, debug=True, depth=5): A TypeConstraint that that the return value of this function will (likely) satisfy given the specified inputs. """ - print("\nhi!\n") - print("infering type %s" % c) try: if hashable(c) and c in known_return_types: return known_return_types[c] elif isinstance(c, types.FunctionType): - print("I'm a function!\n") return infer_return_type_func(c, input_types, debug, depth) elif isinstance(c, types.MethodType): if c.__self__ is not None: @@ -251,7 +244,6 @@ def infer_return_type(c, input_types, debug=True, depth=5): return infer_return_type_func(c.__func__, input_types, debug, depth) elif isinstance(c, BoundMethod): unbound = c.unbound - print("unbound is %s %s %s" % (unbound, type(unbound), dir(unbound))) # Python 2/3 compatibility. method_class = getattr(c.unbound, "__self__", c.unbound).__class__ func = getattr(c.unbound, "__func__", c.unbound) @@ -260,14 +252,12 @@ def infer_return_type(c, input_types, debug=True, depth=5): func, input_types, debug, depth) elif isinstance(c, type): if c in typehints.DISALLOWED_PRIMITIVE_TYPES: - print("Disallowed primitive type %s" % c) return { list: typehints.List[Any], set: typehints.Set[Any], tuple: typehints.Tuple[Any, ...], dict: typehints.Dict[Any, Any] }[c] - print("Allowed primitive time %s" % c) return c else: return Any @@ -281,7 +271,7 @@ def infer_return_type(c, input_types, debug=True, depth=5): return Any -def infer_return_type_func(f, input_types, debug=True, depth=0): +def infer_return_type_func(f, input_types, debug=False, depth=0): """Analyses a function to deduce its return type. Args: @@ -297,7 +287,6 @@ def infer_return_type_func(f, input_types, debug=True, depth=0): Raises: TypeInferenceError: if no type can be inferred. """ - debug = True if debug: print() print(f, id(f), input_types) @@ -374,28 +363,21 @@ def infer_return_type_func(f, input_types, debug=True, depth=0): opname = dis.opname[op] jmp = jmp_state = None - print("Opname %s" % opname) if opname.startswith('CALL_FUNCTION'): - print("Calling function!") standard_args = (arg & 0xF) + old_div((arg & 0xF0), 8) var_args = 'VAR' in opname kw_args = 'KW' in opname pop_count = standard_args + var_args + kw_args + 1 - print("%s, %s, %s, %s" % (standard_args, var_args, kw_args, pop_count)) if depth <= 0: - print("Depth 0") return_type = Any elif arg & 0xF0: - print("idk wtf this is") # TODO(robertwb): Handle this case. return_type = Any elif (isinstance(state.stack[-pop_count], Const) and state.stack[-pop_count].value == list): - print("oook?") # TODO(robertwb + holden): Handle this better. return_type = typehints.List[element_type(state.stack[1])] elif isinstance(state.stack[-pop_count], Const): - print("boople") # TODO(robertwb): Handle this better. if var_args or kw_args: state.stack[-1] = Any @@ -405,12 +387,7 @@ def infer_return_type_func(f, input_types, debug=True, depth=0): debug=debug, depth=depth - 1) else: - print("boop boop") - print("aww yeah? arg is %s and stack minus pop count is %s" % - (arg, state.stack[-pop_count])) - print("v%s" % type(state.stack[-pop_count])) return_type = Any - print("mok?") state.stack[-pop_count:] = [return_type] elif (opname == 'BINARY_SUBSCR' and isinstance(state.stack[1], Const) @@ -448,28 +425,23 @@ def infer_return_type_func(f, input_types, debug=True, depth=0): jmp = arg jmp_state = state.copy() elif opname in ('JUMP_IF_TRUE_OR_POP', 'JUMP_IF_FALSE_OR_POP'): - print("jumping...") jmp = arg jmp_state = state.copy() state.stack.pop() elif opname == 'FOR_ITER': - print("for iter") jmp = pc + arg jmp_state = state.copy() jmp_state.stack.pop() state.stack.append(element_type(state.stack[-1])) elif opname == 'BUILD_LIST': - print("building list") jmp = pc + arg jmp_state = state.copy() jmp_state.stack.pop() state.stack.append(element_type(state.stack[-1])) else: - print("fail") raise TypeInferenceError('unable to handle %s' % opname) if jmp is not None: - print("buttons") # TODO(robertwb): Is this guerenteed to converge? new_state = states[jmp] | jmp_state if jmp < pc and new_state != states[jmp] and jumps[pc] < 5: