diff --git a/validator/python/__init__.py b/validator/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/validator/python/copy.py b/validator/python/copy.py new file mode 100644 index 000000000..c227a2e5c --- /dev/null +++ b/validator/python/copy.py @@ -0,0 +1,433 @@ +"""Generic (shallow and deep) copying operations. + +Interface summary: + + import copy + + x = copy.copy(y) # make a shallow copy of y + x = copy.deepcopy(y) # make a deep copy of y + +For module specific errors, copy.Error is raised. + +The difference between shallow and deep copying is only relevant for +compound objects (objects that contain other objects, like lists or +class instances). + +- A shallow copy constructs a new compound object and then (to the + extent possible) inserts *the same objects* into it that the + original contains. + +- A deep copy constructs a new compound object and then, recursively, + inserts *copies* into it of the objects found in the original. + +Two problems often exist with deep copy operations that don't exist +with shallow copy operations: + + a) recursive objects (compound objects that, directly or indirectly, + contain a reference to themselves) may cause a recursive loop + + b) because deep copy copies *everything* it may copy too much, e.g. + administrative data structures that should be shared even between + copies + +Python's deep copy operation avoids these problems by: + + a) keeping a table of objects already copied during the current + copying pass + + b) letting user-defined classes override the copying operation or the + set of components copied + +This version does not copy types like module, class, function, method, +nor stack trace, stack frame, nor file, socket, window, nor array, nor +any similar types. + +Classes can use the same interfaces to control copying that they use +to control pickling: they can define methods called __getinitargs__(), +__getstate__() and __setstate__(). See the documentation for module +"pickle" for information on these methods. +""" + +import types +import weakref +from copy_reg import dispatch_table + +class Error(Exception): + pass +error = Error # backward compatibility + +try: + from org.python.core import PyStringMap +except ImportError: + PyStringMap = None + +__all__ = ["Error", "copy", "deepcopy"] + +def copy(x): + """Shallow copy operation on arbitrary Python objects. + + See the module's __doc__ string for more info. + """ + + cls = type(x) + + copier = _copy_dispatch.get(cls) + if copier: + return copier(x) + + copier = getattr(cls, "__copy__", None) + if copier: + return copier(x) + + reductor = dispatch_table.get(cls) + if reductor: + rv = reductor(x) + else: + reductor = getattr(x, "__reduce_ex__", None) + if reductor: + rv = reductor(2) + else: + reductor = getattr(x, "__reduce__", None) + if reductor: + rv = reductor() + else: + raise Error("un(shallow)copyable object of type %s" % cls) + + return _reconstruct(x, rv, 0) + + +_copy_dispatch = d = {} + +def _copy_immutable(x): + return x +for t in (type(None), int, long, float, bool, str, tuple, + frozenset, type, xrange, types.ClassType, + types.BuiltinFunctionType, type(Ellipsis), + types.FunctionType, weakref.ref): + d[t] = _copy_immutable +for name in ("ComplexType", "UnicodeType", "CodeType"): + t = getattr(types, name, None) + if t is not None: + d[t] = _copy_immutable + +def _copy_with_constructor(x): + return type(x)(x) +for t in (list, dict, set): + d[t] = _copy_with_constructor + +def _copy_with_copy_method(x): + return x.copy() +if PyStringMap is not None: + d[PyStringMap] = _copy_with_copy_method + +def _copy_inst(x): + if hasattr(x, '__copy__'): + return x.__copy__() + if hasattr(x, '__getinitargs__'): + args = x.__getinitargs__() + y = x.__class__(*args) + else: + y = _EmptyClass() + y.__class__ = x.__class__ + if hasattr(x, '__getstate__'): + state = x.__getstate__() + else: + state = x.__dict__ + if hasattr(y, '__setstate__'): + y.__setstate__(state) + else: + y.__dict__.update(state) + return y +d[types.InstanceType] = _copy_inst + +del d + +def deepcopy(x, memo=None, _nil=[]): + """Deep copy operation on arbitrary Python objects. + + See the module's __doc__ string for more info. + """ + + if memo is None: + memo = {} + + d = id(x) + y = memo.get(d, _nil) + if y is not _nil: + return y + + cls = type(x) + + copier = _deepcopy_dispatch.get(cls) + if copier: + y = copier(x, memo) + else: + try: + issc = issubclass(cls, type) + except TypeError: # cls is not a class (old Boost; see SF #502085) + issc = 0 + if issc: + y = _deepcopy_atomic(x, memo) + else: + copier = getattr(x, "__deepcopy__", None) + if copier: + y = copier(memo) + else: + reductor = dispatch_table.get(cls) + if reductor: + rv = reductor(x) + else: + reductor = getattr(x, "__reduce_ex__", None) + if reductor: + rv = reductor(2) + else: + reductor = getattr(x, "__reduce__", None) + if reductor: + rv = reductor() + else: + raise Error( + "un(deep)copyable object of type %s" % cls) + y = _reconstruct(x, rv, 1, memo) + + memo[d] = y + _keep_alive(x, memo) # Make sure x lives at least as long as d + return y + +_deepcopy_dispatch = d = {} + +def _deepcopy_atomic(x, memo): + return x +d[type(None)] = _deepcopy_atomic +d[type(Ellipsis)] = _deepcopy_atomic +d[int] = _deepcopy_atomic +d[long] = _deepcopy_atomic +d[float] = _deepcopy_atomic +d[bool] = _deepcopy_atomic +try: + d[complex] = _deepcopy_atomic +except NameError: + pass +d[str] = _deepcopy_atomic +try: + d[unicode] = _deepcopy_atomic +except NameError: + pass +try: + d[types.CodeType] = _deepcopy_atomic +except AttributeError: + pass +d[type] = _deepcopy_atomic +d[xrange] = _deepcopy_atomic +d[types.ClassType] = _deepcopy_atomic +d[types.BuiltinFunctionType] = _deepcopy_atomic +d[types.FunctionType] = _deepcopy_atomic +d[weakref.ref] = _deepcopy_atomic + +def _deepcopy_list(x, memo): + y = [] + memo[id(x)] = y + for a in x: + y.append(deepcopy(a, memo)) + return y +d[list] = _deepcopy_list + +def _deepcopy_tuple(x, memo): + y = [] + for a in x: + y.append(deepcopy(a, memo)) + d = id(x) + try: + return memo[d] + except KeyError: + pass + for i in range(len(x)): + if x[i] is not y[i]: + y = tuple(y) + break + else: + y = x + memo[d] = y + return y +d[tuple] = _deepcopy_tuple + +def _deepcopy_dict(x, memo): + y = {} + memo[id(x)] = y + for key, value in x.iteritems(): + y[deepcopy(key, memo)] = deepcopy(value, memo) + return y +d[dict] = _deepcopy_dict +if PyStringMap is not None: + d[PyStringMap] = _deepcopy_dict + +def _deepcopy_method(x, memo): # Copy instance methods + return type(x)(x.im_func, deepcopy(x.im_self, memo), x.im_class) +_deepcopy_dispatch[types.MethodType] = _deepcopy_method + +def _keep_alive(x, memo): + """Keeps a reference to the object x in the memo. + + Because we remember objects by their id, we have + to assure that possibly temporary objects are kept + alive by referencing them. + We store a reference at the id of the memo, which should + normally not be used unless someone tries to deepcopy + the memo itself... + """ + try: + memo[id(memo)].append(x) + except KeyError: + # aha, this is the first one :-) + memo[id(memo)]=[x] + +def _deepcopy_inst(x, memo): + if hasattr(x, '__deepcopy__'): + return x.__deepcopy__(memo) + if hasattr(x, '__getinitargs__'): + args = x.__getinitargs__() + args = deepcopy(args, memo) + y = x.__class__(*args) + else: + y = _EmptyClass() + y.__class__ = x.__class__ + memo[id(x)] = y + if hasattr(x, '__getstate__'): + state = x.__getstate__() + else: + state = x.__dict__ + state = deepcopy(state, memo) + if hasattr(y, '__setstate__'): + y.__setstate__(state) + else: + y.__dict__.update(state) + return y +d[types.InstanceType] = _deepcopy_inst + +def _reconstruct(x, info, deep, memo=None): + if isinstance(info, str): + return x + assert isinstance(info, tuple) + if memo is None: + memo = {} + n = len(info) + assert n in (2, 3, 4, 5) + callable, args = info[:2] + if n > 2: + state = info[2] + else: + state = {} + if n > 3: + listiter = info[3] + else: + listiter = None + if n > 4: + dictiter = info[4] + else: + dictiter = None + if deep: + args = deepcopy(args, memo) + y = callable(*args) + memo[id(x)] = y + + if state: + if deep: + state = deepcopy(state, memo) + if hasattr(y, '__setstate__'): + y.__setstate__(state) + else: + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + else: + slotstate = None + if state is not None: + y.__dict__.update(state) + if slotstate is not None: + for key, value in slotstate.iteritems(): + setattr(y, key, value) + + if listiter is not None: + for item in listiter: + if deep: + item = deepcopy(item, memo) + y.append(item) + if dictiter is not None: + for key, value in dictiter: + if deep: + key = deepcopy(key, memo) + value = deepcopy(value, memo) + y[key] = value + return y + +del d + +del types + +# Helper for instance creation without calling __init__ +class _EmptyClass: + pass + +def _test(): + l = [None, 1, 2L, 3.14, 'xyzzy', (1, 2L), [3.14, 'abc'], + {'abc': 'ABC'}, (), [], {}] + l1 = copy(l) + print l1==l + l1 = map(copy, l) + print l1==l + l1 = deepcopy(l) + print l1==l + class C: + def __init__(self, arg=None): + self.a = 1 + self.arg = arg + if __name__ == '__main__': + import sys + file = sys.argv[0] + else: + file = __file__ + self.fp = open(file) + self.fp.close() + def __getstate__(self): + return {'a': self.a, 'arg': self.arg} + def __setstate__(self, state): + for key, value in state.iteritems(): + setattr(self, key, value) + def __deepcopy__(self, memo=None): + new = self.__class__(deepcopy(self.arg, memo)) + new.a = self.a + return new + c = C('argument sketch') + l.append(c) + l2 = copy(l) + print l == l2 + print l + print l2 + l2 = deepcopy(l) + print l == l2 + print l + print l2 + l.append({l[1]: l, 'xyz': l[2]}) + l3 = copy(l) + import repr + print map(repr.repr, l) + print map(repr.repr, l1) + print map(repr.repr, l2) + print map(repr.repr, l3) + l3 = deepcopy(l) + import repr + print map(repr.repr, l) + print map(repr.repr, l1) + print map(repr.repr, l2) + print map(repr.repr, l3) + class odict(dict): + def __init__(self, d = {}): + self.a = 99 + dict.__init__(self, d) + def __setitem__(self, k, i): + dict.__setitem__(self, k, i) + self.a + o = odict({"A" : "B"}) + x = deepcopy(o) + print(o, x) + +if __name__ == '__main__': + _test() diff --git a/validator/python/copy_reg.py b/validator/python/copy_reg.py new file mode 100644 index 000000000..db1715092 --- /dev/null +++ b/validator/python/copy_reg.py @@ -0,0 +1,201 @@ +"""Helper to provide extensibility for pickle/cPickle. + +This is only useful to add pickle support for extension types defined in +C, not for instances of user-defined classes. +""" + +from types import ClassType as _ClassType + +__all__ = ["pickle", "constructor", + "add_extension", "remove_extension", "clear_extension_cache"] + +dispatch_table = {} + +def pickle(ob_type, pickle_function, constructor_ob=None): + if type(ob_type) is _ClassType: + raise TypeError("copy_reg is not intended for use with classes") + + if not hasattr(pickle_function, '__call__'): + raise TypeError("reduction functions must be callable") + dispatch_table[ob_type] = pickle_function + + # The constructor_ob function is a vestige of safe for unpickling. + # There is no reason for the caller to pass it anymore. + if constructor_ob is not None: + constructor(constructor_ob) + +def constructor(object): + if not hasattr(object, '__call__'): + raise TypeError("constructors must be callable") + +# Example: provide pickling support for complex numbers. + +try: + complex +except NameError: + pass +else: + + def pickle_complex(c): + return complex, (c.real, c.imag) + + pickle(complex, pickle_complex, complex) + +# Support for pickling new-style objects + +def _reconstructor(cls, base, state): + if base is object: + obj = object.__new__(cls) + else: + obj = base.__new__(cls, state) + if base.__init__ != object.__init__: + base.__init__(obj, state) + return obj + +_HEAPTYPE = 1<<9 + +# Python code for object.__reduce_ex__ for protocols 0 and 1 + +def _reduce_ex(self, proto): + assert proto < 2 + for base in self.__class__.__mro__: + if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: + break + else: + base = object # not really reachable + if base is object: + state = None + else: + if base is self.__class__: + raise TypeError, "can't pickle %s objects" % base.__name__ + state = base(self) + args = (self.__class__, base, state) + try: + getstate = self.__getstate__ + except AttributeError: + if getattr(self, "__slots__", None): + raise TypeError("a class that defines __slots__ without " + "defining __getstate__ cannot be pickled") + try: + dict = self.__dict__ + except AttributeError: + dict = None + else: + dict = getstate() + if dict: + return _reconstructor, args, dict + else: + return _reconstructor, args + +# Helper for __reduce_ex__ protocol 2 + +def __newobj__(cls, *args): + return cls.__new__(cls, *args) + +def _slotnames(cls): + """Return a list of slot names for a given class. + + This needs to find slots defined by the class and its bases, so we + can't simply return the __slots__ attribute. We must walk down + the Method Resolution Order and concatenate the __slots__ of each + class found there. (This assumes classes don't modify their + __slots__ attribute to misrepresent their slots after the class is + defined.) + """ + + # Get the value from a cache in the class if possible + names = cls.__dict__.get("__slotnames__") + if names is not None: + return names + + # Not cached -- calculate the value + names = [] + if not hasattr(cls, "__slots__"): + # This class has no slots + pass + else: + # Slots found -- gather slot names from all base classes + for c in cls.__mro__: + if "__slots__" in c.__dict__: + slots = c.__dict__['__slots__'] + # if class has a single slot, it can be given as a string + if isinstance(slots, basestring): + slots = (slots,) + for name in slots: + # special descriptors + if name in ("__dict__", "__weakref__"): + continue + # mangled names + elif name.startswith('__') and not name.endswith('__'): + names.append('_%s%s' % (c.__name__, name)) + else: + names.append(name) + + # Cache the outcome in the class if at all possible + try: + cls.__slotnames__ = names + except: + pass # But don't die if we can't + + return names + +# A registry of extension codes. This is an ad-hoc compression +# mechanism. Whenever a global reference to , is about +# to be pickled, the (, ) tuple is looked up here to see +# if it is a registered extension code for it. Extension codes are +# universal, so that the meaning of a pickle does not depend on +# context. (There are also some codes reserved for local use that +# don't have this restriction.) Codes are positive ints; 0 is +# reserved. + +_extension_registry = {} # key -> code +_inverted_registry = {} # code -> key +_extension_cache = {} # code -> object +# Don't ever rebind those names: cPickle grabs a reference to them when +# it's initialized, and won't see a rebinding. + +def add_extension(module, name, code): + """Register an extension code.""" + code = int(code) + if not 1 <= code <= 0x7fffffff: + raise ValueError, "code out of range" + key = (module, name) + if (_extension_registry.get(key) == code and + _inverted_registry.get(code) == key): + return # Redundant registrations are benign + if key in _extension_registry: + raise ValueError("key %s is already registered with code %s" % + (key, _extension_registry[key])) + if code in _inverted_registry: + raise ValueError("code %s is already in use for key %s" % + (code, _inverted_registry[code])) + _extension_registry[key] = code + _inverted_registry[code] = key + +def remove_extension(module, name, code): + """Unregister an extension code. For testing only.""" + key = (module, name) + if (_extension_registry.get(key) != code or + _inverted_registry.get(code) != key): + raise ValueError("key %s is not registered with code %s" % + (key, code)) + del _extension_registry[key] + del _inverted_registry[code] + if code in _extension_cache: + del _extension_cache[code] + +def clear_extension_cache(): + _extension_cache.clear() + +# Standard extension code assignments + +# Reserved ranges + +# First Last Count Purpose +# 1 127 127 Reserved for Python standard library +# 128 191 64 Reserved for Zope +# 192 239 48 Reserved for 3rd parties +# 240 255 16 Reserved for private use (will never be assigned) +# 256 Inf Inf Reserved for future assignment + +# Extension codes are assigned by the Python Software Foundation. diff --git a/validator/testcases/javascript/actions.py b/validator/testcases/javascript/actions.py index 27c594150..07d35ec19 100644 --- a/validator/testcases/javascript/actions.py +++ b/validator/testcases/javascript/actions.py @@ -1,4 +1,3 @@ -import copy import math import re import types @@ -10,6 +9,7 @@ from validator.constants import (BUGZILLA_BUG, DESCRIPTION_TYPES, FENNEC_GUID, FIREFOX_GUID, MAX_STR_SIZE) from validator.decorator import version_range +from validator.python import copy from jstypes import * diff --git a/validator/testcases/javascript/call_definitions.py b/validator/testcases/javascript/call_definitions.py index 59bbb52eb..3c5df767b 100644 --- a/validator/testcases/javascript/call_definitions.py +++ b/validator/testcases/javascript/call_definitions.py @@ -1,4 +1,3 @@ -import copy import math import re import types @@ -12,6 +11,7 @@ FX9_DEFINITION, FX11_DEFINITION, TB12_DEFINITION, TB13_DEFINITION) from validator.decorator import version_range +from validator.python import copy # Function prototypes should implement the following: # wrapper : The JSWrapper instace that is being called