Oops, something went wrong.
| @@ -0,0 +1,213 @@ | ||
| """A more or less complete user-defined wrapper around dictionary objects.""" | ||
| class UserDict: | ||
| def __init__(*args, **kwargs): | ||
| if not args: | ||
| raise TypeError("descriptor '__init__' of 'UserDict' object " | ||
| "needs an argument") | ||
| self = args[0] | ||
| args = args[1:] | ||
| if len(args) > 1: | ||
| raise TypeError('expected at most 1 arguments, got %d' % len(args)) | ||
| if args: | ||
| dict = args[0] | ||
| elif 'dict' in kwargs: | ||
| dict = kwargs.pop('dict') | ||
| import warnings | ||
| warnings.warn("Passing 'dict' as keyword argument is " | ||
| "deprecated", PendingDeprecationWarning, | ||
| stacklevel=2) | ||
| else: | ||
| dict = None | ||
| self.data = {} | ||
| if dict is not None: | ||
| self.update(dict) | ||
| if len(kwargs): | ||
| self.update(kwargs) | ||
| def __repr__(self): return repr(self.data) | ||
| def __cmp__(self, dict): | ||
| if isinstance(dict, UserDict): | ||
| return cmp(self.data, dict.data) | ||
| else: | ||
| return cmp(self.data, dict) | ||
| __hash__ = None # Avoid Py3k warning | ||
| def __len__(self): return len(self.data) | ||
| def __getitem__(self, key): | ||
| if key in self.data: | ||
| return self.data[key] | ||
| if hasattr(self.__class__, "__missing__"): | ||
| return self.__class__.__missing__(self, key) | ||
| raise KeyError(key) | ||
| def __setitem__(self, key, item): self.data[key] = item | ||
| def __delitem__(self, key): del self.data[key] | ||
| def clear(self): self.data.clear() | ||
| def copy(self): | ||
| if self.__class__ is UserDict: | ||
| return UserDict(self.data.copy()) | ||
| import copy | ||
| data = self.data | ||
| try: | ||
| self.data = {} | ||
| c = copy.copy(self) | ||
| finally: | ||
| self.data = data | ||
| c.update(self) | ||
| return c | ||
| def keys(self): return self.data.keys() | ||
| def items(self): return self.data.items() | ||
| def iteritems(self): return self.data.iteritems() | ||
| def iterkeys(self): return self.data.iterkeys() | ||
| def itervalues(self): return self.data.itervalues() | ||
| def values(self): return self.data.values() | ||
| def has_key(self, key): return key in self.data | ||
| def update(*args, **kwargs): | ||
| if not args: | ||
| raise TypeError("descriptor 'update' of 'UserDict' object " | ||
| "needs an argument") | ||
| self = args[0] | ||
| args = args[1:] | ||
| if len(args) > 1: | ||
| raise TypeError('expected at most 1 arguments, got %d' % len(args)) | ||
| if args: | ||
| dict = args[0] | ||
| elif 'dict' in kwargs: | ||
| dict = kwargs.pop('dict') | ||
| import warnings | ||
| warnings.warn("Passing 'dict' as keyword argument is deprecated", | ||
| PendingDeprecationWarning, stacklevel=2) | ||
| else: | ||
| dict = None | ||
| if dict is None: | ||
| pass | ||
| elif isinstance(dict, UserDict): | ||
| self.data.update(dict.data) | ||
| elif isinstance(dict, type({})) or not hasattr(dict, 'items'): | ||
| self.data.update(dict) | ||
| else: | ||
| for k, v in dict.items(): | ||
| self[k] = v | ||
| if len(kwargs): | ||
| self.data.update(kwargs) | ||
| def get(self, key, failobj=None): | ||
| if key not in self: | ||
| return failobj | ||
| return self[key] | ||
| def setdefault(self, key, failobj=None): | ||
| if key not in self: | ||
| self[key] = failobj | ||
| return self[key] | ||
| def pop(self, key, *args): | ||
| return self.data.pop(key, *args) | ||
| def popitem(self): | ||
| return self.data.popitem() | ||
| def __contains__(self, key): | ||
| return key in self.data | ||
| @classmethod | ||
| def fromkeys(cls, iterable, value=None): | ||
| d = cls() | ||
| for key in iterable: | ||
| d[key] = value | ||
| return d | ||
| class IterableUserDict(UserDict): | ||
| def __iter__(self): | ||
| return iter(self.data) | ||
| import _abcoll | ||
| _abcoll.MutableMapping.register(IterableUserDict) | ||
| class DictMixin: | ||
| # Mixin defining all dictionary methods for classes that already have | ||
| # a minimum dictionary interface including getitem, setitem, delitem, | ||
| # and keys. Without knowledge of the subclass constructor, the mixin | ||
| # does not define __init__() or copy(). In addition to the four base | ||
| # methods, progressively more efficiency comes with defining | ||
| # __contains__(), __iter__(), and iteritems(). | ||
| # second level definitions support higher levels | ||
| def __iter__(self): | ||
| for k in self.keys(): | ||
| yield k | ||
| def has_key(self, key): | ||
| try: | ||
| self[key] | ||
| except KeyError: | ||
| return False | ||
| return True | ||
| def __contains__(self, key): | ||
| return self.has_key(key) | ||
| # third level takes advantage of second level definitions | ||
| def iteritems(self): | ||
| for k in self: | ||
| yield (k, self[k]) | ||
| def iterkeys(self): | ||
| return self.__iter__() | ||
| # fourth level uses definitions from lower levels | ||
| def itervalues(self): | ||
| for _, v in self.iteritems(): | ||
| yield v | ||
| def values(self): | ||
| return [v for _, v in self.iteritems()] | ||
| def items(self): | ||
| return list(self.iteritems()) | ||
| def clear(self): | ||
| for key in self.keys(): | ||
| del self[key] | ||
| def setdefault(self, key, default=None): | ||
| try: | ||
| return self[key] | ||
| except KeyError: | ||
| self[key] = default | ||
| return default | ||
| def pop(self, key, *args): | ||
| if len(args) > 1: | ||
| raise TypeError, "pop expected at most 2 arguments, got "\ | ||
| + repr(1 + len(args)) | ||
| try: | ||
| value = self[key] | ||
| except KeyError: | ||
| if args: | ||
| return args[0] | ||
| raise | ||
| del self[key] | ||
| return value | ||
| def popitem(self): | ||
| try: | ||
| k, v = self.iteritems().next() | ||
| except StopIteration: | ||
| raise KeyError, 'container is empty' | ||
| del self[k] | ||
| return (k, v) | ||
| def update(self, other=None, **kwargs): | ||
| # Make progressively weaker assumptions about "other" | ||
| if other is None: | ||
| pass | ||
| elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups | ||
| for k, v in other.iteritems(): | ||
| self[k] = v | ||
| elif hasattr(other, 'keys'): | ||
| for k in other.keys(): | ||
| self[k] = other[k] | ||
| else: | ||
| for k, v in other: | ||
| self[k] = v | ||
| if kwargs: | ||
| self.update(kwargs) | ||
| def get(self, key, default=None): | ||
| try: | ||
| return self[key] | ||
| except KeyError: | ||
| return default | ||
| def __repr__(self): | ||
| return repr(dict(self.iteritems())) | ||
| def __cmp__(self, other): | ||
| if other is None: | ||
| return 1 | ||
| if isinstance(other, DictMixin): | ||
| other = dict(other.iteritems()) | ||
| return cmp(dict(self.iteritems()), other) | ||
| def __len__(self): | ||
| return len(self.keys()) |
| @@ -0,0 +1,128 @@ | ||
| """Record of phased-in incompatible language changes. | ||
| Each line is of the form: | ||
| FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease "," | ||
| CompilerFlag ")" | ||
| where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples | ||
| of the same form as sys.version_info: | ||
| (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int | ||
| PY_MINOR_VERSION, # the 1; an int | ||
| PY_MICRO_VERSION, # the 0; an int | ||
| PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string | ||
| PY_RELEASE_SERIAL # the 3; an int | ||
| ) | ||
| OptionalRelease records the first release in which | ||
| from __future__ import FeatureName | ||
| was accepted. | ||
| In the case of MandatoryReleases that have not yet occurred, | ||
| MandatoryRelease predicts the release in which the feature will become part | ||
| of the language. | ||
| Else MandatoryRelease records when the feature became part of the language; | ||
| in releases at or after that, modules no longer need | ||
| from __future__ import FeatureName | ||
| to use the feature in question, but may continue to use such imports. | ||
| MandatoryRelease may also be None, meaning that a planned feature got | ||
| dropped. | ||
| Instances of class _Feature have two corresponding methods, | ||
| .getOptionalRelease() and .getMandatoryRelease(). | ||
| CompilerFlag is the (bitfield) flag that should be passed in the fourth | ||
| argument to the builtin function compile() to enable the feature in | ||
| dynamically compiled code. This flag is stored in the .compiler_flag | ||
| attribute on _Future instances. These values must match the appropriate | ||
| #defines of CO_xxx flags in Include/compile.h. | ||
| No feature line is ever to be deleted from this file. | ||
| """ | ||
| all_feature_names = [ | ||
| "nested_scopes", | ||
| "generators", | ||
| "division", | ||
| "absolute_import", | ||
| "with_statement", | ||
| "print_function", | ||
| "unicode_literals", | ||
| ] | ||
| __all__ = ["all_feature_names"] + all_feature_names | ||
| # The CO_xxx symbols are defined here under the same names used by | ||
| # compile.h, so that an editor search will find them here. However, | ||
| # they're not exported in __all__, because they don't really belong to | ||
| # this module. | ||
| CO_NESTED = 0x0010 # nested_scopes | ||
| CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) | ||
| CO_FUTURE_DIVISION = 0x2000 # division | ||
| CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default | ||
| CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement | ||
| CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function | ||
| CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals | ||
| class _Feature: | ||
| def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): | ||
| self.optional = optionalRelease | ||
| self.mandatory = mandatoryRelease | ||
| self.compiler_flag = compiler_flag | ||
| def getOptionalRelease(self): | ||
| """Return first release in which this feature was recognized. | ||
| This is a 5-tuple, of the same form as sys.version_info. | ||
| """ | ||
| return self.optional | ||
| def getMandatoryRelease(self): | ||
| """Return release in which this feature will become mandatory. | ||
| This is a 5-tuple, of the same form as sys.version_info, or, if | ||
| the feature was dropped, is None. | ||
| """ | ||
| return self.mandatory | ||
| def __repr__(self): | ||
| return "_Feature" + repr((self.optional, | ||
| self.mandatory, | ||
| self.compiler_flag)) | ||
| nested_scopes = _Feature((2, 1, 0, "beta", 1), | ||
| (2, 2, 0, "alpha", 0), | ||
| CO_NESTED) | ||
| generators = _Feature((2, 2, 0, "alpha", 1), | ||
| (2, 3, 0, "final", 0), | ||
| CO_GENERATOR_ALLOWED) | ||
| division = _Feature((2, 2, 0, "alpha", 2), | ||
| (3, 0, 0, "alpha", 0), | ||
| CO_FUTURE_DIVISION) | ||
| absolute_import = _Feature((2, 5, 0, "alpha", 1), | ||
| (3, 0, 0, "alpha", 0), | ||
| CO_FUTURE_ABSOLUTE_IMPORT) | ||
| with_statement = _Feature((2, 5, 0, "alpha", 1), | ||
| (2, 6, 0, "alpha", 0), | ||
| CO_FUTURE_WITH_STATEMENT) | ||
| print_function = _Feature((2, 6, 0, "alpha", 2), | ||
| (3, 0, 0, "alpha", 0), | ||
| CO_FUTURE_PRINT_FUNCTION) | ||
| unicode_literals = _Feature((2, 6, 0, "alpha", 2), | ||
| (3, 0, 0, "alpha", 0), | ||
| CO_FUTURE_UNICODE_LITERALS) |
| @@ -0,0 +1,96 @@ | ||
| TOPIC_LOOKUP = {'!': '2-2-4', | ||
| '&': '2-5-2', | ||
| '&&': '2-2-5', | ||
| '((': '2-7-1', | ||
| ':': '2-2-3', | ||
| 'ASSIGNING-VARIABLES': '3-0-0', | ||
| 'Arithmetic': '5-1-0', | ||
| 'BUILTIN-COMMANDS': '6-0-0', | ||
| 'Boolean': '5-2-0', | ||
| 'Brace-Expand': '5-4-0', | ||
| 'Builtin-Procs': '12-2-0', | ||
| 'COMMAND-LANGUAGE': '2-0-0', | ||
| 'Child-Process': '6-7-0', | ||
| 'Commands': '2-1-0', | ||
| 'Completion': '6-5-0', | ||
| 'Compound-Data': '3-3-0', | ||
| 'Concurrency': '2-5-0', | ||
| 'Conditional': '2-3-0', | ||
| 'ENVIRONMENT-VARIABLES': '8-0-0', | ||
| 'Execution': '7-2-0', | ||
| 'External': '6-9-0', | ||
| 'Grouping': '2-4-0', | ||
| 'I/O': '6-1-0', | ||
| 'INTRO': '1-0-0', | ||
| 'Introspection': '6-8-0', | ||
| 'Keywords': '3-1-0', | ||
| 'Lexing': '1-4-0', | ||
| 'OIL-EXTENSINOS': '11-0-0', | ||
| 'OIL-LIBRARIES': '12-0-0', | ||
| 'OSH-Options': '7-3-0', | ||
| 'OTHER-SHELL-SUBLANGUAGES': '5-0-0', | ||
| 'Oil': '1-4-0', | ||
| 'Operators': '3-2-0', | ||
| 'Other': '2-7-0', | ||
| 'Overview': '1-1-0', | ||
| 'PLUGINS-AND-HOOKS': '10-0-0', | ||
| 'Parsing': '7-1-0', | ||
| 'Patterns': '5-3-0', | ||
| 'Quotes': '4-1-0', | ||
| 'Redirects': '2-6-0', | ||
| 'Run-Code': '6-2-0', | ||
| 'SHELL-OPTIONS': '7-0-0', | ||
| 'SPECIAL-VARIABLES': '9-0-0', | ||
| 'Set-Options': '6-3-0', | ||
| 'Shell-Process': '6-6-0', | ||
| 'Special-Vars': '4-3-0', | ||
| 'Substitutions': '4-2-0', | ||
| 'Usage': '1-2-0', | ||
| 'Var-Ops': '4-4-0', | ||
| 'WORD-LANGUAGE': '4-0-0', | ||
| 'Working-Dir': '6-4-0', | ||
| '[[': '2-2-6', | ||
| 'ampersand': '2-5-2', | ||
| 'and': '2-2-5', | ||
| 'bang': '2-2-4', | ||
| 'block': '2-4-2', | ||
| 'bundle-usage': '1-2-1', | ||
| 'caller': '6-8-3', | ||
| 'case': '2-2-1', | ||
| 'colon': '2-2-3', | ||
| 'config': '1-2-4', | ||
| 'coproc': '2-7-3', | ||
| 'dbracket': '2-2-6', | ||
| 'dparen': '2-7-1', | ||
| 'enable': '6-9-3', | ||
| 'false': '2-2-3', | ||
| 'for': '2-3-2', | ||
| 'for-expr': '2-3-2', | ||
| 'function': '2-4-1', | ||
| 'getopt': '6-9-1', | ||
| 'hash': '6-8-2', | ||
| 'help': '6-8-1', | ||
| 'here-doc': '2-6-3', | ||
| 'if': '2-2-2', | ||
| 'kill': '6-9-2', | ||
| 'line-editing': '1-2-6', | ||
| 'oil-usage': '1-2-3', | ||
| 'or': '2-2-5', | ||
| 'osh-usage': '1-2-2', | ||
| 'overview': '1-1-1', | ||
| 'pipe': '2-5-1', | ||
| 'prompt': '1-2-7', | ||
| 'read': '6-1-1', | ||
| 'redir-desc': '2-6-2', | ||
| 'redir-file': '2-6-1', | ||
| 'semicolon': '2-1-2', | ||
| 'simple-command': '2-1-1', | ||
| 'single-command': '1-4-1', | ||
| 'startup': '1-2-5', | ||
| 'subshell': '2-4-3', | ||
| 'time': '2-7-2', | ||
| 'true': '2-2-3', | ||
| 'type': '6-8-4', | ||
| 'until': '2-3-1', | ||
| 'while': '2-3-1', | ||
| '||': '2-2-5'} |
| @@ -0,0 +1,252 @@ | ||
| from asdl import const # For const.NO_INTEGER | ||
| from asdl import py_meta | ||
| from osh.meta import RUNTIME_TYPE_LOOKUP as TYPE_LOOKUP | ||
| class part_value_e(object): | ||
| StringPartValue = 1 | ||
| ArrayPartValue = 2 | ||
| class part_value(py_meta.CompoundObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('part_value') | ||
| class StringPartValue(part_value): | ||
| tag = 1 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('StringPartValue') | ||
| __slots__ = ('s', 'do_split_glob', 'spids') | ||
| def __init__(self, s=None, do_split_glob=None, spids=None): | ||
| self.s = s | ||
| self.do_split_glob = do_split_glob | ||
| self.spids = spids or [] | ||
| class ArrayPartValue(part_value): | ||
| tag = 2 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('ArrayPartValue') | ||
| __slots__ = ('strs', 'spids') | ||
| def __init__(self, strs=None, spids=None): | ||
| self.strs = strs or [] | ||
| self.spids = spids or [] | ||
| class value_e(object): | ||
| Undef = 1 | ||
| Str = 2 | ||
| StrArray = 3 | ||
| class value(py_meta.CompoundObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('value') | ||
| class Undef(value): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('Undef') | ||
| tag = 1 | ||
| class Str(value): | ||
| tag = 2 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('Str') | ||
| __slots__ = ('s', 'spids') | ||
| def __init__(self, s=None, spids=None): | ||
| self.s = s | ||
| self.spids = spids or [] | ||
| class StrArray(value): | ||
| tag = 3 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('StrArray') | ||
| __slots__ = ('strs', 'spids') | ||
| def __init__(self, strs=None, spids=None): | ||
| self.strs = strs or [] | ||
| self.spids = spids or [] | ||
| class cell(py_meta.CompoundObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('cell') | ||
| __slots__ = ('val', 'exported', 'readonly', 'spids') | ||
| def __init__(self, val=None, exported=None, readonly=None, spids=None): | ||
| self.val = val | ||
| self.exported = exported | ||
| self.readonly = readonly | ||
| self.spids = spids or [] | ||
| class var_flags_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('var_flags') | ||
| var_flags_e.Exported = var_flags_e(1, 'Exported') | ||
| var_flags_e.ReadOnly = var_flags_e(2, 'ReadOnly') | ||
| class scope_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('scope') | ||
| scope_e.TempEnv = scope_e(1, 'TempEnv') | ||
| scope_e.LocalOnly = scope_e(2, 'LocalOnly') | ||
| scope_e.GlobalOnly = scope_e(3, 'GlobalOnly') | ||
| scope_e.Dynamic = scope_e(4, 'Dynamic') | ||
| class lvalue_e(object): | ||
| LhsName = 1 | ||
| LhsIndexedName = 2 | ||
| class lvalue(py_meta.CompoundObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('lvalue') | ||
| class LhsName(lvalue): | ||
| tag = 1 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('LhsName') | ||
| __slots__ = ('name', 'spids') | ||
| def __init__(self, name=None, spids=None): | ||
| self.name = name | ||
| self.spids = spids or [] | ||
| class LhsIndexedName(lvalue): | ||
| tag = 2 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('LhsIndexedName') | ||
| __slots__ = ('name', 'index', 'spids') | ||
| def __init__(self, name=None, index=None, spids=None): | ||
| self.name = name | ||
| self.index = index | ||
| self.spids = spids or [] | ||
| class redirect_e(object): | ||
| PathRedirect = 1 | ||
| DescRedirect = 2 | ||
| HereRedirect = 3 | ||
| class redirect(py_meta.CompoundObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('redirect') | ||
| class PathRedirect(redirect): | ||
| tag = 1 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('PathRedirect') | ||
| __slots__ = ('op_id', 'fd', 'filename', 'spids') | ||
| def __init__(self, op_id=None, fd=None, filename=None, spids=None): | ||
| self.op_id = op_id | ||
| self.fd = fd | ||
| self.filename = filename | ||
| self.spids = spids or [] | ||
| class DescRedirect(redirect): | ||
| tag = 2 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('DescRedirect') | ||
| __slots__ = ('op_id', 'fd', 'target_fd', 'spids') | ||
| def __init__(self, op_id=None, fd=None, target_fd=None, spids=None): | ||
| self.op_id = op_id | ||
| self.fd = fd | ||
| self.target_fd = target_fd | ||
| self.spids = spids or [] | ||
| class HereRedirect(redirect): | ||
| tag = 3 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('HereRedirect') | ||
| __slots__ = ('fd', 'body', 'spids') | ||
| def __init__(self, fd=None, body=None, spids=None): | ||
| self.fd = fd | ||
| self.body = body | ||
| self.spids = spids or [] | ||
| class job_status_e(object): | ||
| ProcessStatus = 1 | ||
| PipelineStatus = 2 | ||
| class job_status(py_meta.CompoundObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('job_status') | ||
| class ProcessStatus(job_status): | ||
| tag = 1 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('ProcessStatus') | ||
| __slots__ = ('status', 'spids') | ||
| def __init__(self, status=None, spids=None): | ||
| self.status = status | ||
| self.spids = spids or [] | ||
| class PipelineStatus(job_status): | ||
| tag = 2 | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('PipelineStatus') | ||
| __slots__ = ('statuses', 'spids') | ||
| def __init__(self, statuses=None, spids=None): | ||
| self.statuses = statuses or [] | ||
| self.spids = spids or [] | ||
| class span_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('span') | ||
| span_e.Black = span_e(1, 'Black') | ||
| span_e.Delim = span_e(2, 'Delim') | ||
| span_e.Backslash = span_e(3, 'Backslash') | ||
| class builtin_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('builtin') | ||
| builtin_e.NONE = builtin_e(1, 'NONE') | ||
| builtin_e.READ = builtin_e(2, 'READ') | ||
| builtin_e.ECHO = builtin_e(3, 'ECHO') | ||
| builtin_e.SHIFT = builtin_e(4, 'SHIFT') | ||
| builtin_e.CD = builtin_e(5, 'CD') | ||
| builtin_e.PUSHD = builtin_e(6, 'PUSHD') | ||
| builtin_e.POPD = builtin_e(7, 'POPD') | ||
| builtin_e.DIRS = builtin_e(8, 'DIRS') | ||
| builtin_e.EXPORT = builtin_e(9, 'EXPORT') | ||
| builtin_e.UNSET = builtin_e(10, 'UNSET') | ||
| builtin_e.SET = builtin_e(11, 'SET') | ||
| builtin_e.SHOPT = builtin_e(12, 'SHOPT') | ||
| builtin_e.TRAP = builtin_e(13, 'TRAP') | ||
| builtin_e.UMASK = builtin_e(14, 'UMASK') | ||
| builtin_e.SOURCE = builtin_e(15, 'SOURCE') | ||
| builtin_e.DOT = builtin_e(16, 'DOT') | ||
| builtin_e.EVAL = builtin_e(17, 'EVAL') | ||
| builtin_e.EXEC = builtin_e(18, 'EXEC') | ||
| builtin_e.WAIT = builtin_e(19, 'WAIT') | ||
| builtin_e.JOBS = builtin_e(20, 'JOBS') | ||
| builtin_e.COMPLETE = builtin_e(21, 'COMPLETE') | ||
| builtin_e.COMPGEN = builtin_e(22, 'COMPGEN') | ||
| builtin_e.DEBUG_LINE = builtin_e(23, 'DEBUG_LINE') | ||
| builtin_e.TRUE = builtin_e(24, 'TRUE') | ||
| builtin_e.FALSE = builtin_e(25, 'FALSE') | ||
| builtin_e.COLON = builtin_e(26, 'COLON') | ||
| builtin_e.TEST = builtin_e(27, 'TEST') | ||
| builtin_e.BRACKET = builtin_e(28, 'BRACKET') | ||
| builtin_e.GETOPTS = builtin_e(29, 'GETOPTS') | ||
| builtin_e.COMMAND = builtin_e(30, 'COMMAND') | ||
| builtin_e.TYPE = builtin_e(31, 'TYPE') | ||
| builtin_e.HELP = builtin_e(32, 'HELP') | ||
| builtin_e.DECLARE = builtin_e(33, 'DECLARE') | ||
| builtin_e.TYPESET = builtin_e(34, 'TYPESET') | ||
| class effect_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('effect') | ||
| effect_e.SpliceParts = effect_e(1, 'SpliceParts') | ||
| effect_e.Error = effect_e(2, 'Error') | ||
| effect_e.SpliceAndAssign = effect_e(3, 'SpliceAndAssign') | ||
| effect_e.NoOp = effect_e(4, 'NoOp') | ||
| class process_state_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('process_state') | ||
| process_state_e.Init = process_state_e(1, 'Init') | ||
| process_state_e.Done = process_state_e(2, 'Done') | ||
| class completion_state_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('completion_state') | ||
| completion_state_e.NONE = completion_state_e(1, 'NONE') | ||
| completion_state_e.FIRST = completion_state_e(2, 'FIRST') | ||
| completion_state_e.REST = completion_state_e(3, 'REST') | ||
| completion_state_e.VAR_NAME = completion_state_e(4, 'VAR_NAME') | ||
| completion_state_e.HASH_KEY = completion_state_e(5, 'HASH_KEY') | ||
| completion_state_e.REDIR_FILENAME = completion_state_e(6, 'REDIR_FILENAME') | ||
| class word_style_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('word_style') | ||
| word_style_e.Expr = word_style_e(1, 'Expr') | ||
| word_style_e.Unquoted = word_style_e(2, 'Unquoted') | ||
| word_style_e.DQ = word_style_e(3, 'DQ') | ||
| word_style_e.SQ = word_style_e(4, 'SQ') | ||
| @@ -0,0 +1,39 @@ | ||
| from asdl import const # For const.NO_INTEGER | ||
| from asdl import py_meta | ||
| from osh.meta import TYPES_TYPE_LOOKUP as TYPE_LOOKUP | ||
| class bool_arg_type_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('bool_arg_type') | ||
| bool_arg_type_e.Undefined = bool_arg_type_e(1, 'Undefined') | ||
| bool_arg_type_e.Path = bool_arg_type_e(2, 'Path') | ||
| bool_arg_type_e.Int = bool_arg_type_e(3, 'Int') | ||
| bool_arg_type_e.Str = bool_arg_type_e(4, 'Str') | ||
| bool_arg_type_e.Other = bool_arg_type_e(5, 'Other') | ||
| class redir_arg_type_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('redir_arg_type') | ||
| redir_arg_type_e.Path = redir_arg_type_e(1, 'Path') | ||
| redir_arg_type_e.Desc = redir_arg_type_e(2, 'Desc') | ||
| redir_arg_type_e.Here = redir_arg_type_e(3, 'Here') | ||
| class lex_mode_e(py_meta.SimpleObj): | ||
| ASDL_TYPE = TYPE_LOOKUP.ByTypeName('lex_mode') | ||
| lex_mode_e.NONE = lex_mode_e(1, 'NONE') | ||
| lex_mode_e.COMMENT = lex_mode_e(2, 'COMMENT') | ||
| lex_mode_e.OUTER = lex_mode_e(3, 'OUTER') | ||
| lex_mode_e.DBRACKET = lex_mode_e(4, 'DBRACKET') | ||
| lex_mode_e.SQ = lex_mode_e(5, 'SQ') | ||
| lex_mode_e.DQ = lex_mode_e(6, 'DQ') | ||
| lex_mode_e.DOLLAR_SQ = lex_mode_e(7, 'DOLLAR_SQ') | ||
| lex_mode_e.ARITH = lex_mode_e(8, 'ARITH') | ||
| lex_mode_e.EXTGLOB = lex_mode_e(9, 'EXTGLOB') | ||
| lex_mode_e.VS_1 = lex_mode_e(10, 'VS_1') | ||
| lex_mode_e.VS_2 = lex_mode_e(11, 'VS_2') | ||
| lex_mode_e.VS_ARG_UNQ = lex_mode_e(12, 'VS_ARG_UNQ') | ||
| lex_mode_e.VS_ARG_DQ = lex_mode_e(13, 'VS_ARG_DQ') | ||
| lex_mode_e.BASH_REGEX = lex_mode_e(14, 'BASH_REGEX') | ||
| lex_mode_e.BASH_REGEX_CHARS = lex_mode_e(15, 'BASH_REGEX_CHARS') | ||
| @@ -0,0 +1,204 @@ | ||
| # Access WeakSet through the weakref module. | ||
| # This code is separated-out because it is needed | ||
| # by abc.py to load everything else at startup. | ||
| from _weakref import ref | ||
| __all__ = ['WeakSet'] | ||
| class _IterationGuard(object): | ||
| # This context manager registers itself in the current iterators of the | ||
| # weak container, such as to delay all removals until the context manager | ||
| # exits. | ||
| # This technique should be relatively thread-safe (since sets are). | ||
| def __init__(self, weakcontainer): | ||
| # Don't create cycles | ||
| self.weakcontainer = ref(weakcontainer) | ||
| def __enter__(self): | ||
| w = self.weakcontainer() | ||
| if w is not None: | ||
| w._iterating.add(self) | ||
| return self | ||
| def __exit__(self, e, t, b): | ||
| w = self.weakcontainer() | ||
| if w is not None: | ||
| s = w._iterating | ||
| s.remove(self) | ||
| if not s: | ||
| w._commit_removals() | ||
| class WeakSet(object): | ||
| def __init__(self, data=None): | ||
| self.data = set() | ||
| def _remove(item, selfref=ref(self)): | ||
| self = selfref() | ||
| if self is not None: | ||
| if self._iterating: | ||
| self._pending_removals.append(item) | ||
| else: | ||
| self.data.discard(item) | ||
| self._remove = _remove | ||
| # A list of keys to be removed | ||
| self._pending_removals = [] | ||
| self._iterating = set() | ||
| if data is not None: | ||
| self.update(data) | ||
| def _commit_removals(self): | ||
| l = self._pending_removals | ||
| discard = self.data.discard | ||
| while l: | ||
| discard(l.pop()) | ||
| def __iter__(self): | ||
| with _IterationGuard(self): | ||
| for itemref in self.data: | ||
| item = itemref() | ||
| if item is not None: | ||
| # Caveat: the iterator will keep a strong reference to | ||
| # `item` until it is resumed or closed. | ||
| yield item | ||
| def __len__(self): | ||
| return len(self.data) - len(self._pending_removals) | ||
| def __contains__(self, item): | ||
| try: | ||
| wr = ref(item) | ||
| except TypeError: | ||
| return False | ||
| return wr in self.data | ||
| def __reduce__(self): | ||
| return (self.__class__, (list(self),), | ||
| getattr(self, '__dict__', None)) | ||
| __hash__ = None | ||
| def add(self, item): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| self.data.add(ref(item, self._remove)) | ||
| def clear(self): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| self.data.clear() | ||
| def copy(self): | ||
| return self.__class__(self) | ||
| def pop(self): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| while True: | ||
| try: | ||
| itemref = self.data.pop() | ||
| except KeyError: | ||
| raise KeyError('pop from empty WeakSet') | ||
| item = itemref() | ||
| if item is not None: | ||
| return item | ||
| def remove(self, item): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| self.data.remove(ref(item)) | ||
| def discard(self, item): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| self.data.discard(ref(item)) | ||
| def update(self, other): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| for element in other: | ||
| self.add(element) | ||
| def __ior__(self, other): | ||
| self.update(other) | ||
| return self | ||
| def difference(self, other): | ||
| newset = self.copy() | ||
| newset.difference_update(other) | ||
| return newset | ||
| __sub__ = difference | ||
| def difference_update(self, other): | ||
| self.__isub__(other) | ||
| def __isub__(self, other): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| if self is other: | ||
| self.data.clear() | ||
| else: | ||
| self.data.difference_update(ref(item) for item in other) | ||
| return self | ||
| def intersection(self, other): | ||
| return self.__class__(item for item in other if item in self) | ||
| __and__ = intersection | ||
| def intersection_update(self, other): | ||
| self.__iand__(other) | ||
| def __iand__(self, other): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| self.data.intersection_update(ref(item) for item in other) | ||
| return self | ||
| def issubset(self, other): | ||
| return self.data.issubset(ref(item) for item in other) | ||
| __le__ = issubset | ||
| def __lt__(self, other): | ||
| return self.data < set(ref(item) for item in other) | ||
| def issuperset(self, other): | ||
| return self.data.issuperset(ref(item) for item in other) | ||
| __ge__ = issuperset | ||
| def __gt__(self, other): | ||
| return self.data > set(ref(item) for item in other) | ||
| def __eq__(self, other): | ||
| if not isinstance(other, self.__class__): | ||
| return NotImplemented | ||
| return self.data == set(ref(item) for item in other) | ||
| def __ne__(self, other): | ||
| opposite = self.__eq__(other) | ||
| if opposite is NotImplemented: | ||
| return NotImplemented | ||
| return not opposite | ||
| def symmetric_difference(self, other): | ||
| newset = self.copy() | ||
| newset.symmetric_difference_update(other) | ||
| return newset | ||
| __xor__ = symmetric_difference | ||
| def symmetric_difference_update(self, other): | ||
| self.__ixor__(other) | ||
| def __ixor__(self, other): | ||
| if self._pending_removals: | ||
| self._commit_removals() | ||
| if self is other: | ||
| self.data.clear() | ||
| else: | ||
| self.data.symmetric_difference_update(ref(item, self._remove) for item in other) | ||
| return self | ||
| def union(self, other): | ||
| return self.__class__(e for s in (self, other) for e in s) | ||
| __or__ = union | ||
| def isdisjoint(self, other): | ||
| return len(self.intersection(other)) == 0 |
| @@ -0,0 +1,185 @@ | ||
| # Copyright 2007 Google, Inc. All Rights Reserved. | ||
| # Licensed to PSF under a Contributor Agreement. | ||
| """Abstract Base Classes (ABCs) according to PEP 3119.""" | ||
| import types | ||
| from _weakrefset import WeakSet | ||
| # Instance of old-style class | ||
| class _C: pass | ||
| _InstanceType = type(_C()) | ||
| def abstractmethod(funcobj): | ||
| """A decorator indicating abstract methods. | ||
| Requires that the metaclass is ABCMeta or derived from it. A | ||
| class that has a metaclass derived from ABCMeta cannot be | ||
| instantiated unless all of its abstract methods are overridden. | ||
| The abstract methods can be called using any of the normal | ||
| 'super' call mechanisms. | ||
| Usage: | ||
| class C: | ||
| __metaclass__ = ABCMeta | ||
| @abstractmethod | ||
| def my_abstract_method(self, ...): | ||
| ... | ||
| """ | ||
| funcobj.__isabstractmethod__ = True | ||
| return funcobj | ||
| class abstractproperty(property): | ||
| """A decorator indicating abstract properties. | ||
| Requires that the metaclass is ABCMeta or derived from it. A | ||
| class that has a metaclass derived from ABCMeta cannot be | ||
| instantiated unless all of its abstract properties are overridden. | ||
| The abstract properties can be called using any of the normal | ||
| 'super' call mechanisms. | ||
| Usage: | ||
| class C: | ||
| __metaclass__ = ABCMeta | ||
| @abstractproperty | ||
| def my_abstract_property(self): | ||
| ... | ||
| This defines a read-only property; you can also define a read-write | ||
| abstract property using the 'long' form of property declaration: | ||
| class C: | ||
| __metaclass__ = ABCMeta | ||
| def getx(self): ... | ||
| def setx(self, value): ... | ||
| x = abstractproperty(getx, setx) | ||
| """ | ||
| __isabstractmethod__ = True | ||
| class ABCMeta(type): | ||
| """Metaclass for defining Abstract Base Classes (ABCs). | ||
| Use this metaclass to create an ABC. An ABC can be subclassed | ||
| directly, and then acts as a mix-in class. You can also register | ||
| unrelated concrete classes (even built-in classes) and unrelated | ||
| ABCs as 'virtual subclasses' -- these and their descendants will | ||
| be considered subclasses of the registering ABC by the built-in | ||
| issubclass() function, but the registering ABC won't show up in | ||
| their MRO (Method Resolution Order) nor will method | ||
| implementations defined by the registering ABC be callable (not | ||
| even via super()). | ||
| """ | ||
| # A global counter that is incremented each time a class is | ||
| # registered as a virtual subclass of anything. It forces the | ||
| # negative cache to be cleared before its next use. | ||
| _abc_invalidation_counter = 0 | ||
| def __new__(mcls, name, bases, namespace): | ||
| cls = super(ABCMeta, mcls).__new__(mcls, name, bases, namespace) | ||
| # Compute set of abstract method names | ||
| abstracts = set(name | ||
| for name, value in namespace.items() | ||
| if getattr(value, "__isabstractmethod__", False)) | ||
| for base in bases: | ||
| for name in getattr(base, "__abstractmethods__", set()): | ||
| value = getattr(cls, name, None) | ||
| if getattr(value, "__isabstractmethod__", False): | ||
| abstracts.add(name) | ||
| cls.__abstractmethods__ = frozenset(abstracts) | ||
| # Set up inheritance registry | ||
| cls._abc_registry = WeakSet() | ||
| cls._abc_cache = WeakSet() | ||
| cls._abc_negative_cache = WeakSet() | ||
| cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter | ||
| return cls | ||
| def register(cls, subclass): | ||
| """Register a virtual subclass of an ABC.""" | ||
| if not isinstance(subclass, (type, types.ClassType)): | ||
| raise TypeError("Can only register classes") | ||
| if issubclass(subclass, cls): | ||
| return # Already a subclass | ||
| # Subtle: test for cycles *after* testing for "already a subclass"; | ||
| # this means we allow X.register(X) and interpret it as a no-op. | ||
| if issubclass(cls, subclass): | ||
| # This would create a cycle, which is bad for the algorithm below | ||
| raise RuntimeError("Refusing to create an inheritance cycle") | ||
| cls._abc_registry.add(subclass) | ||
| ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache | ||
| def _dump_registry(cls, file=None): | ||
| """Debug helper to print the ABC registry.""" | ||
| print >> file, "Class: %s.%s" % (cls.__module__, cls.__name__) | ||
| print >> file, "Inv.counter: %s" % ABCMeta._abc_invalidation_counter | ||
| for name in sorted(cls.__dict__.keys()): | ||
| if name.startswith("_abc_"): | ||
| value = getattr(cls, name) | ||
| print >> file, "%s: %r" % (name, value) | ||
| def __instancecheck__(cls, instance): | ||
| """Override for isinstance(instance, cls).""" | ||
| # Inline the cache checking when it's simple. | ||
| subclass = getattr(instance, '__class__', None) | ||
| if subclass is not None and subclass in cls._abc_cache: | ||
| return True | ||
| subtype = type(instance) | ||
| # Old-style instances | ||
| if subtype is _InstanceType: | ||
| subtype = subclass | ||
| if subtype is subclass or subclass is None: | ||
| if (cls._abc_negative_cache_version == | ||
| ABCMeta._abc_invalidation_counter and | ||
| subtype in cls._abc_negative_cache): | ||
| return False | ||
| # Fall back to the subclass check. | ||
| return cls.__subclasscheck__(subtype) | ||
| return (cls.__subclasscheck__(subclass) or | ||
| cls.__subclasscheck__(subtype)) | ||
| def __subclasscheck__(cls, subclass): | ||
| """Override for issubclass(subclass, cls).""" | ||
| # Check cache | ||
| if subclass in cls._abc_cache: | ||
| return True | ||
| # Check negative cache; may have to invalidate | ||
| if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: | ||
| # Invalidate the negative cache | ||
| cls._abc_negative_cache = WeakSet() | ||
| cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter | ||
| elif subclass in cls._abc_negative_cache: | ||
| return False | ||
| # Check the subclass hook | ||
| ok = cls.__subclasshook__(subclass) | ||
| if ok is not NotImplemented: | ||
| assert isinstance(ok, bool) | ||
| if ok: | ||
| cls._abc_cache.add(subclass) | ||
| else: | ||
| cls._abc_negative_cache.add(subclass) | ||
| return ok | ||
| # Check if it's a direct subclass | ||
| if cls in getattr(subclass, '__mro__', ()): | ||
| cls._abc_cache.add(subclass) | ||
| return True | ||
| # Check if it's a subclass of a registered class (recursive) | ||
| for rcls in cls._abc_registry: | ||
| if issubclass(subclass, rcls): | ||
| cls._abc_cache.add(subclass) | ||
| return True | ||
| # Check if it's a subclass of a subclass (recursive) | ||
| for scls in cls.__subclasses__(): | ||
| if issubclass(subclass, scls): | ||
| cls._abc_cache.add(subclass) | ||
| return True | ||
| # No dice; update negative cache | ||
| cls._abc_negative_cache.add(subclass) | ||
| return False |
| @@ -0,0 +1,19 @@ | ||
| #!/usr/bin/env python | ||
| """ | ||
| arith_ast.py | ||
| """ | ||
| import os | ||
| import sys | ||
| from asdl import asdl_ as asdl | ||
| from asdl import py_meta | ||
| this_dir = os.path.dirname(os.path.abspath(sys.argv[0])) | ||
| schema_path = os.path.join(this_dir, 'arith.asdl') | ||
| with open(schema_path) as f: | ||
| module = asdl.parse(f) | ||
| type_lookup = asdl.ResolveTypes(module) | ||
| root = sys.modules[__name__] | ||
| py_meta.MakeTypes(module, root, type_lookup) |
| @@ -0,0 +1,197 @@ | ||
| #!/usr/bin/env python | ||
| from __future__ import print_function | ||
| """ | ||
| arith_ast_test.py: Tests for arith_ast.py | ||
| """ | ||
| import cStringIO | ||
| import unittest | ||
| from asdl import py_meta | ||
| from asdl import asdl_ | ||
| from asdl import const | ||
| from asdl import encode | ||
| from asdl import arith_ast # module under test | ||
| # Sanity check. Doesn't pass because this unit test exposes implementatio | ||
| # details, like the concrete classes. | ||
| #from _tmp import arith_ast_asdl as arith_ast | ||
| ArithVar = arith_ast.ArithVar | ||
| ArithUnary = arith_ast.ArithUnary | ||
| ArithBinary = arith_ast.ArithBinary | ||
| Const = arith_ast.Const | ||
| Slice = arith_ast.Slice | ||
| arith_expr = arith_ast.arith_expr | ||
| source_location = arith_ast.source_location | ||
| op_id_e = arith_ast.op_id_e | ||
| cflow_e = arith_ast.cflow_e | ||
| #cflow_t = arith_ast.cflow_t | ||
| class ArithAstTest(unittest.TestCase): | ||
| def testFieldDefaults(self): | ||
| s = arith_ast.Slice() | ||
| s.a = ArithVar('foo') | ||
| self.assertEqual(None, s.begin) | ||
| self.assertEqual(None, s.end) | ||
| self.assertEqual(None, s.stride) | ||
| print(s) | ||
| func = arith_ast.FuncCall() | ||
| func.name = 'f' | ||
| self.assertEqual([], func.args) | ||
| print(func) | ||
| t = arith_ast.token(5, 'x') | ||
| self.assertEqual(5, t.id) | ||
| self.assertEqual('x', t.value) | ||
| self.assertEqual(const.NO_INTEGER, t.span_id) | ||
| def testTypeCheck(self): | ||
| v = ArithVar('name') | ||
| # Integer is not allowed | ||
| self.assertRaises(AssertionError, ArithVar, 1) | ||
| v = ArithUnary(op_id_e.Minus, Const(99)) | ||
| # Raw integer is not allowed | ||
| self.assertRaises(AssertionError, ArithUnary, op_id_e.Minus, 99) | ||
| v = ArithUnary(op_id_e.Minus, Const(99)) | ||
| # Raw integer is not allowed | ||
| #self.assertRaises(AssertionError, ArithUnary, op_id_e.Minus, op_id_e.Plus) | ||
| def testExtraFields(self): | ||
| v = ArithVar('z') | ||
| # TODO: Attach this to EVERY non-simple constructor? Those are subclasses | ||
| # of Sum types. | ||
| # What about product types? | ||
| #print(v.xspans) | ||
| def testEncode(self): | ||
| obj = arith_ast.Const(99) | ||
| print('Encoding into binary:') | ||
| print(obj) | ||
| enc = encode.Params() | ||
| f = cStringIO.StringIO() | ||
| out = encode.BinOutput(f) | ||
| encode.EncodeRoot(obj, enc, out) | ||
| e = f.getvalue() | ||
| #print(repr(e)) | ||
| #print(e[0:4], e[4:8], e[8:]) | ||
| # Header is OHP version 1 | ||
| self.assertEqual(b'OHP\x01', e[0:4]) | ||
| self.assertEqual(b'\x04', e[4:5]) # alignment 4 | ||
| # TODO: Fix after spids | ||
| return | ||
| self.assertEqual(b'\x02\x00\x00', e[5:8]) # root ref 2 | ||
| self.assertEqual(b'\x01', e[8:9]) # tag 1 is const | ||
| self.assertEqual(b'\x63\x00\x00', e[9:12]) # 0x63 = 99 | ||
| def testConstructorType(self): | ||
| n1 = ArithVar('x') | ||
| n2 = ArithVar(name='y') | ||
| print(n1) | ||
| print(n2) | ||
| # Not good because not assigned? | ||
| n3 = ArithVar() | ||
| # NOTE: You cannot instantiate a product type directly? It's just used for | ||
| # type checking. What about OCaml? | ||
| # That means you just need to create classes for the records (Constructor). | ||
| # They all descend from Obj. They don't need | ||
| n3 = ArithVar() | ||
| try: | ||
| n4 = ArithVar('x', name='X') | ||
| except TypeError as e: | ||
| pass | ||
| else: | ||
| raise AssertionError("Should have failed") | ||
| def testProductType(self): | ||
| print() | ||
| print('-- PRODUCT --') | ||
| print() | ||
| s = source_location() | ||
| s.path = 'hi' | ||
| s.line = 1 | ||
| s.col = 2 | ||
| s.length = 3 | ||
| print(s) | ||
| assert isinstance(s.ASDL_TYPE, asdl_.Product) | ||
| # Implementation detail for dynamic type checking | ||
| assert isinstance(s, py_meta.CompoundObj) | ||
| def testSimpleSumType(self): | ||
| # TODO: Should be op_id_i.Plus -- instance | ||
| # Should be op_id_s.Plus | ||
| print() | ||
| print('-- SIMPLE SUM --') | ||
| print() | ||
| o = op_id_e.Plus | ||
| assert isinstance(o, py_meta.SimpleObj) | ||
| # Implementation detail for dynamic type checking | ||
| assert isinstance(o.ASDL_TYPE, asdl_.Sum) | ||
| def testCompoundSumType(self): | ||
| print() | ||
| print('-- COMPOUND SUM --') | ||
| print() | ||
| # TODO: Should be cflow_t.Break() and cflow_i.Break | ||
| c = arith_ast.Break() | ||
| assert isinstance(c, arith_ast.Break) | ||
| assert isinstance(c, arith_ast.cflow) | ||
| assert isinstance(c, py_meta.CompoundObj) | ||
| # Implementation detail for dynamic type checking | ||
| assert isinstance(c.ASDL_TYPE, asdl_.Constructor), c.ASDL_TYPE | ||
| def testOtherTypes(self): | ||
| c = Const(66) | ||
| print(c) | ||
| print((Slice(Const(1), Const(5), Const(2)))) | ||
| print((op_id_e.Plus)) | ||
| # Class for sum type | ||
| print(arith_expr) | ||
| # Invalid because only half were assigned | ||
| #print(ArithBinary(op_id_e.Plus, Const(5))) | ||
| n = ArithBinary() | ||
| #n.CheckUnassigned() | ||
| n.op_id = op_id_e.Plus | ||
| n.left = Const(5) | ||
| #n.CheckUnassigned() | ||
| n.right = Const(6) | ||
| n.CheckUnassigned() | ||
| arith_expr_e = arith_ast.arith_expr_e | ||
| self.assertEqual(arith_expr_e.Const, c.tag) | ||
| self.assertEqual(arith_expr_e.ArithBinary, n.tag) | ||
| if __name__ == '__main__': | ||
| unittest.main() |
| @@ -0,0 +1,246 @@ | ||
| #!/usr/bin/env python | ||
| from __future__ import print_function | ||
| """ | ||
| arith_parse.py: Parse shell-like and C-like arithmetic. | ||
| """ | ||
| import sys | ||
| from asdl import tdop | ||
| from asdl.tdop import CompositeNode | ||
| from asdl import arith_ast | ||
| op_id = arith_ast.op_id_e # TODO: Rename this back. | ||
| # | ||
| # Null Denotation -- token that takes nothing on the left | ||
| # | ||
| def NullConstant(p, token, bp): | ||
| if token.type == 'number': | ||
| return arith_ast.Const(token.val) | ||
| # We have to wrap a string in some kind of variant. | ||
| if token.type == 'name': | ||
| return arith_ast.ArithVar(token.val) | ||
| raise AssertionError(token.type) | ||
| def NullParen(p, token, bp): | ||
| """ Arithmetic grouping """ | ||
| r = p.ParseUntil(bp) | ||
| p.Eat(')') | ||
| return r | ||
| def NullPrefixOp(p, token, bp): | ||
| """Prefix operator. | ||
| Low precedence: return, raise, etc. | ||
| return x+y is return (x+y), not (return x) + y | ||
| High precedence: logical negation, bitwise complement, etc. | ||
| !x && y is (!x) && y, not !(x && y) | ||
| """ | ||
| r = p.ParseUntil(bp) | ||
| return CompositeNode(token, [r]) | ||
| def NullIncDec(p, token, bp): | ||
| """ ++x or ++x[1] """ | ||
| right = p.ParseUntil(bp) | ||
| if right.token.type not in ('name', 'get'): | ||
| raise tdop.ParseError("Can't assign to %r (%s)" % (right, right.token)) | ||
| return CompositeNode(token, [right]) | ||
| # | ||
| # Left Denotation -- token that takes an expression on the left | ||
| # | ||
| def LeftIncDec(p, token, left, rbp): | ||
| """ For i++ and i-- | ||
| """ | ||
| if left.token.type not in ('name', 'get'): | ||
| raise tdop.ParseError("Can't assign to %r (%s)" % (left, left.token)) | ||
| token.type = 'post' + token.type | ||
| return CompositeNode(token, [left]) | ||
| def LeftIndex(p, token, left, unused_bp): | ||
| """ index f[x+1] """ | ||
| # f[x] or f[x][y] | ||
| if not isinstance(left, arith_ast.ArithVar): | ||
| raise tdop.ParseError("%s can't be indexed" % left) | ||
| index = p.ParseUntil(0) | ||
| if p.AtToken(':'): | ||
| p.Next() | ||
| end = p.ParseUntil(0) | ||
| else: | ||
| end = None | ||
| p.Eat(']') | ||
| # TODO: If you see ], then | ||
| # 1:4 | ||
| # 1:4:2 | ||
| # Both end and step are optional | ||
| if end: | ||
| return arith_ast.Slice(left, index, end, None) | ||
| else: | ||
| return arith_ast.Index(left, index) | ||
| def LeftTernary(p, token, left, bp): | ||
| """ e.g. a > 1 ? x : y """ | ||
| true_expr = p.ParseUntil(bp) | ||
| p.Eat(':') | ||
| false_expr = p.ParseUntil(bp) | ||
| children = [left, true_expr, false_expr] | ||
| return CompositeNode(token, children) | ||
| def LeftBinaryOp(p, token, left, rbp): | ||
| """ Normal binary operator like 1+2 or 2*3, etc. """ | ||
| if token.val == '+': | ||
| op_id_ = op_id.Plus | ||
| elif token.val == '-': | ||
| op_id_ = op_id.Minus | ||
| elif token.val == '*': | ||
| op_id_ = op_id.Star | ||
| else: | ||
| raise AssertionError(token.val) | ||
| return arith_ast.ArithBinary(op_id_, left, p.ParseUntil(rbp)) | ||
| def LeftAssign(p, token, left, rbp): | ||
| """ Normal binary operator like 1+2 or 2*3, etc. """ | ||
| # x += 1, or a[i] += 1 | ||
| if left.token.type not in ('name', 'get'): | ||
| raise tdop.ParseError("Can't assign to %r (%s)" % (left, left.token)) | ||
| return CompositeNode(token, [left, p.ParseUntil(rbp)]) | ||
| def LeftComma(p, token, left, rbp): | ||
| """ foo, bar, baz | ||
| Could be sequencing operator, or tuple without parens | ||
| """ | ||
| r = p.ParseUntil(rbp) | ||
| if left.token.type == ',': # Keep adding more children | ||
| left.children.append(r) | ||
| return left | ||
| children = [left, r] | ||
| return CompositeNode(token, children) | ||
| # For overloading of , inside function calls | ||
| COMMA_PREC = 1 | ||
| def LeftFuncCall(p, token, left, unused_bp): | ||
| """ Function call f(a, b). """ | ||
| args = [] | ||
| # f(x) or f[i](x) | ||
| if not isinstance(left, arith_ast.ArithVar): | ||
| raise tdop.ParseError("%s can't be called" % left) | ||
| func_name = left.name # get a string | ||
| while not p.AtToken(')'): | ||
| # We don't want to grab the comma, e.g. it is NOT a sequence operator. So | ||
| # set the precedence to 5. | ||
| args.append(p.ParseUntil(COMMA_PREC)) | ||
| if p.AtToken(','): | ||
| p.Next() | ||
| p.Eat(")") | ||
| return arith_ast.FuncCall(func_name, args) | ||
| def MakeShellParserSpec(): | ||
| """ | ||
| Create a parser. | ||
| Compare the code below with this table of C operator precedence: | ||
| http://en.cppreference.com/w/c/language/operator_precedence | ||
| """ | ||
| spec = tdop.ParserSpec() | ||
| spec.Left(31, LeftIncDec, ['++', '--']) | ||
| spec.Left(31, LeftFuncCall, ['(']) | ||
| spec.Left(31, LeftIndex, ['[']) | ||
| # 29 -- binds to everything except function call, indexing, postfix ops | ||
| spec.Null(29, NullIncDec, ['++', '--']) | ||
| spec.Null(29, NullPrefixOp, ['+', '!', '~', '-']) | ||
| # Right associative: 2 ** 3 ** 2 == 2 ** (3 ** 2) | ||
| spec.LeftRightAssoc(27, LeftBinaryOp, ['**']) | ||
| spec.Left(25, LeftBinaryOp, ['*', '/', '%']) | ||
| spec.Left(23, LeftBinaryOp, ['+', '-']) | ||
| spec.Left(21, LeftBinaryOp, ['<<', '>>']) | ||
| spec.Left(19, LeftBinaryOp, ['<', '>', '<=', '>=']) | ||
| spec.Left(17, LeftBinaryOp, ['!=', '==']) | ||
| spec.Left(15, LeftBinaryOp, ['&']) | ||
| spec.Left(13, LeftBinaryOp, ['^']) | ||
| spec.Left(11, LeftBinaryOp, ['|']) | ||
| spec.Left(9, LeftBinaryOp, ['&&']) | ||
| spec.Left(7, LeftBinaryOp, ['||']) | ||
| spec.LeftRightAssoc(5, LeftTernary, ['?']) | ||
| # Right associative: a = b = 2 is a = (b = 2) | ||
| spec.LeftRightAssoc(3, LeftAssign, [ | ||
| '=', | ||
| '+=', '-=', '*=', '/=', '%=', | ||
| '<<=', '>>=', '&=', '^=', '|=']) | ||
| spec.Left(COMMA_PREC, LeftComma, [',']) | ||
| # 0 precedence -- doesn't bind until ) | ||
| spec.Null(0, NullParen, ['(']) # for grouping | ||
| # -1 precedence -- never used | ||
| spec.Null(-1, NullConstant, ['name', 'number']) | ||
| spec.Null(-1, tdop.NullError, [')', ']', ':', 'eof']) | ||
| return spec | ||
| def MakeParser(s): | ||
| """Used by tests.""" | ||
| spec = MakeShellParserSpec() | ||
| lexer = tdop.Tokenize(s) | ||
| p = tdop.Parser(spec, lexer) | ||
| return p | ||
| def ParseShell(s, expected=None): | ||
| """Used by tests.""" | ||
| p = MakeParser(s) | ||
| tree = p.Parse() | ||
| sexpr = repr(tree) | ||
| if expected is not None: | ||
| assert sexpr == expected, '%r != %r' % (sexpr, expected) | ||
| #print('%-40s %s' % (s, sexpr)) | ||
| return tree | ||
| def main(argv): | ||
| try: | ||
| s = argv[1] | ||
| except IndexError: | ||
| print('Usage: ./arith_parse.py EXPRESSION') | ||
| else: | ||
| try: | ||
| tree = ParseShell(s) | ||
| except tdop.ParseError as e: | ||
| print('Error parsing %r: %s' % (s, e), file=sys.stderr) | ||
| if __name__ == '__main__': | ||
| main(sys.argv) |
| @@ -0,0 +1,223 @@ | ||
| #!/usr/bin/env python | ||
| from __future__ import print_function | ||
| from asdl import tdop | ||
| from asdl import arith_ast | ||
| from asdl import arith_parse # module under test | ||
| def _assertParseError(make_parser, s, error_substring=''): | ||
| p = make_parser(s) | ||
| try: | ||
| node = p.Parse() | ||
| except tdop.ParseError as e: | ||
| err = str(e) | ||
| if error_substring in err: | ||
| print('got expected error for %s: %s' % (s, err)) | ||
| else: | ||
| raise AssertionError('Expected %r to be in %r' % (error_substring, err)) | ||
| else: | ||
| raise AssertionError('%r should have failed' % s) | ||
| def TestArith(t_parse): | ||
| t_parse('1+2+3', '(+ (+ 1 2) 3)') | ||
| t_parse('1+2*3', '(+ 1 (* 2 3))') | ||
| t_parse('4*(2+3)', '(* 4 (+ 2 3))') | ||
| t_parse('(2+3)*4', '(* (+ 2 3) 4)') | ||
| return | ||
| t_parse('1<2', '(< 1 2)') | ||
| t_parse('x=3', '(= x 3)') | ||
| t_parse('x = 2*3', '(= x (* 2 3))') | ||
| t_parse('x = y', '(= x y)') | ||
| t_parse('x*y - y*z', '(- (* x y) (* y z))') | ||
| t_parse('x/y - y%z', '(- (/ x y) (% y z))') | ||
| t_parse("x = y", "(= x y)") | ||
| t_parse('2 ** 3 ** 2', '(** 2 (** 3 2))') | ||
| t_parse('a = b = 10', '(= a (= b 10))') | ||
| t_parse('x = ((y*4)-2)', '(= x (- (* y 4) 2))') | ||
| t_parse('x - -y', '(- x (- y))') | ||
| t_parse("-1 * -2", "(* (- 1) (- 2))") | ||
| t_parse("-x * -y", "(* (- x) (- y))") | ||
| t_parse('x - -234', '(- x (- 234))') | ||
| # Python doesn't allow this | ||
| t_parse('x += y += 3', '(+= x (+= y 3))') | ||
| # This is sort of nonsensical, but bash allows it. The 1 is discarded as | ||
| # the first element of the comma operator. | ||
| t_parse('x[1,2]', '(get x (, 1 2))') | ||
| # Python doesn't have unary + | ||
| t_parse('+1 - +2', '(- (+ 1) (+ 2))') | ||
| # LHS | ||
| t_parse('f[x] += 1', '(+= (get f x) 1)') | ||
| def TestBitwise(t_parse): | ||
| t_parse("~1 | ~2", "(| (~ 1) (~ 2))") | ||
| t_parse("x & y | a & b", "(| (& x y) (& a b))") | ||
| t_parse("~x ^ y", "(^ (~ x) y)") | ||
| t_parse("x << y | y << z", "(| (<< x y) (<< y z))") | ||
| t_parse("a ^= b-1", "(^= a (- b 1))") | ||
| def TestLogical(t_parse): | ||
| t_parse("a && b || c && d", "(|| (&& a b) (&& c d))") | ||
| t_parse("!a && !b", "(&& (! a) (! b))") | ||
| t_parse("a != b && c == d", "(&& (!= a b) (== c d))") | ||
| t_parse("a > b ? 0 : 1", "(? (> a b) 0 1)") | ||
| t_parse("a > b ? x+1 : y+1", "(? (> a b) (+ x 1) (+ y 1))") | ||
| t_parse("1 ? true1 : 2 ? true2 : false", "(? 1 true1 (? 2 true2 false))") | ||
| t_parse("1 ? true1 : (2 ? true2 : false)", "(? 1 true1 (? 2 true2 false))") | ||
| t_parse("1 ? (2 ? true : false1) : false2", "(? 1 (? 2 true false1) false2)") | ||
| t_parse("1 ? 2 ? true : false1 : false2", "(? 1 (? 2 true false1) false2)") | ||
| # Should have higher precedence than comma | ||
| t_parse("x ? 1 : 2, y ? 3 : 4", "(, (? x 1 2) (? y 3 4))") | ||
| def TestUnary(t_parse): | ||
| t_parse("!x", "(! x)") | ||
| t_parse("x--", "(post-- x)") | ||
| t_parse("x[1]--", "(post-- (get x 1))") | ||
| t_parse("--x", "(-- x)") | ||
| t_parse("++x[1]", "(++ (get x 1))") | ||
| t_parse("!x--", "(! (post-- x))") | ||
| t_parse("~x++", "(~ (post++ x))") | ||
| t_parse("x++ - y++", "(- (post++ x) (post++ y))") | ||
| t_parse("++x - ++y", "(- (++ x) (++ y))") | ||
| # | ||
| # 1. x++ f() x[] left associative | ||
| # f(x)[1]++ means | ||
| # (++ (get (call f x) 1)) | ||
| # 2. ++x + - ! ~ right associative | ||
| # -++x means (- (++ x)) | ||
| def TestArrays(t_parse): | ||
| """Shared between shell, oil, and Python.""" | ||
| t_parse('x[1]', '(get x 1)') | ||
| t_parse('x[a+b]', '(get x (+ a b))') | ||
| def TestComma(t_parse): | ||
| t_parse('x=1,y=2,z=3', '(, (= x 1) (= y 2) (= z 3))') | ||
| def TestFuncCalls(t_parse): | ||
| t_parse('x = y(2)*3 + y(4)*5', '(= x (+ (* (call y 2) 3) (* (call y 4) 5)))') | ||
| t_parse('x(1,2)+y(3,4)', '(+ (call x 1 2) (call y 3 4))') | ||
| t_parse('x(a,b,c[d])', '(call x a b (get c d))') | ||
| t_parse('x(1,2)*j+y(3,4)*k+z(5,6)*l', | ||
| '(+ (+ (* (call x 1 2) j) (* (call y 3 4) k)) (* (call z 5 6) l))') | ||
| t_parse('print(test(2,3))', '(call print (call test 2 3))') | ||
| t_parse('print("x")', '(call print x)') | ||
| t_parse('min(255,n*2)', '(call min 255 (* n 2))') | ||
| t_parse('c = pal[i*8]', '(= c (get pal (* i 8)))') | ||
| def TestErrors(p): | ||
| _assertParseError(p, '}') | ||
| _assertParseError(p, ']') | ||
| _assertParseError(p, '{') # depends on language | ||
| _assertParseError(p, "x+1 = y", "Can't assign") | ||
| _assertParseError(p, "(x+1)++", "Can't assign") | ||
| # Should be an EOF error | ||
| _assertParseError(p, 'foo ? 1 :', 'Unexpected end') | ||
| _assertParseError(p, 'foo ? 1 ', 'expected :') | ||
| _assertParseError(p, '%', "can't be used in prefix position") | ||
| error_str = "can't be used in prefix" | ||
| _assertParseError(p, '}') | ||
| _assertParseError(p, '{') | ||
| _assertParseError(p, ']', error_str) | ||
| _assertParseError(p, '1 ( 2', "can't be called") | ||
| _assertParseError(p, '(x+1) ( 2 )', "can't be called") | ||
| #_assertParseError(p, '1 ) 2') | ||
| _assertParseError(p, '1 [ 2 ]', "can't be indexed") | ||
| arith_expr_e = arith_ast.arith_expr_e | ||
| #source_location = arith_ast.source_location | ||
| #op_id_e = arith_ast.op_id_e | ||
| class Visitor(object): | ||
| def __init__(self): | ||
| pass | ||
| # In Python, they do introspection on method names. | ||
| # method = 'visit_' + node.__class__.__name__ | ||
| # I'm not going to bother, because I have ASDL! I want the generic visitor. | ||
| def Visit(self, node): | ||
| raise NotImplementedError | ||
| # Like ast.NodeVisitor().generic_visit! | ||
| def VisitChildren(self, node): | ||
| #print dir(node) | ||
| # TODO: Use node.ASDL_TYPE.GetFields() | ||
| # Only compound children get visited? | ||
| print([name for name in dir(node) if not name.startswith('_')]) | ||
| # Call self.Visit()! | ||
| class PrettyPrinter(Visitor): | ||
| def Visit(self, node): | ||
| if node.tag == arith_expr_e.ArithUnary: | ||
| print('ArithUnary %s' % node.child) | ||
| else: | ||
| self.VisitChildren(node) | ||
| def t_parse(s, expected=None): | ||
| p = arith_parse.MakeParser(s) | ||
| tree = p.Parse() | ||
| print(tree) | ||
| #v = PrettyPrinter() | ||
| #v.Visit(tree) | ||
| #print('%-40s %s' % (s, sexpr)) | ||
| return tree | ||
| def main(): | ||
| p = arith_parse.MakeParser | ||
| TestArith(t_parse) | ||
| return | ||
| TestBitwise(t_parse) | ||
| TestLogical(t_parse) | ||
| TestUnary(t_parse) | ||
| TestArrays(t_parse) | ||
| TestFuncCalls(t_parse) | ||
| TestComma(t_parse) | ||
| TestErrors(p) | ||
| if __name__ == '__main__': | ||
| main() |
| @@ -0,0 +1,86 @@ | ||
| #!/usr/bin/env python | ||
| from __future__ import print_function | ||
| """ | ||
| asdl_demo.py | ||
| """ | ||
| import sys | ||
| from asdl import asdl_ as asdl | ||
| from asdl import arith_parse | ||
| from asdl import py_meta | ||
| from asdl import encode | ||
| from asdl import format as fmt | ||
| from osh.meta import Id | ||
| from core import util | ||
| log = util.log | ||
| def main(argv): | ||
| try: | ||
| action = argv[1] | ||
| except IndexError: | ||
| raise RuntimeError('Action required') | ||
| if action == 'py': # Prints the module | ||
| schema_path = argv[2] | ||
| with open(schema_path) as f: | ||
| module = asdl.parse(f) | ||
| app_types = {'id': asdl.UserType(Id)} | ||
| type_lookup = asdl.ResolveTypes(module, app_types) | ||
| # Note this is a big tree. But we really want a graph of pointers to | ||
| # instances. | ||
| # Type(name, Product(...)) | ||
| # Type(name, Sum([Constructor(...), ...])) | ||
| #print(module) | ||
| root = sys.modules[__name__] | ||
| # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a | ||
| # demo. | ||
| py_meta.MakeTypes(module, root, type_lookup) | ||
| log('Dynamically created a Python module with these types:') | ||
| for name in dir(root): | ||
| print('\t' + name) | ||
| elif action == 'arith-encode': # oheap encoding | ||
| expr = argv[2] | ||
| out_path = argv[3] | ||
| obj = arith_parse.ParseShell(expr) | ||
| print('Encoding %r into binary:' % expr) | ||
| print(obj) | ||
| enc = encode.Params() | ||
| with open(out_path, 'wb') as f: | ||
| out = encode.BinOutput(f) | ||
| encode.EncodeRoot(obj, enc, out) | ||
| elif action == 'arith-format': # pretty printing | ||
| expr = argv[2] | ||
| obj = arith_parse.ParseShell(expr) | ||
| #out = fmt.TextOutput(sys.stdout) | ||
| tree = fmt.MakeTree(obj) | ||
| #treee= ['hi', 'there', ['a', 'b'], 'c'] | ||
| f = fmt.DetectConsoleOutput(sys.stdout) | ||
| fmt.PrintTree(tree, f) | ||
| print() | ||
| # Might need to print the output? | ||
| # out.WriteToFile? | ||
| else: | ||
| raise RuntimeError('Invalid action %r' % action) | ||
| if __name__ == '__main__': | ||
| try: | ||
| main(sys.argv) | ||
| except RuntimeError as e: | ||
| print('FATAL: %r' % e, file=sys.stderr) | ||
| sys.exit(1) |
| @@ -0,0 +1,36 @@ | ||
| #!/usr/bin/env python | ||
| """ | ||
| const.py | ||
| """ | ||
| DEFAULT_INT_WIDTH = 3 # 24 bits | ||
| # 2^24 - 1 is used as an invalid/uninitialized value for ASDL integers. | ||
| # Why? We have a few use cases for invalid/sentinel values: | ||
| # - span_id, line_id. Sometimes we don't have a span ID. | ||
| # - file descriptor: 'read x < f.txt' vs 'read x 0< f.txt' | ||
| # | ||
| # Other options for representation: | ||
| # | ||
| # 1. ADSL could use signed integers, then -1 is valid. | ||
| # 2. Use a type like fd = None | Some(int fd) | ||
| # | ||
| # I don't like #1 because ASDL is lazily-decoded, and then we have to do sign | ||
| # extension on demand. (24 bits to 32 or 64). As far as I can tell, sign | ||
| # extension requires a branch, at least in portable C (on the sign bit). | ||
| # | ||
| # Thes second option is semantically cleaner. But it needlessly | ||
| # inflates the size of both the source code and the data. Instead of having a | ||
| # single "inline" integer, we would need a reference to another value. | ||
| # | ||
| # We could also try to do some fancy thing like fd = None | | ||
| # Range<1..max_fd>(fd), with smart encoding. But that is overkill for these | ||
| # use cases. | ||
| # | ||
| # Using InvalidInt instead of -1 seems like a good compromise. | ||
| NO_INTEGER = (1 << (DEFAULT_INT_WIDTH * 8)) - 1 | ||
| # NOTE: In Python: 1 << (n * 8) - 1 is wrong! I thought that bit shift would | ||
| # have higher precedence. |
| @@ -0,0 +1,294 @@ | ||
| """ | ||
| encode.py | ||
| """ | ||
| from asdl import asdl_ as asdl | ||
| from asdl import py_meta | ||
| from asdl import const | ||
| from core import util | ||
| class EncodeError(Exception): | ||
| def __init__(self, *args, **kwargs): | ||
| Exception.__init__(self, *args, **kwargs) | ||
| self.details_printed = False | ||
| _DEFAULT_ALIGNMENT = 4 | ||
| class BinOutput(object): | ||
| """Write aligned blocks here. Keeps track of block indexes for refs.""" | ||
| def __init__(self, f, alignment=_DEFAULT_ALIGNMENT): | ||
| self.f = f | ||
| # index of last block, to return as a ref. | ||
| self.last_block = 0 | ||
| self.alignment = alignment | ||
| def WriteRootRef(self, chunk): | ||
| self.f.seek(5) # seek past 'OHP\x01\x04' | ||
| assert len(chunk) == 3 | ||
| self.f.write(chunk) | ||
| def Write(self, chunk): | ||
| """ | ||
| Return a block pointer/index. | ||
| """ | ||
| # Input should be padded | ||
| assert len(chunk) % self.alignment == 0 | ||
| self.f.write(chunk) | ||
| ref = self.last_block | ||
| num_blocks = len(chunk) // self.alignment # int division | ||
| #print('WROTE %d blocks' % num_blocks) | ||
| self.last_block += num_blocks | ||
| # Return a reference to the beginning | ||
| return ref | ||
| class Params(object): | ||
| """Encoding parameters. | ||
| Hm most of these settings should be per-field, expressed in the schema. The | ||
| only global one is the ref/pointer alignment. 4 and 8 are the most likely | ||
| choices, and 4 is probably fine, because you have 64 MB of addressable memory | ||
| with 24 bit pointers. | ||
| """ | ||
| def __init__(self, alignment=_DEFAULT_ALIGNMENT, | ||
| int_width=const.DEFAULT_INT_WIDTH): | ||
| self.alignment = alignment | ||
| self.pointer_type = 'uint32_t' | ||
| self.tag_width = 1 # for ArithVar vs ArithWord. | ||
| self.int_width = int_width | ||
| self.ref_width = int_width # Constant 3, used by gen_cpp | ||
| # used for fd, line/col | ||
| # also I guess steuff like SimpleCommand | ||
| self.index_width = 2 # 16 bits, e.g. max 64K entries in an array | ||
| self.max_int = 1 << (self.int_width * 8) | ||
| self.max_index = 1 << (self.index_width * 8) | ||
| self.max_tag = 1 << (self.tag_width * 8) | ||
| def Tag(self, i, chunk): | ||
| if i > self.max_tag: | ||
| raise AssertionError('Invalid id %r' % i) | ||
| chunk.append(i & 0xFF) | ||
| def Int(self, n, chunk): | ||
| if n < 0: | ||
| raise EncodeError( | ||
| "ASDL can't currently encode negative numbers. Got %d" % n) | ||
| if n > self.max_int: | ||
| raise EncodeError( | ||
| '%d is too big to fit in %d bytes' % (n, self.int_width)) | ||
| for i in range(self.int_width): | ||
| chunk.append(n & 0xFF) | ||
| n >>= 8 | ||
| def Ref(self, n, chunk): | ||
| # NOTE: ref width is currently the same as int width. Could be different. | ||
| self.Int(n, chunk) | ||
| def _Pad(self, chunk): | ||
| n = len(chunk) | ||
| a = self.alignment | ||
| if n % a != 0: | ||
| chunk.extend(b'\x00' * (a - (n % a))) | ||
| return chunk | ||
| # Right now all strings are references. Later they could be inline. | ||
| def Str(self, s, chunk): | ||
| # NOTE: For variable, proc, and function names, it could make sense to | ||
| # pre-compute and store a hash value. They will be looked up in the stack | ||
| # and so forth. | ||
| # - You could also return a obj number or object ID. | ||
| chunk.extend(s) | ||
| chunk.append(0) # NUL terminator | ||
| def PaddedStr(self, s): | ||
| # NOTE: | ||
| # - The encoder could also have an intern table to save space. | ||
| # - Str and PaddedStr will both return char* ? Should we allow them to | ||
| # VARY with the same schema, is a value/ref type PART of the schema? It's | ||
| # basically small size optimization and "flexible array" optimization. I | ||
| # think you want that possibility. | ||
| chunk = bytearray() | ||
| self.Str(s, chunk) | ||
| return self._Pad(chunk) | ||
| def Bytes(self, buf, chunk): | ||
| n = len(buf) | ||
| if n >= self.max_index: | ||
| raise EncodeError("bytes object is too long (%d)" % n) | ||
| for i in range(self.index_width): | ||
| chunk.append(n & 0xFF) | ||
| n >>= 8 | ||
| chunk.extend(buf) | ||
| def PaddedBytes(self, buf): | ||
| chunk = bytearray() | ||
| self.Bytes(buf, chunk) | ||
| return self._Pad(chunk) | ||
| def PaddedBlock(self, chunk): | ||
| return self._Pad(chunk) | ||
| def EncodeArray(obj_list, item_desc, enc, out): | ||
| """ | ||
| Args: | ||
| obj_list: List of Obj values | ||
| Returns: | ||
| ref | ||
| """ | ||
| array_chunk = bytearray() | ||
| enc.Int(len(obj_list), array_chunk) # Length prefix | ||
| if isinstance(item_desc, asdl.IntType) or \ | ||
| isinstance(item_desc, asdl.BoolType): | ||
| for item in obj_list: | ||
| enc.Int(item, array_chunk) | ||
| elif isinstance(item_desc, asdl.UserType): | ||
| # Assume Id for now | ||
| for item in obj_list: | ||
| enc.Int(item.enum_value, array_chunk) | ||
| elif isinstance(item_desc, asdl.StrType): | ||
| for item in obj_list: | ||
| ref = out.Write(enc.PaddedStr(item)) | ||
| enc.Ref(ref, array_chunk) | ||
| elif isinstance(item_desc, asdl.Sum) and asdl.is_simple(item_desc): | ||
| for item in obj_list: | ||
| enc.Int(item.enum_id, array_chunk) | ||
| else: | ||
| # A simple value is either an int, enum, or pointer. (Later: Iter<Str> | ||
| # might be possible for locality.) | ||
| assert isinstance(item_desc, asdl.Sum) or \ | ||
| isinstance(item_desc, asdl.Product), item_desc | ||
| # This is like vector<T*> | ||
| # Later: | ||
| # - Product types can be put in line | ||
| # - Sum types can even be put in line, if you have List<T> rather than | ||
| # Array<T>. Array implies O(1) random access; List doesn't. | ||
| for item in obj_list: | ||
| try: | ||
| ref = EncodeObj(item, enc, out) | ||
| except EncodeError as e: | ||
| if not e.details_printed: | ||
| util.log("Error encoding array: %s (item %s)", e, item) | ||
| e.details_printed = True | ||
| raise | ||
| enc.Ref(ref, array_chunk) | ||
| this_ref = out.Write(enc.PaddedBlock(array_chunk)) | ||
| return this_ref | ||
| def EncodeObj(obj, enc, out): | ||
| """ | ||
| Args: | ||
| obj: Obj to encode | ||
| enc: encoding params | ||
| out: output file | ||
| Returns: | ||
| ref: Reference to the last block | ||
| """ | ||
| # Algorithm: Depth first, post-order traversal. First obj is the first leaf. | ||
| # last obj is the root. | ||
| # | ||
| # Array is a homogeneous type. | ||
| this_chunk = bytearray() | ||
| assert isinstance(obj, py_meta.CompoundObj), \ | ||
| '%r is not a compound obj (%r)' % (obj, obj.__class__) | ||
| # Constructor objects have a tag. | ||
| if isinstance(obj.ASDL_TYPE, asdl.Constructor): | ||
| enc.Tag(obj.tag, this_chunk) | ||
| for name, desc in obj.ASDL_TYPE.GetFields(): # encode in order | ||
| field_val = getattr(obj, name) | ||
| # TODO: | ||
| # - Float would be inline, etc. | ||
| # - Repeated value: write them all adjacent to each other? | ||
| is_maybe = False | ||
| if isinstance(desc, asdl.MaybeType): | ||
| is_maybe = True | ||
| desc = desc.desc # descent | ||
| # | ||
| # Now look at types | ||
| # | ||
| if isinstance(desc, asdl.IntType) or isinstance(desc, asdl.BoolType): | ||
| enc.Int(field_val, this_chunk) | ||
| elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc): | ||
| # Encode enums as integers. TODO later: Don't use 3 bytes! Can use 1 | ||
| # byte for most enums. | ||
| enc.Int(field_val.enum_id, this_chunk) | ||
| # Write variable length field first, assuming that it's a ref/pointer. | ||
| # TODO: allow one inline, hanging string or array per record. | ||
| elif isinstance(desc, asdl.StrType): | ||
| ref = out.Write(enc.PaddedStr(field_val)) | ||
| enc.Ref(ref, this_chunk) | ||
| elif isinstance(desc, asdl.ArrayType): | ||
| item_desc = desc.desc | ||
| ref = EncodeArray(field_val, item_desc, enc, out) | ||
| enc.Ref(ref, this_chunk) | ||
| elif isinstance(desc, asdl.UserType): | ||
| if is_maybe and field_val is None: # e.g. id? prefix_op | ||
| enc.Ref(0, this_chunk) | ||
| else: | ||
| # Assume Id for now | ||
| enc.Int(field_val.enum_value, this_chunk) | ||
| else: | ||
| if is_maybe and field_val is None: | ||
| enc.Ref(0, this_chunk) | ||
| else: | ||
| try: | ||
| ref = EncodeObj(field_val, enc, out) | ||
| except EncodeError as e: | ||
| if not e.details_printed: | ||
| util.log("Error encoding %s : %s (val %s)", name, e, field_val) | ||
| e.details_printed = True | ||
| raise | ||
| enc.Ref(ref, this_chunk) | ||
| # Write the parent record | ||
| this_ref = out.Write(enc.PaddedBlock(this_chunk)) | ||
| return this_ref | ||
| def EncodeRoot(obj, enc, out): | ||
| ref = out.Write(b'OHP\x01') # header, version 1 | ||
| assert ref == 0 | ||
| # 4-byte alignment, then 3 byte placeholder for the root ref. | ||
| ref = out.Write(b'\4\0\0\0') | ||
| assert ref == 1 | ||
| root_ref = EncodeObj(obj, enc, out) | ||
| chunk = bytearray() | ||
| enc.Ref(root_ref, chunk) | ||
| out.WriteRootRef(chunk) # back up and write it | ||
| #print("Root obj ref:", root_ref) |
| @@ -0,0 +1,37 @@ | ||
| #!/usr/bin/env python | ||
| """ | ||
| encode_test.py: Tests for encode.py | ||
| """ | ||
| import unittest | ||
| from asdl import encode # module under test | ||
| from asdl import const | ||
| class EncoderTest(unittest.TestCase): | ||
| def testEncoder(self): | ||
| p = encode.Params(16) | ||
| chunk = bytearray() | ||
| p.Int(1, chunk) | ||
| self.assertEqual(b'\x01\x00\x00', chunk) | ||
| chunk = bytearray() | ||
| p.Int(const.NO_INTEGER, chunk) | ||
| self.assertEqual(b'\xff\xff\xff', chunk) | ||
| chunk = p.PaddedBytes('0123456789') | ||
| # 2 byte length -- max 64K entries | ||
| self.assertEqual(b'\x0A\x000123456789\x00\x00\x00\x00', bytes(chunk)) | ||
| chunk = p.PaddedStr('0123456789') | ||
| # 2 byte length -- max 64K entries | ||
| self.assertEqual(b'0123456789\x00\x00\x00\x00\x00\x00', bytes(chunk)) | ||
| #p.Block([b'a', b'bc']) | ||
| if __name__ == '__main__': | ||
| unittest.main() |
| @@ -0,0 +1,49 @@ | ||
| #!/usr/bin/python -S | ||
| """ | ||
| format_test.py: Tests for format.py | ||
| """ | ||
| import cStringIO | ||
| import unittest | ||
| from asdl import format as fmt | ||
| from asdl import arith_ast # module under test | ||
| class FormatTest(unittest.TestCase): | ||
| def testSimpleSum(self): | ||
| node = arith_ast.op_id_e.Plus | ||
| print(node) | ||
| f = cStringIO.StringIO() | ||
| ast_f = fmt.TextOutput(f) | ||
| tree = fmt.MakeTree(node) | ||
| fmt.PrintTree(tree, ast_f) | ||
| # Hm this prints 'Plus'. Doesn't print the class or the number. | ||
| # But those aren't intrinsic. These are mostly used for ther IDENTITY. | ||
| # I think the ASDL_TYPE field contains the relevant info. Yes! | ||
| pretty_str = f.getvalue() | ||
| print(pretty_str) | ||
| def testRepeatedString(self): | ||
| node = arith_ast.assign('declare', ['-r', '-x']) | ||
| f = cStringIO.StringIO() | ||
| ast_f = fmt.TextOutput(f) | ||
| tree = fmt.MakeTree(node) | ||
| #print(tree) | ||
| fmt.PrintTree(tree, ast_f) | ||
| pretty_str = f.getvalue() | ||
| print(pretty_str) | ||
| self.assertEqual('(assign name:declare flags:[-r -x])', pretty_str) | ||
| if __name__ == '__main__': | ||
| unittest.main() |
| @@ -0,0 +1,399 @@ | ||
| #!/usr/bin/env python | ||
| from __future__ import print_function | ||
| """ | ||
| asdl_cpp.py | ||
| Turn an ASDL schema into C++ code. | ||
| TODO: | ||
| - Optional fields | ||
| - in osh, it's only used in two places: | ||
| - arith_expr? for slice length | ||
| - word? for var replace | ||
| - So you're already using pointers, can encode the NULL pointer. | ||
| - Change everything to use references instead of pointers? Non-nullable. | ||
| - Unify ClassDefVisitor and MethodBodyVisitor. | ||
| - Whether you need a separate method body should be a flag. | ||
| - offset calculations are duplicated | ||
| - generate a C++ pretty-printer | ||
| Technically we don't even need alignment? I guess the reason is to increase | ||
| address space. If 1, then we have 16MiB of code. If 4, then we have 64 MiB. | ||
| Everything is decoded on the fly, or is a char*, which I don't think has to be | ||
| aligned (because the natural alignment woudl be 1 byte anyway.) | ||
| """ | ||
| import sys | ||
| from asdl import asdl_ as asdl | ||
| from asdl import encode | ||
| from asdl import visitor | ||
| from osh.meta import Id | ||
| class ChainOfVisitors: | ||
| def __init__(self, *visitors): | ||
| self.visitors = visitors | ||
| def VisitModule(self, module): | ||
| for v in self.visitors: | ||
| v.VisitModule(module) | ||
| _BUILTINS = { | ||
| 'string': 'char*', # A read-only string is a char* | ||
| 'int': 'int', | ||
| 'bool': 'bool', | ||
| 'id': 'Id', # Application specific hack for now | ||
| } | ||
| class ForwardDeclareVisitor(visitor.AsdlVisitor): | ||
| """Print forward declarations. | ||
| ASDL allows forward references of types, but C++ doesn't. | ||
| """ | ||
| def VisitCompoundSum(self, sum, name, depth): | ||
| self.Emit("class %(name)s_t;" % locals(), depth) | ||
| def VisitProduct(self, product, name, depth): | ||
| self.Emit("class %(name)s_t;" % locals(), depth) | ||
| def EmitFooter(self): | ||
| self.Emit("", 0) # blank line | ||
| class ClassDefVisitor(visitor.AsdlVisitor): | ||
| """Generate C++ classes and type-safe enums.""" | ||
| def __init__(self, f, enc_params, type_lookup, enum_types=None): | ||
| visitor.AsdlVisitor.__init__(self, f) | ||
| self.ref_width = enc_params.ref_width | ||
| self.type_lookup = type_lookup | ||
| self.enum_types = enum_types or {} | ||
| self.pointer_type = enc_params.pointer_type | ||
| self.footer = [] # lines | ||
| def _GetCppType(self, field): | ||
| """Return a string for the C++ name of the type.""" | ||
| type_name = field.type | ||
| cpp_type = _BUILTINS.get(type_name) | ||
| if cpp_type is not None: | ||
| return cpp_type | ||
| typ = self.type_lookup.ByTypeName(type_name) | ||
| if isinstance(typ, asdl.Sum) and asdl.is_simple(typ): | ||
| # Use the enum instead of the class. | ||
| return "%s_e" % type_name | ||
| # - Pointer for optional type. | ||
| # - ints and strings should generally not be optional? We don't have them | ||
| # in osh yet, so leave it out for now. | ||
| if field.opt: | ||
| return "%s_t*" % type_name | ||
| return "%s_t&" % type_name | ||
| def EmitFooter(self): | ||
| for line in self.footer: | ||
| self.f.write(line) | ||
| def _EmitEnum(self, sum, name, depth): | ||
| enum = [] | ||
| for i in range(len(sum.types)): | ||
| type = sum.types[i] | ||
| enum.append("%s = %d" % (type.name, i + 1)) # zero is reserved | ||
| self.Emit("enum class %s_e : uint8_t {" % name, depth) | ||
| self.Emit(", ".join(enum), depth + 1) | ||
| self.Emit("};", depth) | ||
| self.Emit("", depth) | ||
| def VisitSimpleSum(self, sum, name, depth): | ||
| self._EmitEnum(sum, name, depth) | ||
| def VisitCompoundSum(self, sum, name, depth): | ||
| # This is a sign that Python needs string interpolation!!! | ||
| def Emit(s, depth=depth): | ||
| self.Emit(s % sys._getframe(1).f_locals, depth) | ||
| self._EmitEnum(sum, name, depth) | ||
| Emit("class %(name)s_t : public Obj {") | ||
| Emit(" public:") | ||
| # All sum types have a tag | ||
| Emit("%(name)s_e tag() const {", depth + 1) | ||
| Emit("return static_cast<%(name)s_e>(bytes_[0]);", depth + 2) | ||
| Emit("}", depth + 1) | ||
| Emit("};") | ||
| Emit("") | ||
| # TODO: This should be replaced with a call to the generic | ||
| # self.VisitChildren() | ||
| super_name = "%s_t" % name | ||
| for t in sum.types: | ||
| self.VisitConstructor(t, super_name, depth) | ||
| # rudimentary attribute handling | ||
| for field in sum.attributes: | ||
| type = str(field.type) | ||
| assert type in asdl.builtin_types, type | ||
| Emit("%s %s;" % (type, field.name), depth + 1) | ||
| def VisitConstructor(self, cons, def_name, depth): | ||
| #print(dir(cons)) | ||
| if cons.fields: | ||
| self.Emit("class %s : public %s {" % (cons.name, def_name), depth) | ||
| self.Emit(" public:", depth) | ||
| offset = 1 # for the ID | ||
| for f in cons.fields: | ||
| self.VisitField(f, cons.name, offset, depth + 1) | ||
| offset += self.ref_width | ||
| self.Emit("};", depth) | ||
| self.Emit("", depth) | ||
| def VisitProduct(self, product, name, depth): | ||
| self.Emit("class %(name)s_t : public Obj {" % locals(), depth) | ||
| self.Emit(" public:", depth) | ||
| offset = 0 | ||
| for f in product.fields: | ||
| type_name = '%s_t' % name | ||
| self.VisitField(f, type_name, offset, depth + 1) | ||
| offset += self.ref_width | ||
| for field in product.attributes: | ||
| # rudimentary attribute handling | ||
| type = str(field.type) | ||
| assert type in asdl.builtin_types, type | ||
| self.Emit("%s %s;" % (type, field.name), depth + 1) | ||
| self.Emit("};", depth) | ||
| self.Emit("", depth) | ||
| def VisitField(self, field, type_name, offset, depth): | ||
| """ | ||
| Even though they are inline, some of them can't be in the class {}, because | ||
| static_cast<> requires inheritance relationships to be already declared. We | ||
| have to print all the classes first, then all the bodies that might use | ||
| static_cast<>. | ||
| http://stackoverflow.com/questions/5808758/why-is-a-static-cast-from-a-pointer-to-base-to-a-pointer-to-derived-invalid | ||
| """ | ||
| ctype = self._GetCppType(field) | ||
| name = field.name | ||
| pointer_type = self.pointer_type | ||
| # Either 'left' or 'BoolBinary::left', depending on whether it's inline. | ||
| # Mutated later. | ||
| maybe_qual_name = name | ||
| func_proto = None | ||
| func_header = None | ||
| body_line1 = None | ||
| inline_body = None | ||
| if field.seq: # Array/repeated | ||
| # For size accessor, follow the ref, and then it's the first integer. | ||
| size_header = ( | ||
| 'inline int %(name)s_size(const %(pointer_type)s* base) const {') | ||
| size_body = "return Ref(base, %(offset)d).Int(0);" | ||
| self.Emit(size_header % locals(), depth) | ||
| self.Emit(size_body % locals(), depth + 1) | ||
| self.Emit("}", depth) | ||
| ARRAY_OFFSET = 'int a = (index+1) * 3;' | ||
| A_POINTER = ( | ||
| 'inline const %(ctype)s %(maybe_qual_name)s(' | ||
| 'const %(pointer_type)s* base, int index) const') | ||
| if ctype in ('bool', 'int'): | ||
| func_header = A_POINTER + ' {' | ||
| body_line1 = ARRAY_OFFSET | ||
| inline_body = 'return Ref(base, %(offset)d).Int(a);' | ||
| elif ctype.endswith('_e') or ctype in self.enum_types: | ||
| func_header = A_POINTER + ' {' | ||
| body_line1 = ARRAY_OFFSET | ||
| inline_body = ( | ||
| 'return static_cast<const %(ctype)s>(Ref(base, %(offset)d).Int(a));') | ||
| elif ctype == 'char*': | ||
| func_header = A_POINTER + ' {' | ||
| body_line1 = ARRAY_OFFSET | ||
| inline_body = 'return Ref(base, %(offset)d).Str(base, a);' | ||
| else: | ||
| # Write function prototype now; write body later. | ||
| func_proto = A_POINTER + ';' | ||
| maybe_qual_name = '%s::%s' % (type_name, name) | ||
| func_def = A_POINTER + ' {' | ||
| # This static_cast<> (downcast) causes problems if put within "class | ||
| # {}". | ||
| func_body = ( | ||
| 'return static_cast<const %(ctype)s>(' | ||
| 'Ref(base, %(offset)d).Ref(base, a));') | ||
| self.footer.extend(visitor.FormatLines(func_def % locals(), 0)) | ||
| self.footer.extend(visitor.FormatLines(ARRAY_OFFSET, 1)) | ||
| self.footer.extend(visitor.FormatLines(func_body % locals(), 1)) | ||
| self.footer.append('}\n\n') | ||
| maybe_qual_name = name # RESET for later | ||
| else: # not repeated | ||
| SIMPLE = "inline %(ctype)s %(maybe_qual_name)s() const {" | ||
| POINTER = ( | ||
| 'inline const %(ctype)s %(maybe_qual_name)s(' | ||
| 'const %(pointer_type)s* base) const') | ||
| if ctype in ('bool', 'int'): | ||
| func_header = SIMPLE | ||
| inline_body = 'return Int(%(offset)d);' | ||
| elif ctype.endswith('_e') or ctype in self.enum_types: | ||
| func_header = SIMPLE | ||
| inline_body = 'return static_cast<const %(ctype)s>(Int(%(offset)d));' | ||
| elif ctype == 'char*': | ||
| func_header = POINTER + " {" | ||
| inline_body = 'return Str(base, %(offset)d);' | ||
| else: | ||
| # Write function prototype now; write body later. | ||
| func_proto = POINTER + ";" | ||
| maybe_qual_name = '%s::%s' % (type_name, name) | ||
| func_def = POINTER + ' {' | ||
| if field.opt: | ||
| func_body = ( | ||
| 'return static_cast<const %(ctype)s>(Optional(base, %(offset)d));') | ||
| else: | ||
| func_body = ( | ||
| 'return static_cast<const %(ctype)s>(Ref(base, %(offset)d));') | ||
| # depth 0 for bodies | ||
| self.footer.extend(visitor.FormatLines(func_def % locals(), 0)) | ||
| self.footer.extend(visitor.FormatLines(func_body % locals(), 1)) | ||
| self.footer.append('}\n\n') | ||
| maybe_qual_name = name # RESET for later | ||
| if func_proto: | ||
| self.Emit(func_proto % locals(), depth) | ||
| else: | ||
| self.Emit(func_header % locals(), depth) | ||
| if body_line1: | ||
| self.Emit(body_line1, depth + 1) | ||
| self.Emit(inline_body % locals(), depth + 1) | ||
| self.Emit("}", depth) | ||
| # Used by osh/ast_gen.py | ||
| class CEnumVisitor(visitor.AsdlVisitor): | ||
| def VisitSimpleSum(self, sum, name, depth): | ||
| # Just use #define, since enums aren't namespaced. | ||
| for i, variant in enumerate(sum.types): | ||
| self.Emit('#define %s__%s %d' % (name, variant.name, i + 1), depth) | ||
| self.Emit("", depth) | ||
| def main(argv): | ||
| try: | ||
| action = argv[1] | ||
| except IndexError: | ||
| raise RuntimeError('Action required') | ||
| # TODO: Also generate a switch/static_cast<> pretty printer in C++! For | ||
| # debugging. Might need to detect cycles though. | ||
| if action == 'cpp': | ||
| schema_path = argv[2] | ||
| app_types = {'id': asdl.UserType(Id)} | ||
| with open(schema_path) as input_f: | ||
| module, type_lookup = asdl.LoadSchema(input_f, app_types) | ||
| # TODO: gen_cpp.py should be a library and the application should add Id? | ||
| # Or we should enable ASDL metaprogramming, and let Id be a metaprogrammed | ||
| # simple sum type. | ||
| f = sys.stdout | ||
| # How do mutation of strings, arrays, etc. work? Are they like C++ | ||
| # containers, or their own? I think they mirror the oil language | ||
| # semantics. | ||
| # Every node should have a mirror. MutableObj. MutableRef (pointer). | ||
| # MutableArithVar -- has std::string. The mirrors are heap allocated. | ||
| # All the mutable ones should support Dump()/Encode()? | ||
| # You can just write more at the end... don't need to disturb existing | ||
| # nodes? Rewrite pointers. | ||
| alignment = 4 | ||
| enc = encode.Params(alignment) | ||
| d = {'pointer_type': enc.pointer_type} | ||
| f.write("""\ | ||
| #include <cstdint> | ||
| class Obj { | ||
| public: | ||
| // Decode a 3 byte integer from little endian | ||
| inline int Int(int n) const; | ||
| inline const Obj& Ref(const %(pointer_type)s* base, int n) const; | ||
| inline const Obj* Optional(const %(pointer_type)s* base, int n) const; | ||
| // NUL-terminated | ||
| inline const char* Str(const %(pointer_type)s* base, int n) const; | ||
| protected: | ||
| uint8_t bytes_[1]; // first is ID; rest are a payload | ||
| }; | ||
| """ % d) | ||
| # Id should be treated as an enum. | ||
| c = ChainOfVisitors( | ||
| ForwardDeclareVisitor(f), | ||
| ClassDefVisitor(f, enc, type_lookup, enum_types=['Id'])) | ||
| c.VisitModule(module) | ||
| f.write("""\ | ||
| inline int Obj::Int(int n) const { | ||
| return bytes_[n] + (bytes_[n+1] << 8) + (bytes_[n+2] << 16); | ||
| } | ||
| inline const Obj& Obj::Ref(const %(pointer_type)s* base, int n) const { | ||
| int offset = Int(n); | ||
| return reinterpret_cast<const Obj&>(base[offset]); | ||
| } | ||
| inline const Obj* Obj::Optional(const %(pointer_type)s* base, int n) const { | ||
| int offset = Int(n); | ||
| if (offset) { | ||
| return reinterpret_cast<const Obj*>(base + offset); | ||
| } else { | ||
| return nullptr; | ||
| } | ||
| } | ||
| inline const char* Obj::Str(const %(pointer_type)s* base, int n) const { | ||
| int offset = Int(n); | ||
| return reinterpret_cast<const char*>(base + offset); | ||
| } | ||
| """ % d) | ||
| # uint32_t* and char*/Obj* aren't related, so we need to use | ||
| # reinterpret_cast<>. | ||
| # http://stackoverflow.com/questions/10151834/why-cant-i-static-cast-between-char-and-unsigned-char | ||
| else: | ||
| raise RuntimeError('Invalid action %r' % action) | ||
| if __name__ == '__main__': | ||
| try: | ||
| main(sys.argv) | ||
| except RuntimeError as e: | ||
| print('FATAL: %s' % e, file=sys.stderr) | ||
| sys.exit(1) |
| @@ -0,0 +1,135 @@ | ||
| #!/usr/bin/env python | ||
| """ | ||
| gen_python.py | ||
| Generate Python code from and ASDL schema. | ||
| TODO: | ||
| - What about Id? app_types? | ||
| """ | ||
| import sys | ||
| from asdl import asdl_ as asdl | ||
| from asdl import visitor | ||
| class GenClassesVisitor(visitor.AsdlVisitor): | ||
| def VisitSimpleSum(self, sum, name, depth): | ||
| self.Emit('class %s_e(py_meta.SimpleObj):' % name, depth) | ||
| self.Emit(' ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % name, depth) | ||
| self.Emit('', depth) | ||
| # Just use #define, since enums aren't namespaced. | ||
| for i, variant in enumerate(sum.types): | ||
| attr = '%s_e.%s = %s_e(%d, %r)' % ( | ||
| name, variant.name, name, i + 1, variant.name) | ||
| self.Emit(attr, depth) | ||
| self.Emit('', depth) | ||
| def _GenClass(self, desc, name, super_name, depth, tag_num=None): | ||
| self.Emit('class %s(%s):' % (name, super_name), depth) | ||
| if tag_num is not None: | ||
| self.Emit(' tag = %d' % tag_num, depth) | ||
| field_names = [f.name for f in desc.fields] | ||
| quoted_fields = repr(tuple(field_names)) | ||
| # NOTE: FIELDS is a duplicate of __slots__, used for pretty printing and | ||
| # oheap serialization. TODO: measure the effect of __slots__, and then get | ||
| # rid of FIELDS? Or you can just make it an alias. | ||
| # FIELDS = self.__slots__. | ||
| self.Emit(' ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % name, depth) | ||
| self.Emit(' __slots__ = %s' % quoted_fields, depth) | ||
| self.Emit('', depth) | ||
| # TODO: leave out spids? Mark it as an attribute? | ||
| args = ', '.join('%s=None' % f.name for f in desc.fields) | ||
| self.Emit(' def __init__(self, %s):' % args, depth) | ||
| for f in desc.fields: | ||
| # This logic is like _MakeFieldDescriptors | ||
| default = None | ||
| if f.opt: # Maybe | ||
| if f.type == 'int': | ||
| default = 'const.NO_INTEGER' | ||
| elif f.type == 'string': | ||
| default = "''" | ||
| else: | ||
| default = 'None' | ||
| elif f.seq: # Array | ||
| default = '[]' | ||
| # PROBLEM: Optional ints can't be zero! | ||
| # self.span_id = span_id or const.NO_INTEGER | ||
| # I don't want to add if statements checking against None? | ||
| # For now don't use optional ints. We don't need it. | ||
| default_str = (' or %s' % default) if default else '' | ||
| self.Emit(' self.%s = %s%s' % (f.name, f.name, default_str), depth) | ||
| self.Emit('', depth) | ||
| def VisitConstructor(self, cons, def_name, tag_num, depth): | ||
| if cons.fields: | ||
| self._GenClass(cons, cons.name, def_name, depth, tag_num=tag_num) | ||
| else: | ||
| self.Emit("class %s(%s):" % (cons.name, def_name), depth) | ||
| self.Emit(' ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % cons.name, depth) | ||
| self.Emit(' tag = %d' % tag_num, depth) | ||
| self.Emit('', depth) | ||
| def VisitCompoundSum(self, sum, name, depth): | ||
| # define command_e | ||
| self.Emit('class %s_e(object):' % name, depth) | ||
| for i, variant in enumerate(sum.types): | ||
| self.Emit(' %s = %d' % (variant.name, i + 1), depth) | ||
| self.Emit('', depth) | ||
| self.Emit('class %s(py_meta.CompoundObj):' % name, depth) | ||
| self.Emit(' ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % name, depth) | ||
| self.Emit('', depth) | ||
| # define command_t, and then make subclasses | ||
| super_name = '%s' % name | ||
| for i, t in enumerate(sum.types): | ||
| tag_num = i + 1 | ||
| self.VisitConstructor(t, super_name, tag_num, depth) | ||
| def VisitProduct(self, product, name, depth): | ||
| self._GenClass(product, name, 'py_meta.CompoundObj', depth) | ||
| def EmitFooter(self): | ||
| pass | ||
| def main(argv): | ||
| schema_path = argv[1] | ||
| type_lookup_import = argv[2] | ||
| with open(schema_path) as input_f: | ||
| module = asdl.parse(input_f) | ||
| f = sys.stdout | ||
| f.write("""\ | ||
| from asdl import const # For const.NO_INTEGER | ||
| from asdl import py_meta | ||
| %s | ||
| """ % type_lookup_import) | ||
| v = GenClassesVisitor(f) | ||
| v.VisitModule(module) | ||
| if __name__ == '__main__': | ||
| try: | ||
| main(sys.argv) | ||
| except RuntimeError as e: | ||
| print >>sys.stderr, 'FATAL: %s' % e | ||
| sys.exit(1) |