Permalink
Please sign in to comment.
Browse files
Generate Python source code from an ASDL schema.
This is in contrast to using metaprogramming at runtime. OSH works with this new code, and it's a lot faster! It's not enabled yet because a few unit tests still fail, due to lack of __eq__. But the shell runs and passes spec tests. Also: - Decode the output of everything we encoded in the oheap benchmark. Document an idea to both measure decoding speed and collect data about Array/Str lengths. This could help with record sizing, inlining, and other decisions that affect locality. - In ASDL constructors, Raise TypeError instead of AssertionError. This is more consistent with Python itself.
- Loading branch information...
Showing
with
267 additions
and 18 deletions.
- +5 −2 asdl/arith_ast_test.py
- +4 −2 asdl/asdl_demo.py
- +2 −0 asdl/encode.py
- +3 −6 asdl/gen_cpp.py
- +159 −0 asdl/gen_python.py
- +13 −2 asdl/py_meta.py
- +21 −0 asdl/run.sh
- +46 −0 benchmarks/oheap.sh
- +12 −4 osh/ast_.py
- +2 −2 scripts/count.sh
| @@ -0,0 +1,159 @@ | ||
| #!/usr/bin/python | ||
| """ | ||
| gen_python.py | ||
| """ | ||
| import sys | ||
| from asdl import gen_cpp | ||
| from asdl import asdl_ as asdl | ||
| class GenClassesVisitor(gen_cpp.AsdlVisitor): | ||
| # TODO: | ||
| # - __eq__ isn't the same | ||
| # - DESCRIPTOR and FIELDS are dummies right now. | ||
| # - I think FIELDS is used for encoding. | ||
| # | ||
| # - Debug mode: | ||
| # - _CheckType(value, desc) on initialization and __setattr__. | ||
| # - check unassigned. Why is it done with unit tests with CheckUnassigned, | ||
| # but also in _Init? | ||
| def VisitSimpleSum(self, sum, name, depth): | ||
| self.Emit('class %s_e(SimpleObj):' % name, depth) | ||
| self.Emit(' pass', depth) | ||
| self.Emit('', depth) | ||
| # Just use #define, since enums aren't namespaced. | ||
| for i, variant in enumerate(sum.types): | ||
| attr = '%s_e.%s = %s_e(%d, %r)' % ( | ||
| name, variant.name, name, i + 1, variant.name) | ||
| self.Emit(attr, depth) | ||
| self.Emit('', depth) | ||
| def _GenClass(self, desc, name, super_name, depth, tag_num=None, | ||
| add_spids=True): | ||
| self.Emit('class %s(%s):' % (name, super_name), depth) | ||
| if tag_num is not None: | ||
| self.Emit(' tag = %d' % tag_num, depth) | ||
| field_names = [f.name for f in desc.fields] | ||
| if add_spids: | ||
| field_names.append('spids') | ||
| quoted_fields = repr(tuple(field_names)) | ||
| # NOTE: FIELDS is a duplicate of __slots__, used for pretty printing and | ||
| # oheap serialization. TODO: measure the effect of __slots__, and then get | ||
| # rid of FIELDS? Or you can just make it an alias. | ||
| # FIELDS = self.__slots__. | ||
| self.Emit(' FIELDS = %s' % quoted_fields, depth) | ||
| # Dummy values. | ||
| self.Emit(' DESCRIPTOR_LOOKUP = {}', depth) | ||
| self.Emit(' DESCRIPTOR = None', depth) | ||
| self.Emit(' __slots__ = %s' % quoted_fields, depth) | ||
| self.Emit('', depth) | ||
| args = ', '.join('%s=None' % f.name for f in desc.fields) | ||
| self.Emit(' def __init__(self, %s):' % args, depth) | ||
| for f in desc.fields: | ||
| # This logic is like _MakeFieldDescriptors | ||
| default = None | ||
| if f.opt: # Maybe | ||
| if f.type == 'int': | ||
| default = 'const.NO_INTEGER' | ||
| elif f.type == 'string': | ||
| default = "''" | ||
| else: | ||
| default = 'None' | ||
| elif f.seq: # Array | ||
| default = '[]' | ||
| default_str = (' or %s' % default) if default else '' | ||
| self.Emit(' self.%s = %s%s' % (f.name, f.name, default_str), depth) | ||
| # Like add_spids in _MakeFieldDescriptors. TODO: This should be optional | ||
| # for token and span! Also for runtime.asdl. We need to make it optional. | ||
| if add_spids: | ||
| self.Emit(' self.spids = []', depth) | ||
| self.Emit('', depth) | ||
| self.Emit(' def CheckUnassigned(self):', depth) | ||
| self.Emit(' pass', depth) | ||
| self.Emit('', depth) | ||
| def VisitConstructor(self, cons, def_name, tag_num, depth): | ||
| if cons.fields: | ||
| self._GenClass(cons, cons.name, def_name, depth, tag_num=tag_num) | ||
| else: | ||
| self.Emit("class %s(%s):" % (cons.name, def_name), depth) | ||
| self.Emit(' tag = %d' % tag_num, depth) | ||
| self.Emit('', depth) | ||
| def VisitCompoundSum(self, sum, name, depth): | ||
| # define command_e | ||
| self.Emit('class %s_e(object):' % name, depth) | ||
| for i, variant in enumerate(sum.types): | ||
| self.Emit(' %s = %d' % (variant.name, i + 1), depth) | ||
| self.Emit('', depth) | ||
| self.Emit('class %s(object):' % name, depth) | ||
| self.Emit(' pass', depth) | ||
| self.Emit('', depth) | ||
| # define command_t, and then make subclasses | ||
| super_name = '%s' % name | ||
| for i, t in enumerate(sum.types): | ||
| tag_num = i + 1 | ||
| self.VisitConstructor(t, super_name, tag_num, depth) | ||
| def VisitProduct(self, product, name, depth): | ||
| self._GenClass(product, name, 'object', depth) | ||
| def EmitFooter(self): | ||
| pass | ||
| def main(argv): | ||
| schema_path = argv[1] | ||
| with open(schema_path) as input_f: | ||
| module = asdl.parse(input_f) | ||
| f = sys.stdout | ||
| # For const.NO_INTEGER | ||
| f.write("""\ | ||
| from asdl import const | ||
| # Copied from py_meta | ||
| class Obj(object): | ||
| # NOTE: We're using CAPS for these static fields, since they are constant at | ||
| # runtime after metaprogramming. | ||
| DESCRIPTOR = None # Used for type checking | ||
| class SimpleObj(Obj): | ||
| def __init__(self, enum_id, name): | ||
| self.enum_id = enum_id | ||
| self.name = name | ||
| def __repr__(self): | ||
| return '<%s %s %s>' % (self.__class__.__name__, self.name, self.enum_id) | ||
| """) | ||
| v = GenClassesVisitor(f) | ||
| v.VisitModule(module) | ||
| if __name__ == '__main__': | ||
| try: | ||
| main(sys.argv) | ||
| except RuntimeError as e: | ||
| print >>sys.stderr, 'FATAL: %s' % e | ||
| sys.exit(1) |
0 comments on commit
00b36fa