Import of my ASDL fork from the sketch repo.

It dynamically generates Python metaclasses, walks them to produce a
binary encoding, and generates C++ source code that can read this
encoding (lazily).

There is some duplicate demo code inarith_parse.py and tdop.py.

asdl/arith.asdl

@@ -0,0 +1,36 @@
module arith {
  op_id = Plus | Minus | Star

  -- why does source_location have to be lower case?  It's not a constructor?
  -- it has no tag?
  source_location = (string path, int line, int col, int length)

  -- TODO:
  -- - Add optional.  For slicing maybe -- optional end.
  -- - Add repeated.  For function call maybe.

  arith_expr =
    Const(int i)  -- 
  | ArithVar(string name)  -- eval variable 
  | ArithUnary(op_id op_id, arith_expr a)
  | ArithBinary(op_id op_id, arith_expr left, arith_expr right)
  | FuncCall(string name, arith_expr* args)
  | ForwardRef(bool_expr b)
  | Index(arith_expr a, arith_expr index)
    -- Using Python's style for now.  Bash uses length instead of end.
  | Slice(arith_expr a, arith_expr? begin, arith_expr? end, arith_expr? stride)

  -- Hm these have to be primitives.  Maybe it should just be a location index.
  -- But honestly that is the same as a pointer in the heap pattern.
  --attributes (source_location loc)

  word = (string value)

  bool_expr =
    BoolBinary(word left, word right)
  | BoolUnary(word child)
  | LogicalNot(bool_expr b)
  | LogicalAnd(bool_expr left, bool_expr right)
  | LogicalOr(bool_expr left, bool_expr right)
}

asdl/arith_ast.py

@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""
arith_ast.py
"""

import os
import sys

import asdl
import py_meta

this_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
schema_path = os.path.join(this_dir, 'arith.asdl')

module = asdl.parse(schema_path)
root = sys.modules[__name__]
py_meta.MakeTypes(module, root)

asdl/arith_ast_test.py

@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""
arith_ast_test.py: Tests for arith_ast.py
"""

import unittest

import arith_ast  # module under test
import py_meta


class ArithAstTest(unittest.TestCase):

  def testTypes(self):
    ArithVar = arith_ast.ArithVar
    ArithUnary = arith_ast.ArithUnary
    ArithBinary = arith_ast.ArithBinary
    Const = arith_ast.Const
    Slice = arith_ast.Slice
    arith_expr = arith_ast.arith_expr
    source_location = arith_ast.source_location
    op_id = arith_ast.op_id

    print(ArithVar)
    print('FIELDS', ArithVar.FIELDS)
    print('DESCRIPTOR_LOOKUP', ArithVar.DESCRIPTOR_LOOKUP)
    print('DESCRIPTOR', ArithVar.DESCRIPTOR)

    print(ArithUnary)
    print(ArithUnary.FIELDS)

    print(ArithBinary)
    print(ArithBinary.FIELDS)

    n1 = ArithVar('x')
    n2 = ArithVar(name='y')
    print(n1)
    print(n2)

    # Not good because not assigned?
    n3 = ArithVar()

    # NOTE: You cannot instantiate a product type directly?  It's just used for
    # type checking.  What about OCaml?
    # That means you just need to create classes for the records (Constructor).
    # They all descend from Obj.  They don't need

    n3 = ArithVar()
    try:
      n4 = ArithVar('x', name='X')
    except AssertionError as e:
      pass
    else:
      raise AssertionError("Should have failed")

    #n5 = ArithVar(None)

    print(source_location())
    
    c = Const(66)
    print(c)
    # Test out hierarchy
    assert isinstance(c, Const)
    assert isinstance(c, arith_expr)
    assert isinstance(c, py_meta.CompoundObj)

    #print(Const('invalid'))

    print(Slice(Const(1), Const(5), Const(2)))

    print(op_id.Plus)

    # Class for sum type
    print(arith_expr)

    # Invalid because only half were assigned
    #print(ArithBinary(op_id.Plus, Const(5)))

    n = ArithBinary()
    #n.CheckUnassigned()
    n.op_id = op_id.Plus
    n.left = Const(5)
    #n.CheckUnassigned()
    n.right = Const(6)
    n.CheckUnassigned()


if __name__ == '__main__':
  unittest.main()

asdl/arith_demo.cc

@@ -0,0 +1,134 @@
#include <string>
#include <stdio.h>
#include <stdlib.h>

#include "arith.asdl.h"

void PrintExpr(const uint32_t* base, const arith_expr_t& e, int indent) {
  for (int i = 0; i < indent; ++i) {
    putchar('\t');
  }
  printf("t%hhu ", e.tag());

  switch (e.tag()) {
  case arith_expr_e::Const: {
    auto& e2 = static_cast<const Const&>(e);
    printf("CONST %d\n", e2.i());
    break;
  }
  case arith_expr_e::ArithVar: {
    auto& e2 = static_cast<const ArithVar&>(e);
    printf("VAR %s\n", e2.name(base));
    break;
  }
  case arith_expr_e::ArithUnary: {
    auto& e2 = static_cast<const ArithUnary&>(e);
    printf("UNARY\n");
    break;
  }
  case arith_expr_e::ArithBinary: {
    auto& e2 = static_cast<const ArithBinary&>(e);
    printf("BINARY\n");
    for (int i = 0; i < indent+1; ++i) {
      putchar('\t');
    }
    // TODO: 
    // char* DebugString(op_id_e op_id)
    //printf("INT %d\n", e2.Int(1));
    printf("%hhu\n", e2.op_id());

    PrintExpr(base, e2.left(base), indent+1);
    PrintExpr(base, e2.right(base), indent+1);
    break;
  }
  case arith_expr_e::FuncCall: {
    auto& e2 = static_cast<const FuncCall&>(e);
    printf("FUNC CALL\n");
    for (int i = 0; i < indent+1; ++i) {
      putchar('\t');
    }
    printf("name %s\n", e2.name(base));
    for (int i = 0; i < e2.args_size(base); ++i) {
      PrintExpr(base, e2.args(base, i), indent+1);
    }
    break;
  }
  case arith_expr_e::Index: {
    auto& e2 = static_cast<const Index&>(e);
    printf("INDEX\n");
    PrintExpr(base, e2.a(base), indent+1);
    PrintExpr(base, e2.index(base), indent+1);
    break;
  }
  case arith_expr_e::Slice: {
    auto& e2 = static_cast<const Slice&>(e);
    printf("SLICE\n");
    const arith_expr_t* begin = e2.begin(base);
    const arith_expr_t* end = e2.end(base);
    const arith_expr_t* stride = e2.stride(base);
    PrintExpr(base, e2.a(base), indent+1);
    if (begin) PrintExpr(base, *begin, indent+1);
    if (end) PrintExpr(base, *end, indent+1);
    if (stride) {
      PrintExpr(base, *stride, indent+1);
    } else {
      for (int i = 0; i < indent+1; ++i) {
        putchar('\t');
      }
      printf("stride: %p\n", stride);
    }
    break;
  }
  default:
    printf("OTHER\n");
    break;
  }
}

// Returns the root ref, or -1 for invalid
int GetRootRef(uint8_t* image) {
  if (image[0] != 'O') return -1;
  if (image[1] != 'H') return -1;
  if (image[2] != 'P') return -1;
  if (image[3] != 1) return -1;  // version 1
  if (image[4] != 4) return -1;  // alignment 4

  return image[5] + (image[6] << 8) + (image[7] << 16);
}

int main(int argc, char **argv) {
  if (argc == 0) {
    printf("Expected filename\n");
    return 1;
  }
  FILE *f = fopen(argv[1], "rb");
  if (!f) {
    printf("Error opening %s", argv[1]);
    return 1;
  }
  fseek(f, 0, SEEK_END);
  size_t num_bytes = ftell(f);
  fseek(f, 0, SEEK_SET);  //same as rewind(f);

  uint8_t* image = static_cast<uint8_t*>(malloc(num_bytes + 1));
  fread(image, num_bytes, 1, f);
  fclose(f);

  image[num_bytes] = 0;
  printf("Read %zu bytes\n", num_bytes);

  int root_ref = GetRootRef(image);
  if (root_ref == -1) {
    printf("Invalid image\n");
    return 1;
  }
  // Hm we could make the root ref be a BYTE offset?
  int alignment = 4;
  printf("alignment: %d root: %d\n", alignment, root_ref);

  auto base = reinterpret_cast<uint32_t*>(image);

  size_t offset = alignment * root_ref;
  auto expr = reinterpret_cast<arith_expr_t*>(image + offset);
  PrintExpr(base, *expr, 0);
}