Improved to AST to facilitate code generation.

Signed-off-by: Tim Henderson <tim.tadh@gmail.com>

scan.py

@@ -20,6 +20,10 @@ class C_Lexer(object):
     tokens = tokens
     literals = literals
 
+    def __init__(self):
+        self.token_stack = []
+        self.next_token = None
+
     comment = r'(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)'
     @TOKEN(comment)
     def t_COMMENT(self, token):
@@ -127,6 +131,15 @@ def t_error(self,t):
     # Build the lexer
     def build(self,**kwargs):
         self.lexer = lex.lex(object=self, **kwargs)
+        def h(self, f, *args, **kwargs):
+            def token(*args, **kwargs):
+                '''A decorator on the original token function'''
+                t = f()
+                self.token_stack.append(self.next_token)
+                self.next_token = t
+                return t
+            return token
+        self.lexer.token = h(self, self.lexer.token)
 
     # Test it output
     def test(self,data):
@@ -137,13 +150,13 @@ def test(self,data):
              print tok
 
 # Build the lexer and try it out
-m = C_Lexer()
-m.build()           # Build the lexer
-lexer = m.lexer
+#m = C_Lexer()
+#m.build()           # Build the lexer
+#lexer = m.lexer
 
 if __name__ == '__main__':
     # Test it out
-    f = open('test3.c', 'r')
+    f = open('test.c', 'r')
     data = f.read()
     f.close()
 

scan.py~

@@ -20,6 +20,10 @@ class C_Lexer(object):
     tokens = tokens
     literals = literals
 
+    def __init__(self):
+        self.token_stack = []
+        self.next_token = None
+
     comment = r'(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)'
     @TOKEN(comment)
     def t_COMMENT(self, token):
@@ -127,6 +131,15 @@ class C_Lexer(object):
     # Build the lexer
     def build(self,**kwargs):
         self.lexer = lex.lex(object=self, **kwargs)
+        def h(self, f, *args, **kwargs):
+            def token(*args, **kwargs):
+                '''A decorator on the original token function'''
+                t = f()
+                self.token = self.next_token
+                self.next_token = t
+                return t
+            return token
+        self.lexer.token = h(self, self.lexer.token)
 
     # Test it output
     def test(self,data):
@@ -137,9 +150,9 @@ class C_Lexer(object):
              print tok
 
 # Build the lexer and try it out
-m = C_Lexer()
-m.build()           # Build the lexer
-lexer = m.lexer
+#m = C_Lexer()
+#m.build()           # Build the lexer
+#lexer = m.lexer
 
 if __name__ == '__main__':
     # Test it out

temp

@@ -0,0 +1,104 @@
+@TOKEN(r'>>=')
+    def t_RIGHT_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'<<=')
+    def t_LEFT_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\+=')
+    def t_ADD_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\-=')
+    def t_SUB_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\*=')
+    def t_MUL_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\/=')
+    def t_DIV_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\%=')
+    def t_MOD_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\&=')
+    def t_AND_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\^=')
+    def t_XOR_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\|=')
+    def t_OR_ASSIGN(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'>>')
+    def t_RIGHT_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'<<')
+    def t_LEFT_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\+\+')
+    def t_INC_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\-\-')
+    def t_DEC_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\->')
+    def t_PTR_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\&\&')
+    def t_AND_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\|\|')
+    def t_OR_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\<=')
+    def t_LE_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\>=')
+    def t_GE_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'==')
+    def t_EQ_OP(self, token):
+        self.token = token
+        return token
+
+@TOKEN(r'\!=')
+    def t_NE_OP(self, token):
+        self.token = token
+        return token

test.c

@@ -1,19 +1,21 @@
-int hello;
+
 //HELLO;
-hello = 010;
-hello = 123;
-hello = 0xff;
-hello = 5e0123;
-hello = .1234e3;
-hello /= hello;
-char c;
-c = 'x';
-char *string;
-string = "this is string\0";
-/*this is a comment
+int hello = 010;
+int hello = 123;
+int hello = 0xff;
+int hello = 5e0123;
+int hello = .1234e3;
+char c = 'x';
+char *string = "this is string";
+/*/*this is a comment
 it goes on for multiple lines
 it has "string" inside it
-hurray*/
+hurray
 int x = 5;
-/*int x = 6;*/
-int x = 7;
+/*int x = 6;
+int x = 7;
+*/
+int main()
+{
+    hello %= hello; 
+}

test.c~

@@ -0,0 +1,21 @@
+
+//HELLO;
+int hello = 010;
+int hello = 123;
+int hello = 0xff;
+int hello = 5e0123;
+int hello = .1234e3;
+char c = 'x';
+char *string = "this is string";
+/*this is a comment
+it goes on for multiple lines
+it has "string" inside it
+hurray*/
+int x = 5;
+/*int x = 6;*/
+int x = 7;
+
+int main()
+{
+    hello %= hello; 
+}

test3.c

@@ -1,4 +1,4 @@
 int main()
 {
-    int flags;
+    return 0;
 }

test3.c~

@@ -1,4 +1,4 @@
 int main()
 {
-    goto flags;
+    int flags;
 }

visualize.py

@@ -1,5 +1,6 @@
 import gram
 import subprocess
+import sys
 
 header_string =         "digraph %s {\n    node [shape=box];\n"
 subgraph_prefix =       "    {\n        rank=same; \n"
@@ -63,10 +64,7 @@ def ast_dot(root, path, name="AST"):
 if __name__ == "__main__":
     c_parser = gram.C_Parser()
     c_parser.build()
-    f = open('test2.c', 'r')
-    data = f.read()
-    f.close()
-    top = c_parser.parse(data)
+    top = c_parser.parse(sys.stdin.read())
     ast_dot(top, "AST.dot")
     print "Running dot..."
     popen_obj = subprocess.Popen(["dot", "-Tpdf",  "AST.dot",  "-o",  "AST.pdf"])

visualize.py~

@@ -0,0 +1,69 @@
+import gram
+import subprocess
+import sys
+
+header_string =         "digraph %s {\n    node [shape=box];\n"
+subgraph_prefix =       "    {\n        rank=same; \n"
+node_string_nocolor =   "        %s [label=%s];\n"
+node_string_color =     "        %s [label=%s, style=filled, fillcolor=\"%s\"];\n"
+subgraph_postfix =      "    }\n"
+edge_string =           "    %s -> %s;\n"
+footer_string =         "}\n"
+
+class Node(object):
+    """Example node with the proper attributes."""
+    def __init__(self, string, children=[]):
+        super(Node, self).__init__()
+        self.graph_id = ""
+        self.string = string
+        self.children = children
+
+    def __str__(self):
+        return self.string
+
+node_count = 0
+
+def ast_walk_tree(node, rank, subgraph_list=[]):
+    global node_count
+    node_count += 1
+    node.graph_id = str(node_count)
+
+    if len(subgraph_list)-1 < rank:
+        subgraph_list.append([])
+
+    subgraph_list[rank].append(node)
+
+    for child in node.children:
+        ast_walk_tree(child, rank+1, subgraph_list)
+
+    return subgraph_list
+
+def ast_dot(root, f, name="AST"):
+    f.write(header_string % name)
+
+    global node_count
+    node_count = 0
+
+    subgraph_list = ast_walk_tree(root, 0, [])
+    for subgraph in subgraph_list:
+        f.write(subgraph_prefix)
+        for node in subgraph:
+            if hasattr(node,"graph_color"):
+                f.write(node_string_color % (node.graph_id, str(node), node.graph_color))
+            else:
+                f.write(node_string_nocolor % (node.graph_id, str(node)))
+        f.write(subgraph_postfix)
+
+    for subgraph in subgraph_list:
+        for node in subgraph:
+            for child in node.children:
+                f.write(edge_string % (node.graph_id, child.graph_id))
+    f.write(footer_string)
+
+if __name__ == "__main__":
+    c_parser = gram.C_Parser()
+    c_parser.build()
+    top = c_parser.parse(sys.stdin.read())
+    ast_dot(top, sys.stdout)
+    print "Running dot..."
+    popen_obj = subprocess.Popen(["dot", "-Tpdf",  "AST.dot",  "-o",  "AST.pdf"])