updating t_lex for modern irken.

stnbu · Mar 15, 2017 · 9145c2e · 9145c2e
1 parent 2d504b5
commit 9145c2e
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 30 deletions.
diff --git a/parse/lexer.py b/parse/lexer.py
@@ -197,13 +197,13 @@ def gen_irken (self, file):
         for t in tables:
             indirect.append (unique[t])
         W ("(define dfa \n")
-        W ("  (let* (\n")
+        W ("  (let (\n")
         items = unique.items()
         items.sort (lambda a,b: cmp (a[1], b[1]))
         for table, index in items:
             W ("         (t%d \"%s\")\n" % (index, ''.join (["\\x%02x" % (ord (x),) for x in table])))
         W ("        )\n")
-        W ("   #(%s)))\n" % (" ".join (["t%d" % (x,) for x in indirect])))
+        W ("   (list->vector (LIST %s))))\n" % (" ".join (["t%d" % (x,) for x in indirect])))
 
         # find the sink state
         sink = self.find_sink()

diff --git a/tests/t_lex.scm b/tests/t_lex.scm
@@ -13,38 +13,38 @@
 
 (define eof-token (token:t 'eof "eof"))
 
+;; defines the <step> function (DFA) from the lexer generator
+(include "parse/lexstep.scm")
+
 (define (lex producer consumer)
   ;; producer gives us characters
   ;; consumer takes tokens
 
   (let ((action 'not-final)
 	(state 0))
 
-      (define (final? action)
-	(not (eq? action 'not-final)))
-
-      ;; defines the <step> function (DFA) from the lexer generator
-      (include "parse/lexstep.scm")
+    (define (final? action)
+      (not (eq? action 'not-final)))
 
-      (let loop ((ch (producer))
-		 (last 'not-final)
-		 (current (list:nil)))
-	(cond ((char=? ch #\eof) (consumer eof-token) #t)
-	      (else
-	       (set! state (step ch state))
-	       (set! action finals[state])
-	       (cond ((and (not (final? last)) (final? action))
-		      ;; we've entered a new final state
-		      (loop (producer) action (list:cons ch current)))
-		     ((and (final? last) (not (final? action)))
-		      ;; we've left a final state - longest match - emit token
-		      (consumer (token:t last (list->string (reverse current))))
-		      (set! state 0)
-		      (loop ch 'not-final (list:nil)))
-		     (else
-		      ;; accumulate this character
-		      (loop (producer) action (list:cons ch current)))))))
-      ))
+    (let loop ((ch (producer))
+               (last 'not-final)
+               (current (list:nil)))
+      (cond ((char=? ch #\eof) (consumer eof-token) #t)
+            (else
+             (set! state (step ch state))
+             (set! action finals[state])
+             (cond ((and (not (final? last)) (final? action))
+                    ;; we've entered a new final state
+                    (loop (producer) action (list:cons ch current)))
+                   ((and (final? last) (not (final? action)))
+                    ;; we've left a final state - longest match - emit token
+                    (consumer (token:t last (list->string (reverse current))))
+                    (set! state 0)
+                    (loop ch 'not-final (list:nil)))
+                   (else
+                    ;; accumulate this character
+                    (loop (producer) action (list:cons ch current)))))))
+    ))
 
 (define (make-lex-generator file)
 
@@ -54,12 +54,10 @@
   (make-generator
    (lambda (consumer)
      (lex producer consumer)
-     (let forever ()
-       (consumer eof-token)
-       (forever))
+     (forever (consumer eof-token))
      )))
 
-(let ((f (file/open-read "nodes.py")))
+(let ((f (file/open-read "parse/gen_irken.py")))
   (define g (make-lex-generator f))
   (let loop ((tok (g)))
     (cond ((eq? tok eof-token) "done")