Permalink
Browse files

updating `t_lex` for modern irken.

  • Loading branch information...
samrushing committed Mar 15, 2017
1 parent 2d504b5 commit 9145c2e9590955945e61f40f2461026dc1215803
Showing with 28 additions and 30 deletions.
  1. +2 −2 parse/lexer.py
  2. +26 −28 tests/t_lex.scm
@@ -197,13 +197,13 @@ def gen_irken (self, file):
for t in tables:
indirect.append (unique[t])
W ("(define dfa \n")
W (" (let* (\n")
W (" (let (\n")
items = unique.items()
items.sort (lambda a,b: cmp (a[1], b[1]))
for table, index in items:
W (" (t%d \"%s\")\n" % (index, ''.join (["\\x%02x" % (ord (x),) for x in table])))
W (" )\n")
W (" #(%s)))\n" % (" ".join (["t%d" % (x,) for x in indirect])))
W (" (list->vector (LIST %s))))\n" % (" ".join (["t%d" % (x,) for x in indirect])))
# find the sink state
sink = self.find_sink()
@@ -13,38 +13,38 @@
(define eof-token (token:t 'eof "eof"))
;; defines the <step> function (DFA) from the lexer generator
(include "parse/lexstep.scm")
(define (lex producer consumer)
;; producer gives us characters
;; consumer takes tokens
(let ((action 'not-final)
(state 0))
(define (final? action)
(not (eq? action 'not-final)))
;; defines the <step> function (DFA) from the lexer generator
(include "parse/lexstep.scm")
(define (final? action)
(not (eq? action 'not-final)))
(let loop ((ch (producer))
(last 'not-final)
(current (list:nil)))
(cond ((char=? ch #\eof) (consumer eof-token) #t)
(else
(set! state (step ch state))
(set! action finals[state])
(cond ((and (not (final? last)) (final? action))
;; we've entered a new final state
(loop (producer) action (list:cons ch current)))
((and (final? last) (not (final? action)))
;; we've left a final state - longest match - emit token
(consumer (token:t last (list->string (reverse current))))
(set! state 0)
(loop ch 'not-final (list:nil)))
(else
;; accumulate this character
(loop (producer) action (list:cons ch current)))))))
))
(let loop ((ch (producer))
(last 'not-final)
(current (list:nil)))
(cond ((char=? ch #\eof) (consumer eof-token) #t)
(else
(set! state (step ch state))
(set! action finals[state])
(cond ((and (not (final? last)) (final? action))
;; we've entered a new final state
(loop (producer) action (list:cons ch current)))
((and (final? last) (not (final? action)))
;; we've left a final state - longest match - emit token
(consumer (token:t last (list->string (reverse current))))
(set! state 0)
(loop ch 'not-final (list:nil)))
(else
;; accumulate this character
(loop (producer) action (list:cons ch current)))))))
))
(define (make-lex-generator file)
@@ -54,12 +54,10 @@
(make-generator
(lambda (consumer)
(lex producer consumer)
(let forever ()
(consumer eof-token)
(forever))
(forever (consumer eof-token))
)))
(let ((f (file/open-read "nodes.py")))
(let ((f (file/open-read "parse/gen_irken.py")))
(define g (make-lex-generator f))
(let loop ((tok (g)))
(cond ((eq? tok eof-token) "done")

0 comments on commit 9145c2e

Please sign in to comment.