pyparse-suffix.rkt

) ; grammar

) ; parser

) ; pyparse


(define (token-generator)
  (define next (read))

  (match next
    [(? eof-object?)
      (token-EOF)]

    ; ID:
    [`(ID ,name)
      (token-NAME name)]

    ; KEYWORD:
    [`(KEYWORD ,symbol)
      symbol]

    ; PUNCT:
    [`(PUNCT ,p)
      ; BUG: This will fail if we ever include positional info:
      (string->symbol p)]

    ; SPECIAL:
    ['(NEWLINE)
      (token-NEWLINE)]

    ['(INDENT)
      (token-INDENT)]

    ['(DEDENT)
      (token-DEDENT)]

    ['(ENDMARKER)
      (token-ENDMARKER)]

    ; LIT:
    [`(LIT ,(and value (? number?)))
      (token-NUMBER value)]

    [`(LIT ,(and value (or (? bytes?) (? string?))))
      (token-STRING value)]

      ))

(pretty-write (pyparse token-generator))