You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
15 lines
341 B
Racket
15 lines
341 B
Racket
#lang racket/base
|
|
(require parser-tools/lex ragg/support racket/function)
|
|
(provide tokenize lex)
|
|
|
|
(define lex
|
|
(lexer-src-pos
|
|
[(char-set "><-.,+[]") lexeme]
|
|
[whitespace (token '_ lexeme #:skip? #t)]
|
|
[(eof) (void)]))
|
|
|
|
(define (tokenize ip)
|
|
(port-count-lines! ip)
|
|
(define next-token-thunk (thunk (lex ip)))
|
|
next-token-thunk)
|