You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
16 lines
320 B
Racket
16 lines
320 B
Racket
#lang racket/base
|
|
(require parser-tools/lex ragg/support)
|
|
(provide tokenize)
|
|
|
|
(define (tokenize ip)
|
|
(port-count-lines! ip)
|
|
|
|
(define lex
|
|
(lexer
|
|
[(char-set "><-.,+[]") lexeme]
|
|
[whitespace (token 'white #:skip? #t)]
|
|
[(eof) (void)]))
|
|
|
|
(define next-token-func (λ _ (lex ip)))
|
|
next-token-func)
|