You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
beautiful-racket/br-bf/tokenizer.rkt

16 lines
320 B
Racket

#lang racket/base
(require parser-tools/lex ragg/support)
(provide tokenize)
(define (tokenize ip)
(port-count-lines! ip)
(define lex
(lexer
[(char-set "><-.,+[]") lexeme]
[whitespace (token 'white #:skip? #t)]
[(eof) (void)]))
(define next-token-func (λ _ (lex ip)))
next-token-func)