You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
20 lines
514 B
Racket
20 lines
514 B
Racket
#lang br/quicklang
|
|
(require brag/support)
|
|
|
|
(define (token? x)
|
|
(or (eof-object? x) (string? x) (token-struct? x)))
|
|
|
|
(define/contract (tokenize port)
|
|
(input-port? . -> . (-> token?))
|
|
(define/contract (next-token)
|
|
(-> token?)
|
|
(define our-lexer
|
|
(lexer
|
|
[(eof) eof]
|
|
[(from/to "//" "\n") (next-token)]
|
|
[(from/to "@$" "$@")
|
|
(token 'SEXP-TOK (trim-ends "@$" lexeme "$@"))]
|
|
[any-char (token 'CHAR-TOK lexeme)]))
|
|
(our-lexer port))
|
|
next-token)
|
|
(provide tokenize) |