You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
24 lines
512 B
Racket
24 lines
512 B
Racket
9 years ago
|
#lang racket/base
|
||
|
|
(require parser-tools/lex ragg/support)
|
||
|
|
(provide tokenize)
|
||
|
|
|
||
|
|
(define (tokenize ip)
|
||
|
|
(port-count-lines! ip)
|
||
|
|
|
||
|
|
(define my-lexer
|
||
|
|
(lexer-src-pos
|
||
|
|
[(repetition 1 +inf.0 numeric)
|
||
|
|
(token 'INTEGER (string->number lexeme))]
|
||
|
|
[upper-case
|
||
|
|
(token 'STRING lexeme)]
|
||
|
|
["b"
|
||
|
|
(token 'STRING " ")]
|
||
|
|
[";"
|
||
|
|
(token ";" lexeme)]
|
||
|
|
[whitespace
|
||
|
|
(token 'WHITESPACE lexeme #:skip? #t)]
|
||
|
|
[(eof)
|
||
|
|
(void)]))
|
||
|
|
|
||
|
|
(define (next-token) (my-lexer ip))
|
||
|
|
next-token)
|