You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
beautiful-racket/beautiful-racket-demo/basic-demo/tokenizer.rkt

46 lines
1.4 KiB
Racket

#lang br
8 years ago
(require parser-tools/lex
(prefix-in : parser-tools/lex-sre)
brag/support)
(provide (all-defined-out))
8 years ago
(define basic-lexer
(lexer-src-pos
[(eof) eof]
[whitespace (token lexeme #:skip? #t)]
[(from/to "rem" "\n")
(token 'REM
(string-downcase lexeme)
#:position (pos lexeme-start)
#:line (line lexeme-start)
#:column (col lexeme-start)
#:span (- (pos lexeme-end)
(pos lexeme-start)))]
[(:or "print" "goto" "end")
(token (string-downcase lexeme)
(string-downcase lexeme)
#:position (pos lexeme-start)
#:line (line lexeme-start)
#:column (col lexeme-start)
#:span (- (pos lexeme-end)
(pos lexeme-start)))]
[(:+ numeric)
(token 'NUMBER
(string->number lexeme)
#:position (pos lexeme-start)
#:line (line lexeme-start)
#:column (col lexeme-start)
#:span (- (pos lexeme-end)
(pos lexeme-start)))]
[(from/to "\"" "\"")
(token 'STRING
(trim-ends "\"" lexeme "\"")
#:position (pos lexeme-start)
#:line (line lexeme-start)
#:column (col lexeme-start)
#:span (- (pos lexeme-end)
(pos lexeme-start)))]))
9 years ago
8 years ago
(define (tokenize ip)
(port-count-lines! ip)
8 years ago
(thunk (basic-lexer ip)))