|
|
@ -8,39 +8,12 @@
|
|
|
|
(lexer-src-pos
|
|
|
|
(lexer-src-pos
|
|
|
|
[(eof) eof]
|
|
|
|
[(eof) eof]
|
|
|
|
[whitespace (token lexeme #:skip? #t)]
|
|
|
|
[whitespace (token lexeme #:skip? #t)]
|
|
|
|
[(from/to "rem" "\n")
|
|
|
|
[(from/to "rem" "\n") (token 'REM (string-downcase lexeme))]
|
|
|
|
(token 'REM
|
|
|
|
[(:or "print" "goto" "end") (token (string-downcase lexeme)
|
|
|
|
(string-downcase lexeme)
|
|
|
|
(string-downcase lexeme))]
|
|
|
|
#:position (pos lexeme-start)
|
|
|
|
[(:+ numeric) (token 'NUMBER (string->number lexeme))]
|
|
|
|
#:line (line lexeme-start)
|
|
|
|
[(from/to "\"" "\"") (token 'STRING (trim-ends "\"" lexeme "\""))]))
|
|
|
|
#:column (col lexeme-start)
|
|
|
|
|
|
|
|
#:span (- (pos lexeme-end)
|
|
|
|
|
|
|
|
(pos lexeme-start)))]
|
|
|
|
|
|
|
|
[(:or "print" "goto" "end")
|
|
|
|
|
|
|
|
(token (string-downcase lexeme)
|
|
|
|
|
|
|
|
(string-downcase lexeme)
|
|
|
|
|
|
|
|
#:position (pos lexeme-start)
|
|
|
|
|
|
|
|
#:line (line lexeme-start)
|
|
|
|
|
|
|
|
#:column (col lexeme-start)
|
|
|
|
|
|
|
|
#:span (- (pos lexeme-end)
|
|
|
|
|
|
|
|
(pos lexeme-start)))]
|
|
|
|
|
|
|
|
[(:+ numeric)
|
|
|
|
|
|
|
|
(token 'NUMBER
|
|
|
|
|
|
|
|
(string->number lexeme)
|
|
|
|
|
|
|
|
#:position (pos lexeme-start)
|
|
|
|
|
|
|
|
#:line (line lexeme-start)
|
|
|
|
|
|
|
|
#:column (col lexeme-start)
|
|
|
|
|
|
|
|
#:span (- (pos lexeme-end)
|
|
|
|
|
|
|
|
(pos lexeme-start)))]
|
|
|
|
|
|
|
|
[(from/to "\"" "\"")
|
|
|
|
|
|
|
|
(token 'STRING
|
|
|
|
|
|
|
|
(trim-ends "\"" lexeme "\"")
|
|
|
|
|
|
|
|
#:position (pos lexeme-start)
|
|
|
|
|
|
|
|
#:line (line lexeme-start)
|
|
|
|
|
|
|
|
#:column (col lexeme-start)
|
|
|
|
|
|
|
|
#:span (- (pos lexeme-end)
|
|
|
|
|
|
|
|
(pos lexeme-start)))]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
(define (tokenize ip)
|
|
|
|
(define (tokenize ip)
|
|
|
|
(port-count-lines! ip)
|
|
|
|
(port-count-lines! ip)
|
|
|
|
(thunk (basic-lexer ip)))
|
|
|
|
(λ () (basic-lexer ip)))
|