|
|
@ -1,15 +1,14 @@
|
|
|
|
#lang br
|
|
|
|
#lang br
|
|
|
|
(require parser-tools/lex
|
|
|
|
(require parser-tools/lex parser-tools/lex-sre
|
|
|
|
(prefix-in : parser-tools/lex-sre)
|
|
|
|
|
|
|
|
ragg/support
|
|
|
|
ragg/support
|
|
|
|
racket/string)
|
|
|
|
racket/string)
|
|
|
|
(provide tokenize)
|
|
|
|
(provide tokenize)
|
|
|
|
|
|
|
|
|
|
|
|
(define-lex-abbrevs
|
|
|
|
(define-lex-abbrevs
|
|
|
|
(natural (repetition 1 +inf.0 numeric))
|
|
|
|
(natural (repetition 1 +inf.0 numeric))
|
|
|
|
(integer (:seq (:? "-") natural))
|
|
|
|
(number (union (seq (? "-") natural)
|
|
|
|
(number (:seq integer (:? (:seq "." natural))))
|
|
|
|
(seq (? "-") (? natural) (seq "." natural))))
|
|
|
|
(quoted-string (:seq "\"" (repetition 0 +inf.0 (char-complement "\"")) "\"")))
|
|
|
|
(quoted-string (seq "\"" (repetition 0 +inf.0 (char-complement "\"")) "\"")))
|
|
|
|
|
|
|
|
|
|
|
|
(define (tokenize input-port)
|
|
|
|
(define (tokenize input-port)
|
|
|
|
(define (next-token)
|
|
|
|
(define (next-token)
|
|
|
@ -17,17 +16,17 @@
|
|
|
|
(lexer
|
|
|
|
(lexer
|
|
|
|
[(eof) eof]
|
|
|
|
[(eof) eof]
|
|
|
|
[(union #\tab #\space
|
|
|
|
[(union #\tab #\space
|
|
|
|
(:seq number " REM" (repetition 1 +inf.0 (char-complement #\newline)) #\newline)) (get-token input-port)]
|
|
|
|
(seq number " REM" (repetition 1 +inf.0 (char-complement #\newline)) #\newline)) (get-token input-port)]
|
|
|
|
[(:seq #\newline (repetition 0 +inf.0 whitespace)) (token 'CR "cr")]
|
|
|
|
[(seq #\newline (repetition 0 +inf.0 whitespace)) (token 'CR "cr")]
|
|
|
|
[(union "PRINT" "FOR" "TO" "STEP" "IF" "GOTO"
|
|
|
|
[(union "PRINT" "FOR" "TO" "STEP" "IF" "GOTO"
|
|
|
|
"INPUT" "LET" "NEXT" "RETURN"
|
|
|
|
"INPUT" "LET" "NEXT" "RETURN"
|
|
|
|
"CLEAR" "LIST" "RUN" "END"
|
|
|
|
"CLEAR" "LIST" "RUN" "END"
|
|
|
|
"THEN" "ELSE" "GOSUB" "AND" "OR"
|
|
|
|
"THEN" "ELSE" "GOSUB" "AND" "OR"
|
|
|
|
";" "=" "(" ")" "+" "-" "*" "/"
|
|
|
|
";" "=" "(" ")" "+" "-" "*" "/"
|
|
|
|
"<=" ">=" "<>" "><" "<" ">" "=" ":") lexeme]
|
|
|
|
"<=" ">=" "<>" "<" ">" "=" ":") lexeme]
|
|
|
|
[(union ",") (get-token input-port)]
|
|
|
|
[(union ",") (get-token input-port)]
|
|
|
|
[number (token 'NUMBER (string->number lexeme))]
|
|
|
|
[number (token 'NUMBER (string->number lexeme))]
|
|
|
|
[(:seq (repetition 1 +inf.0 upper-case) (:? "$")) (token 'ID (string->symbol lexeme))]
|
|
|
|
[(seq (repetition 1 +inf.0 upper-case) (? "$")) (token 'ID (string->symbol lexeme))]
|
|
|
|
[upper-case (token 'UPPERCASE (string->symbol lexeme))]
|
|
|
|
[upper-case (token 'UPPERCASE (string->symbol lexeme))]
|
|
|
|
[quoted-string (token 'STRING (string-trim lexeme "\""))]))
|
|
|
|
[quoted-string (token 'STRING (string-trim lexeme "\""))]))
|
|
|
|
(get-token input-port))
|
|
|
|
(get-token input-port))
|
|
|
|