move fix upstream

dev-srcloc
Matthew Butterick 8 years ago
parent ae07f3da64
commit ec6b858f39

@ -80,10 +80,7 @@
(lookup (string->symbol next-token) next-token no-position no-position)] (lookup (string->symbol next-token) next-token no-position no-position)]
[(? char?) [(? char?)
(lookup (cond (lookup (string->symbol (string next-token)) next-token no-position no-position)]
[(eqv? #\newline next-token) '|\n|] ; supports handling whitespace char
[(eqv? #\tab next-token) '|\t|] ; as literal token
[else (string->symbol (string next-token))]) next-token no-position no-position)]
;; Compatibility ;; Compatibility
[(? lex:token?) [(? lex:token?)

@ -32,14 +32,21 @@
(define lex/1 (define lex/1
(lexer-src-pos (lexer-src-pos
;; handle whitespace chars within quotes as literal tokens: "\n" "\t" '\n' '\t' ;; handle whitespace chars within quotes as literal tokens: "\n" "\t" '\n' '\t'
;; by matching the escaped version, and then unescaping them before they become token-LITs
[(:: "'" [(:: "'"
(:* (:or "\\'" "\\n" "\\t" (:~ "'" "\\"))) (:* (:or "\\'" "\\n" "\\t" (:~ "'" "\\")))
"'") "'")
(token-LIT lexeme)] (token-LIT (case lexeme
[("'\\n'") "'\n'"]
[("'\\t'") "'\t'"]
[else lexeme]))]
[(:: "\"" [(:: "\""
(:* (:or "\\\"" "\\n" "\\t" (:~ "\"" "\\"))) (:* (:or "\\\"" "\\n" "\\t" (:~ "\"" "\\")))
"\"") "\"")
(token-LIT lexeme)] (token-LIT (case lexeme
[("\"\\n\"") "\"\n\""]
[("\"\\t\"") "\"\t\""]
[else lexeme]))]
["(" ["("
(token-LPAREN lexeme)] (token-LPAREN lexeme)]
["[" ["["

Loading…
Cancel
Save