move fix upstream

dev-srcloc
Matthew Butterick 8 years ago
parent ae07f3da64
commit ec6b858f39

@ -80,10 +80,7 @@
(lookup (string->symbol next-token) next-token no-position no-position)]
[(? char?)
(lookup (cond
[(eqv? #\newline next-token) '|\n|] ; supports handling whitespace char
[(eqv? #\tab next-token) '|\t|] ; as literal token
[else (string->symbol (string next-token))]) next-token no-position no-position)]
(lookup (string->symbol (string next-token)) next-token no-position no-position)]
;; Compatibility
[(? lex:token?)

@ -32,14 +32,21 @@
(define lex/1
(lexer-src-pos
;; handle whitespace chars within quotes as literal tokens: "\n" "\t" '\n' '\t'
;; by matching the escaped version, and then unescaping them before they become token-LITs
[(:: "'"
(:* (:or "\\'" "\\n" "\\t" (:~ "'" "\\")))
"'")
(token-LIT lexeme)]
(token-LIT (case lexeme
[("'\\n'") "'\n'"]
[("'\\t'") "'\t'"]
[else lexeme]))]
[(:: "\""
(:* (:or "\\\"" "\\n" "\\t" (:~ "\"" "\\")))
"\"")
(token-LIT lexeme)]
(token-LIT (case lexeme
[("\"\\n\"") "\"\n\""]
[("\"\\t\"") "\"\t\""]
[else lexeme]))]
["("
(token-LPAREN lexeme)]
["["

Loading…
Cancel
Save