diff --git a/brag/brag/codegen/runtime.rkt b/brag/brag/codegen/runtime.rkt index 063469a..980bec2 100755 --- a/brag/brag/codegen/runtime.rkt +++ b/brag/brag/codegen/runtime.rkt @@ -80,10 +80,7 @@ (lookup (string->symbol next-token) next-token no-position no-position)] [(? char?) - (lookup (cond - [(eqv? #\newline next-token) '|\n|] ; supports handling whitespace char - [(eqv? #\tab next-token) '|\t|] ; as literal token - [else (string->symbol (string next-token))]) next-token no-position no-position)] + (lookup (string->symbol (string next-token)) next-token no-position no-position)] ;; Compatibility [(? lex:token?) diff --git a/brag/brag/rules/lexer.rkt b/brag/brag/rules/lexer.rkt index 675c10d..c411ba7 100755 --- a/brag/brag/rules/lexer.rkt +++ b/brag/brag/rules/lexer.rkt @@ -32,14 +32,21 @@ (define lex/1 (lexer-src-pos ;; handle whitespace chars within quotes as literal tokens: "\n" "\t" '\n' '\t' + ;; by matching the escaped version, and then unescaping them before they become token-LITs [(:: "'" (:* (:or "\\'" "\\n" "\\t" (:~ "'" "\\"))) "'") - (token-LIT lexeme)] + (token-LIT (case lexeme + [("'\\n'") "'\n'"] + [("'\\t'") "'\t'"] + [else lexeme]))] [(:: "\"" (:* (:or "\\\"" "\\n" "\\t" (:~ "\"" "\\"))) "\"") - (token-LIT lexeme)] + (token-LIT (case lexeme + [("\"\\n\"") "\"\n\""] + [("\"\\t\"") "\"\t\""] + [else lexeme]))] ["(" (token-LPAREN lexeme)] ["["