diff --git a/brag-lib/brag/examples/codepoints.rkt b/brag-lib/brag/examples/codepoints.rkt index c4be182..2a618f8 100644 --- a/brag-lib/brag/examples/codepoints.rkt +++ b/brag-lib/brag/examples/codepoints.rkt @@ -1,5 +1,6 @@ #lang brag -start: A c def +start: A c def hello-world A : "\"\101\\" ; A c : '\'\U0063\\' ; c -def : "*\u64\x65f" ; de \ No newline at end of file +def : "*\u64\\\x65f\"" ; de +hello-world : "\150\145\154\154\157\40\167\157\162\154\144" \ No newline at end of file diff --git a/brag-lib/brag/rules/lexer.rkt b/brag-lib/brag/rules/lexer.rkt index 9ff7976..020e8f9 100755 --- a/brag-lib/brag/rules/lexer.rkt +++ b/brag-lib/brag/rules/lexer.rkt @@ -1,4 +1,4 @@ -#lang debug at-exp racket/base +#lang at-exp racket/base (require (for-syntax racket/base "parser.rkt")) (require br-parser-tools/lex (prefix-in : br-parser-tools/lex-sre) @@ -79,12 +79,13 @@ ;; it can be lexed as (:: escaped-backlash double-quote) = \\ + " ;; or (:: backlash escaped-double-quote) = \ + \" ;; because escapes should be "left associative", - ;; we forbid the second possibility + ;; we forbid the second possibility + ;; There are still some weird corner cases but the current tests work. + ;; with single and double quotes in the mix, + ;; I'm not sure how much better this can be. (complement (:: any-string backslash escaped-double-quote any-string))) double-quote) ;; end with double quote - (let () - (displayln lexeme) - (token-LIT (unescape-double-quoted-lexeme lexeme start-pos end-pos)))] + (token-LIT (unescape-double-quoted-lexeme lexeme start-pos end-pos))] ;; single-quoted string follows the same pattern, ;; but with escaped-single-quote instead of escaped-double-quote [(:: single-quote diff --git a/brag-lib/brag/test/test-codepoints.rkt b/brag-lib/brag/test/test-codepoints.rkt index d92652f..bb3b3b7 100755 --- a/brag-lib/brag/test/test-codepoints.rkt +++ b/brag-lib/brag/test/test-codepoints.rkt @@ -3,5 +3,8 @@ (require brag/examples/codepoints rackunit) -(check-equal? (parse-to-datum '("\"A\\" "'c\\" "*def")) - '(start (A "\"A\\") (c "'c\\") (def "*def"))) +(check-equal? (parse-to-datum '("\"A\\" "'c\\" "*d\\ef\"" "hello world")) + '(start (A "\"A\\") + (c "'c\\") + (def "*d\\ef\"") + (hello-world "hello world")))