more torturous test

brag-lib/brag/examples/codepoints.rkt

@@ -1,5 +1,6 @@
 #lang brag
-start: A c def
+start: A c def hello-world
 A : "\"\101\\" ; A
 c : '\'\U0063\\' ; c
-def : "*\u64\x65f" ; de
+def : "*\u64\\\x65f\"" ; de
+hello-world : "\150\145\154\154\157\40\167\157\162\154\144"

brag-lib/brag/rules/lexer.rkt

@@ -1,4 +1,4 @@
-#lang debug at-exp racket/base
+#lang at-exp racket/base
 (require (for-syntax racket/base "parser.rkt"))
 (require br-parser-tools/lex
          (prefix-in : br-parser-tools/lex-sre)
@@ -79,12 +79,13 @@
          ;; it can be lexed as (:: escaped-backlash double-quote) = \\ + "
          ;; or  (:: backlash escaped-double-quote) = \ + \"
          ;; because escapes should be "left associative",
-         ;; we forbid the second possibility         
+         ;; we forbid the second possibility
+         ;; There are still some weird corner cases but the current tests work.
+         ;; with single and double quotes in the mix,
+         ;; I'm not sure how much better this can be.
          (complement (:: any-string backslash escaped-double-quote any-string)))
         double-quote) ;; end with double quote
-    (let ()
+    (token-LIT (unescape-double-quoted-lexeme lexeme start-pos end-pos))]
-      (displayln lexeme)
-      (token-LIT (unescape-double-quoted-lexeme lexeme start-pos end-pos)))]
    ;; single-quoted string follows the same pattern,
    ;; but with escaped-single-quote instead of escaped-double-quote
    [(:: single-quote

brag-lib/brag/test/test-codepoints.rkt

@@ -3,5 +3,8 @@
 (require brag/examples/codepoints
          rackunit)
 
-(check-equal? (parse-to-datum '("\"A\\" "'c\\" "*def"))
+(check-equal? (parse-to-datum '("\"A\\" "'c\\" "*d\\ef\"" "hello world"))
-              '(start (A "\"A\\") (c "'c\\") (def "*def")))
+              '(start (A "\"A\\")
+                      (c "'c\\")
+                      (def "*d\\ef\"")
+                      (hello-world "hello world")))