From ae07f3da6446f6b342317c8f3f69937907845ca0 Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 19 Dec 2016 15:04:26 -0800
Subject: [PATCH] add support for newline & tab literals in grammars

---
 brag/brag/codegen/runtime.rkt      |  5 ++++-
 brag/brag/examples/whitespace.rkt  |  6 ++++++
 brag/brag/rules/lexer.rkt          |  5 +++--
 brag/brag/test/test-whitespace.rkt | 12 ++++++++++++
 4 files changed, 25 insertions(+), 3 deletions(-)
 create mode 100644 brag/brag/examples/whitespace.rkt
 create mode 100755 brag/brag/test/test-whitespace.rkt

diff --git a/brag/brag/codegen/runtime.rkt b/brag/brag/codegen/runtime.rkt
index 980bec2..063469a 100755
--- a/brag/brag/codegen/runtime.rkt
+++ b/brag/brag/codegen/runtime.rkt
@@ -80,7 +80,10 @@
          (lookup (string->symbol next-token) next-token no-position no-position)]
         
         [(? char?)
-         (lookup (string->symbol (string next-token)) next-token no-position no-position)]
+         (lookup (cond
+                   [(eqv? #\newline next-token) '|\n|] ; supports handling whitespace char
+                   [(eqv? #\tab next-token) '|\t|] ; as literal token
+                   [else (string->symbol (string next-token))]) next-token no-position no-position)]
         
         ;; Compatibility 
         [(? lex:token?)
diff --git a/brag/brag/examples/whitespace.rkt b/brag/brag/examples/whitespace.rkt
new file mode 100644
index 0000000..b76ebf4
--- /dev/null
+++ b/brag/brag/examples/whitespace.rkt
@@ -0,0 +1,6 @@
+#lang brag
+start: (tab | space | newline | letter)*
+tab: '\t'
+space: " "
+newline: "\n"
+letter: "x" | "y" | "z"
\ No newline at end of file
diff --git a/brag/brag/rules/lexer.rkt b/brag/brag/rules/lexer.rkt
index b83cf25..675c10d 100755
--- a/brag/brag/rules/lexer.rkt
+++ b/brag/brag/rules/lexer.rkt
@@ -31,12 +31,13 @@
 
 (define lex/1
   (lexer-src-pos
+   ;; handle whitespace chars within quotes as literal tokens: "\n" "\t" '\n' '\t'
    [(:: "'"
-        (:* (:or "\\'" (:~ "'" "\\")))
+        (:* (:or "\\'" "\\n" "\\t" (:~ "'" "\\")))
         "'")
     (token-LIT lexeme)]
    [(:: "\""
-        (:* (:or "\\\"" (:~ "\"" "\\")))
+        (:* (:or "\\\"" "\\n" "\\t" (:~ "\"" "\\")))
         "\"")
     (token-LIT lexeme)]
    ["("
diff --git a/brag/brag/test/test-whitespace.rkt b/brag/brag/test/test-whitespace.rkt
new file mode 100755
index 0000000..c895ca8
--- /dev/null
+++ b/brag/brag/test/test-whitespace.rkt
@@ -0,0 +1,12 @@
+#lang racket/base
+(require brag/examples/whitespace
+         brag/support
+         rackunit)
+
+(check-equal?
+ (parse-tree "\ty\n x\tz")
+ '(start (tab "\t") (letter "y") (newline "\n") (space " ") (letter "x") (tab "\t") (letter "z")))
+
+(check-equal?
+ (parse-tree "\t\n \t")
+ '(start (tab "\t") (newline "\n") (space " ") (tab "\t")))