From 509de033a0bbb0061e71b7492157634b692438d3 Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 30 Jan 2017 11:20:01 -0800
Subject: [PATCH] tweakage

---
 .../basic-demo/lexer-test.rkt                 | 56 +++++++++++++++++++
 beautiful-racket-demo/basic-demo/lexer.rkt    |  9 ++-
 beautiful-racket-demo/basic-demo/sample.rkt   |  2 +-
 brag/brag/brag.scrbl                          |  7 +++
 4 files changed, 70 insertions(+), 4 deletions(-)
 create mode 100644 beautiful-racket-demo/basic-demo/lexer-test.rkt

diff --git a/beautiful-racket-demo/basic-demo/lexer-test.rkt b/beautiful-racket-demo/basic-demo/lexer-test.rkt
new file mode 100644
index 0000000..227ca99
--- /dev/null
+++ b/beautiful-racket-demo/basic-demo/lexer-test.rkt
@@ -0,0 +1,56 @@
+#lang br
+(require "lexer.rkt" brag/support rackunit)
+
+(define (lex str)
+  (apply-lexer basic-lexer str))
+
+(check-equal? (lex "") empty)
+(check-equal?
+ (lex " ")
+ (list (srcloc-token (token " " #:skip? #t)
+                     (srcloc 'string #f #f 1 1))))
+(check-equal?
+ (lex "rem ignored\n")
+ (list (srcloc-token (token 'REM "rem ignored\n")
+                     (srcloc 'string #f #f 1 12))))
+(check-equal?
+ (lex "print")
+ (list (srcloc-token "print"
+                     (srcloc 'string #f #f 1 5))))
+(check-equal?
+ (lex "goto")
+ (list (srcloc-token "goto"
+                     (srcloc 'string #f #f 1 4))))
+(check-equal?
+ (lex "end")
+ (list (srcloc-token "end"
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex "+")
+ (list (srcloc-token "+"
+                     (srcloc 'string #f #f 1 1))))
+(check-equal?
+ (lex "12")
+ (list (srcloc-token (token 'INTEGER 12)
+                     (srcloc 'string #f #f 1 2))))
+(check-equal?
+ (lex "1.2")
+ (list (srcloc-token (token 'DECIMAL 1.2)
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex "12.")
+ (list (srcloc-token (token 'DECIMAL 12.)
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex ".12")
+ (list (srcloc-token (token 'DECIMAL .12)
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex "\"foo\"")
+ (list (srcloc-token (token 'STRING "foo")
+                     (srcloc 'string #f #f 1 5))))
+(check-equal?
+ (lex "'foo'")
+ (list (srcloc-token (token 'STRING "foo")
+                     (srcloc 'string #f #f 1 5))))
+(check-exn exn:fail:read? (lambda () (lex "x")))
\ No newline at end of file
diff --git a/beautiful-racket-demo/basic-demo/lexer.rkt b/beautiful-racket-demo/basic-demo/lexer.rkt
index 94e9a77..7a2ed08 100644
--- a/beautiful-racket-demo/basic-demo/lexer.rkt
+++ b/beautiful-racket-demo/basic-demo/lexer.rkt
@@ -10,9 +10,12 @@
    [(from/to "rem" "\n") (token 'REM lexeme)]
    [(:or "print" "goto" "end" "+" ":") lexeme]
    [digits (token 'INTEGER (string->number lexeme))]
-   [(:or (:seq digits ".") (:seq (:? digits) "." digits))
+   [(:or (:seq (:? digits) "." digits)
+         (:seq digits "."))
     (token 'DECIMAL (string->number lexeme))]
-   [(from/to "\"" "\"")
-    (token 'STRING (trim-ends "\"" lexeme "\""))]))
+   [(:or (from/to "\"" "\"") (from/to "'" "'"))
+    (token 'STRING
+           (substring lexeme
+                      1 (sub1 (string-length lexeme))))]))
 
 (provide basic-lexer)
diff --git a/beautiful-racket-demo/basic-demo/sample.rkt b/beautiful-racket-demo/basic-demo/sample.rkt
index f306001..e94f036 100644
--- a/beautiful-racket-demo/basic-demo/sample.rkt
+++ b/beautiful-racket-demo/basic-demo/sample.rkt
@@ -2,7 +2,7 @@
 30 rem print "ignored"
 50 print "never gets here"
 40 end
-60 print "three" : print 1.0 + 3
+60 print 'three' : print 1.0 + 3
 70 goto 11. + 18.5 + .5
 10 print "one"
 20 print : goto 60.0 : end
\ No newline at end of file
diff --git a/brag/brag/brag.scrbl b/brag/brag/brag.scrbl
index 648a818..1efbfc6 100755
--- a/brag/brag/brag.scrbl
+++ b/brag/brag/brag.scrbl
@@ -974,6 +974,13 @@ DrRacket should highlight the offending locations in the source.}
 Repeatedly apply @racket[tokenizer-maker] to @racket[source], gathering the resulting tokens into a list. @racket[source] can be a string or an input port. Useful for testing or debugging a tokenizer.
 }
 
+@defproc[(apply-lexer [lexer procedure?] 
+                [source (or/c string?
+                                    input-port?)])
+         list?]{
+Repeatedly apply @racket[lexer] to @racket[source], gathering the resulting tokens into a list. @racket[source] can be a string or an input port. Useful for testing or debugging a lexer.
+}
+
 
 @defproc[(trim-ends [left-str string?]
 [str string?]