tweakage

beautiful-racket-demo/basic-demo/lexer-test.rkt

@@ -0,0 +1,56 @@
+#lang br
+(require "lexer.rkt" brag/support rackunit)
+
+(define (lex str)
+  (apply-lexer basic-lexer str))
+
+(check-equal? (lex "") empty)
+(check-equal?
+ (lex " ")
+ (list (srcloc-token (token " " #:skip? #t)
+                     (srcloc 'string #f #f 1 1))))
+(check-equal?
+ (lex "rem ignored\n")
+ (list (srcloc-token (token 'REM "rem ignored\n")
+                     (srcloc 'string #f #f 1 12))))
+(check-equal?
+ (lex "print")
+ (list (srcloc-token "print"
+                     (srcloc 'string #f #f 1 5))))
+(check-equal?
+ (lex "goto")
+ (list (srcloc-token "goto"
+                     (srcloc 'string #f #f 1 4))))
+(check-equal?
+ (lex "end")
+ (list (srcloc-token "end"
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex "+")
+ (list (srcloc-token "+"
+                     (srcloc 'string #f #f 1 1))))
+(check-equal?
+ (lex "12")
+ (list (srcloc-token (token 'INTEGER 12)
+                     (srcloc 'string #f #f 1 2))))
+(check-equal?
+ (lex "1.2")
+ (list (srcloc-token (token 'DECIMAL 1.2)
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex "12.")
+ (list (srcloc-token (token 'DECIMAL 12.)
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex ".12")
+ (list (srcloc-token (token 'DECIMAL .12)
+                     (srcloc 'string #f #f 1 3))))
+(check-equal?
+ (lex "\"foo\"")
+ (list (srcloc-token (token 'STRING "foo")
+                     (srcloc 'string #f #f 1 5))))
+(check-equal?
+ (lex "'foo'")
+ (list (srcloc-token (token 'STRING "foo")
+                     (srcloc 'string #f #f 1 5))))
+(check-exn exn:fail:read? (lambda () (lex "x")))

beautiful-racket-demo/basic-demo/lexer.rkt

@@ -10,9 +10,12 @@
    [(from/to "rem" "\n") (token 'REM lexeme)]
    [(:or "print" "goto" "end" "+" ":") lexeme]
    [digits (token 'INTEGER (string->number lexeme))]
-   [(:or (:seq digits ".") (:seq (:? digits) "." digits))
+   [(:or (:seq (:? digits) "." digits)
+         (:seq digits "."))
     (token 'DECIMAL (string->number lexeme))]
-   [(from/to "\"" "\"")
-    (token 'STRING (trim-ends "\"" lexeme "\""))]))
+   [(:or (from/to "\"" "\"") (from/to "'" "'"))
+    (token 'STRING
+           (substring lexeme
+                      1 (sub1 (string-length lexeme))))]))
 
 (provide basic-lexer)

beautiful-racket-demo/basic-demo/sample.rkt

@@ -2,7 +2,7 @@
 30 rem print "ignored"
 50 print "never gets here"
 40 end
-60 print "three" : print 1.0 + 3
+60 print 'three' : print 1.0 + 3
 70 goto 11. + 18.5 + .5
 10 print "one"
 20 print : goto 60.0 : end

brag/brag/brag.scrbl

@@ -974,6 +974,13 @@ DrRacket should highlight the offending locations in the source.}
 Repeatedly apply @racket[tokenizer-maker] to @racket[source], gathering the resulting tokens into a list. @racket[source] can be a string or an input port. Useful for testing or debugging a tokenizer.
 }
 
+@defproc[(apply-lexer [lexer procedure?] 
+                [source (or/c string?
+                                    input-port?)])
+         list?]{
+Repeatedly apply @racket[lexer] to @racket[source], gathering the resulting tokens into a list. @racket[source] can be a string or an input port. Useful for testing or debugging a lexer.
+}
+
 
 @defproc[(trim-ends [left-str string?]
 [str string?]