diff --git a/beautiful-racket-demo/basic-demo/lexer-test.rkt b/beautiful-racket-demo/basic-demo/lexer-test.rkt new file mode 100644 index 0000000..227ca99 --- /dev/null +++ b/beautiful-racket-demo/basic-demo/lexer-test.rkt @@ -0,0 +1,56 @@ +#lang br +(require "lexer.rkt" brag/support rackunit) + +(define (lex str) + (apply-lexer basic-lexer str)) + +(check-equal? (lex "") empty) +(check-equal? + (lex " ") + (list (srcloc-token (token " " #:skip? #t) + (srcloc 'string #f #f 1 1)))) +(check-equal? + (lex "rem ignored\n") + (list (srcloc-token (token 'REM "rem ignored\n") + (srcloc 'string #f #f 1 12)))) +(check-equal? + (lex "print") + (list (srcloc-token "print" + (srcloc 'string #f #f 1 5)))) +(check-equal? + (lex "goto") + (list (srcloc-token "goto" + (srcloc 'string #f #f 1 4)))) +(check-equal? + (lex "end") + (list (srcloc-token "end" + (srcloc 'string #f #f 1 3)))) +(check-equal? + (lex "+") + (list (srcloc-token "+" + (srcloc 'string #f #f 1 1)))) +(check-equal? + (lex "12") + (list (srcloc-token (token 'INTEGER 12) + (srcloc 'string #f #f 1 2)))) +(check-equal? + (lex "1.2") + (list (srcloc-token (token 'DECIMAL 1.2) + (srcloc 'string #f #f 1 3)))) +(check-equal? + (lex "12.") + (list (srcloc-token (token 'DECIMAL 12.) + (srcloc 'string #f #f 1 3)))) +(check-equal? + (lex ".12") + (list (srcloc-token (token 'DECIMAL .12) + (srcloc 'string #f #f 1 3)))) +(check-equal? + (lex "\"foo\"") + (list (srcloc-token (token 'STRING "foo") + (srcloc 'string #f #f 1 5)))) +(check-equal? + (lex "'foo'") + (list (srcloc-token (token 'STRING "foo") + (srcloc 'string #f #f 1 5)))) +(check-exn exn:fail:read? (lambda () (lex "x"))) \ No newline at end of file diff --git a/beautiful-racket-demo/basic-demo/lexer.rkt b/beautiful-racket-demo/basic-demo/lexer.rkt index 94e9a77..7a2ed08 100644 --- a/beautiful-racket-demo/basic-demo/lexer.rkt +++ b/beautiful-racket-demo/basic-demo/lexer.rkt @@ -10,9 +10,12 @@ [(from/to "rem" "\n") (token 'REM lexeme)] [(:or "print" "goto" "end" "+" ":") lexeme] [digits (token 'INTEGER (string->number lexeme))] - [(:or (:seq digits ".") (:seq (:? digits) "." digits)) + [(:or (:seq (:? digits) "." digits) + (:seq digits ".")) (token 'DECIMAL (string->number lexeme))] - [(from/to "\"" "\"") - (token 'STRING (trim-ends "\"" lexeme "\""))])) + [(:or (from/to "\"" "\"") (from/to "'" "'")) + (token 'STRING + (substring lexeme + 1 (sub1 (string-length lexeme))))])) (provide basic-lexer) diff --git a/beautiful-racket-demo/basic-demo/sample.rkt b/beautiful-racket-demo/basic-demo/sample.rkt index f306001..e94f036 100644 --- a/beautiful-racket-demo/basic-demo/sample.rkt +++ b/beautiful-racket-demo/basic-demo/sample.rkt @@ -2,7 +2,7 @@ 30 rem print "ignored" 50 print "never gets here" 40 end -60 print "three" : print 1.0 + 3 +60 print 'three' : print 1.0 + 3 70 goto 11. + 18.5 + .5 10 print "one" 20 print : goto 60.0 : end \ No newline at end of file diff --git a/brag/brag/brag.scrbl b/brag/brag/brag.scrbl index 648a818..1efbfc6 100755 --- a/brag/brag/brag.scrbl +++ b/brag/brag/brag.scrbl @@ -974,6 +974,13 @@ DrRacket should highlight the offending locations in the source.} Repeatedly apply @racket[tokenizer-maker] to @racket[source], gathering the resulting tokens into a list. @racket[source] can be a string or an input port. Useful for testing or debugging a tokenizer. } +@defproc[(apply-lexer [lexer procedure?] + [source (or/c string? + input-port?)]) + list?]{ +Repeatedly apply @racket[lexer] to @racket[source], gathering the resulting tokens into a list. @racket[source] can be a string or an input port. Useful for testing or debugging a lexer. +} + @defproc[(trim-ends [left-str string?] [str string?]