diff --git a/beautiful-racket-demo/basic-demo/lexer-test.rkt b/beautiful-racket-demo/basic-demo/lexer-test.rkt index d20de3a..c0ea029 100644 --- a/beautiful-racket-demo/basic-demo/lexer-test.rkt +++ b/beautiful-racket-demo/basic-demo/lexer-test.rkt @@ -13,7 +13,7 @@ (lex "rem ignored\n") (list (srcloc-token (token 'REM "rem ignored") (srcloc 'string #f #f 1 11)) - (srcloc-token (token 'NEWLINE) + (srcloc-token (token 'NEWLINE "\n") (srcloc 'string #f #f 12 1)))) (check-equal? (lex "print") diff --git a/beautiful-racket-demo/basic-demo/lexer.rkt b/beautiful-racket-demo/basic-demo/lexer.rkt index cb99d7d..08a2cf7 100644 --- a/beautiful-racket-demo/basic-demo/lexer.rkt +++ b/beautiful-racket-demo/basic-demo/lexer.rkt @@ -6,7 +6,7 @@ (define basic-lexer (lexer-srcloc [(eof) (return-without-srcloc eof)] - ["\n" (token 'NEWLINE)] + ["\n" (token 'NEWLINE lexeme)] [whitespace (token lexeme #:skip? #t)] [(from/stop-before "rem" "\n") (token 'REM lexeme)] [(:or "print" "goto" "end" "+" ":") lexeme] diff --git a/beautiful-racket-demo/basic-demo/main.rkt b/beautiful-racket-demo/basic-demo/main.rkt index b77a60a..b18b5ba 100644 --- a/beautiful-racket-demo/basic-demo/main.rkt +++ b/beautiful-racket-demo/basic-demo/main.rkt @@ -4,13 +4,7 @@ (module+ reader (provide read-syntax)) (define (read-syntax path port) - (define-values (pline pcol ppos) (port-next-location port)) - (define port+newline - (input-port-append #f port (open-input-string "\n"))) - (port-count-lines! port+newline) - (set-port-next-location! port+newline pline pcol ppos) - (define parse-tree - (parse path (make-tokenizer port+newline path))) + (define parse-tree (parse path (make-tokenizer port path))) (strip-bindings #`(module basic-mod basic-demo/expander #,parse-tree))) \ No newline at end of file diff --git a/beautiful-racket-demo/basic-demo/parse-only.rkt b/beautiful-racket-demo/basic-demo/parse-only.rkt new file mode 100644 index 0000000..a2d7b7d --- /dev/null +++ b/beautiful-racket-demo/basic-demo/parse-only.rkt @@ -0,0 +1,14 @@ +#lang br/quicklang +(require "parser.rkt" "tokenizer.rkt") + +(define (read-syntax path port) + (define parse-tree (parse path (make-tokenizer port path))) + (strip-bindings + #`(module basic-parser-mod basic-demo/parse-only + #,parse-tree))) +(module+ reader (provide read-syntax)) + +(define-macro (parser-only-mb PARSE-TREE) + #'(#%module-begin + 'PARSE-TREE)) +(provide (rename-out [parser-only-mb #%module-begin])) \ No newline at end of file diff --git a/beautiful-racket-demo/basic-demo/parser.rkt b/beautiful-racket-demo/basic-demo/parser.rkt index 5ce052b..aaeab76 100644 --- a/beautiful-racket-demo/basic-demo/parser.rkt +++ b/beautiful-racket-demo/basic-demo/parser.rkt @@ -1,7 +1,7 @@ #lang brag b-program : [b-line] (/NEWLINE [b-line])* -b-line : @b-line-number [b-statement] (/":" [b-statement])* -b-line-number : INTEGER +b-line : b-line-number [b-statement] (/":" [b-statement])* +@b-line-number : INTEGER @b-statement : b-rem | b-end | b-print | b-goto b-rem : REM b-end : /"end" diff --git a/beautiful-racket-demo/basic-demo/sample.rkt b/beautiful-racket-demo/basic-demo/sample.rkt index b6d912f..a3089f6 100644 --- a/beautiful-racket-demo/basic-demo/sample.rkt +++ b/beautiful-racket-demo/basic-demo/sample.rkt @@ -1,5 +1,5 @@ #lang basic-demo -30 rem print "ignored" +30 rem print 'ignored' 35 50 print "never gets here" 40 end diff --git a/beautiful-racket-demo/basic-demo/tokenize-only.rkt b/beautiful-racket-demo/basic-demo/tokenize-only.rkt new file mode 100644 index 0000000..9606413 --- /dev/null +++ b/beautiful-racket-demo/basic-demo/tokenize-only.rkt @@ -0,0 +1,14 @@ +#lang br/quicklang +(require brag/support "tokenizer.rkt") + +(define (read-syntax path port) + (define tokens (apply-tokenizer make-tokenizer port)) + (strip-bindings + #`(module basic-tokens-mod basic-demo/tokenize-only + #,@tokens))) +(module+ reader (provide read-syntax)) + +(define-macro (parser-only-mb TOKEN ...) + #'(#%module-begin + (list TOKEN ...))) +(provide (rename-out [parser-only-mb #%module-begin])) \ No newline at end of file