update reader & tokenizer

pull/10/head
Matthew Butterick 8 years ago
parent fb26d2b7cd
commit 3347a86c0d

@ -2,7 +2,7 @@
(require "tokenizer.rkt" "parser.rkt") (require "tokenizer.rkt" "parser.rkt")
(define (read-syntax path port) (define (read-syntax path port)
(define parse-tree (parse path (tokenize port))) (define parse-tree (parse path (make-tokenizer port)))
(define module-datum `(module jsonic-module jsonic-demo/expander (define module-datum `(module jsonic-module jsonic-demo/expander
,parse-tree)) ,parse-tree))
(datum->syntax #f module-datum)) (datum->syntax #f module-datum))

@ -1,15 +1,15 @@
#lang br/quicklang #lang br/quicklang
(require brag/support) (require brag/support)
(define (tokenize port) (define (make-tokenizer port)
(define (next-token) (define (next-token)
(define our-lexer (define jsonic-lexer
(lexer (lexer
[(eof) eof] [(eof) eof]
[(from/to "//" "\n") (next-token)] [(from/to "//" "\n") (next-token)]
[(from/to "@$" "$@") [(from/to "@$" "$@")
(token 'SEXP-TOK (trim-ends "@$" lexeme "$@"))] (token 'SEXP-TOK (trim-ends "@$" lexeme "$@"))]
[any-char (token 'CHAR-TOK lexeme)])) [any-char (token 'CHAR-TOK lexeme)]))
(our-lexer port)) (jsonic-lexer port))
next-token) next-token)
(provide tokenize) (provide make-tokenizer)
Loading…
Cancel
Save