From 7a7e162988f82023b1f1c5c1b4ca9f3fdf11f5c8 Mon Sep 17 00:00:00 2001 From: Matthew Butterick Date: Sat, 28 Jan 2017 10:54:08 -0800 Subject: [PATCH] update reader & tokenizer --- .../jsonic-demo-2/parser-test.rkt | 13 +++++++++---- beautiful-racket-demo/jsonic-demo-2/reader.rkt | 4 ++-- .../jsonic-demo-2/tokenizer.rkt | 16 +++++++++------- .../jsonic-demo-3/parser-test.rkt | 13 +++++++++---- beautiful-racket-demo/jsonic-demo-3/reader.rkt | 4 ++-- .../jsonic-demo-3/tokenizer.rkt | 16 +++++++++------- 6 files changed, 40 insertions(+), 26 deletions(-) diff --git a/beautiful-racket-demo/jsonic-demo-2/parser-test.rkt b/beautiful-racket-demo/jsonic-demo-2/parser-test.rkt index 6d68926..2cea79c 100644 --- a/beautiful-racket-demo/jsonic-demo-2/parser-test.rkt +++ b/beautiful-racket-demo/jsonic-demo-2/parser-test.rkt @@ -2,18 +2,23 @@ (require "parser.rkt" "tokenizer.rkt" brag/support rackunit) (check-equal? - (parse-tree (apply-tokenizer tokenize "// line commment\n")) + (parse-tree + (apply-tokenizer-maker make-tokenizer "// line commment\n")) '(jsonic-program)) (check-equal? - (parse-tree (apply-tokenizer tokenize "@$ 42 $@")) + (parse-tree + (apply-tokenizer-maker make-tokenizer "@$ 42 $@")) '(jsonic-program (s-exp " 42 "))) (check-equal? - (parse-tree (apply-tokenizer tokenize "hi")) + (parse-tree + (apply-tokenizer-maker make-tokenizer "hi")) '(jsonic-program (json-char "h") (json-char "i"))) (check-equal? - (parse-tree (apply-tokenizer tokenize "hi\n// comment\n@$ 42 $@")) + (parse-tree + (apply-tokenizer-maker make-tokenizer + "hi\n// comment\n@$ 42 $@")) '(jsonic-program (json-char "h") (json-char "i") diff --git a/beautiful-racket-demo/jsonic-demo-2/reader.rkt b/beautiful-racket-demo/jsonic-demo-2/reader.rkt index 82b8b1e..04b8e9a 100644 --- a/beautiful-racket-demo/jsonic-demo-2/reader.rkt +++ b/beautiful-racket-demo/jsonic-demo-2/reader.rkt @@ -2,8 +2,8 @@ (require "tokenizer.rkt" "parser.rkt" racket/contract) (define (read-syntax path port) - (define parse-tree (parse path (tokenize port))) - (define module-datum `(module jsonic-module jsonic-demo-2/expander + (define parse-tree (parse path (make-tokenizer port))) + (define module-datum `(module jsonic-module jsonic-demo/expander ,parse-tree)) (datum->syntax #f module-datum)) (provide (contract-out diff --git a/beautiful-racket-demo/jsonic-demo-2/tokenizer.rkt b/beautiful-racket-demo/jsonic-demo-2/tokenizer.rkt index 5887f01..995c833 100644 --- a/beautiful-racket-demo/jsonic-demo-2/tokenizer.rkt +++ b/beautiful-racket-demo/jsonic-demo-2/tokenizer.rkt @@ -13,10 +13,10 @@ (check-true (token? (token 'A-TOKEN-STRUCT "hi"))) (check-false (token? 42))) -(define (tokenize port) +(define (make-tokenizer port) (port-count-lines! port) (define (next-token) - (define our-lexer + (define jsonic-lexer (lexer [(eof) eof] [(from/to "//" "\n") (next-token)] @@ -33,22 +33,24 @@ #:column (col lexeme-start) #:span (- (pos lexeme-end) (pos lexeme-start)))])) - (our-lexer port)) + (jsonic-lexer port)) next-token) (provide (contract-out - [tokenize (input-port? . -> . (-> token?))])) + [make-tokenizer (input-port? . -> . (-> token?))])) (module+ test - (check-equal? (apply-tokenizer tokenize "// comment\n") empty) (check-equal? - (apply-tokenizer tokenize "@$ (+ 6 7) $@") + (apply-tokenizer-maker make-tokenizer "// comment\n") + empty) + (check-equal? + (apply-tokenizer-maker make-tokenizer "@$ (+ 6 7) $@") (list (token 'SEXP-TOK " (+ 6 7) " #:position 3 #:line 1 #:column 2 #:span 9))) (check-equal? - (apply-tokenizer tokenize "hi") + (apply-tokenizer-maker make-tokenizer "hi") (list (token 'CHAR-TOK "h" #:position 1 #:line 1 diff --git a/beautiful-racket-demo/jsonic-demo-3/parser-test.rkt b/beautiful-racket-demo/jsonic-demo-3/parser-test.rkt index 6d68926..2cea79c 100644 --- a/beautiful-racket-demo/jsonic-demo-3/parser-test.rkt +++ b/beautiful-racket-demo/jsonic-demo-3/parser-test.rkt @@ -2,18 +2,23 @@ (require "parser.rkt" "tokenizer.rkt" brag/support rackunit) (check-equal? - (parse-tree (apply-tokenizer tokenize "// line commment\n")) + (parse-tree + (apply-tokenizer-maker make-tokenizer "// line commment\n")) '(jsonic-program)) (check-equal? - (parse-tree (apply-tokenizer tokenize "@$ 42 $@")) + (parse-tree + (apply-tokenizer-maker make-tokenizer "@$ 42 $@")) '(jsonic-program (s-exp " 42 "))) (check-equal? - (parse-tree (apply-tokenizer tokenize "hi")) + (parse-tree + (apply-tokenizer-maker make-tokenizer "hi")) '(jsonic-program (json-char "h") (json-char "i"))) (check-equal? - (parse-tree (apply-tokenizer tokenize "hi\n// comment\n@$ 42 $@")) + (parse-tree + (apply-tokenizer-maker make-tokenizer + "hi\n// comment\n@$ 42 $@")) '(jsonic-program (json-char "h") (json-char "i") diff --git a/beautiful-racket-demo/jsonic-demo-3/reader.rkt b/beautiful-racket-demo/jsonic-demo-3/reader.rkt index 16c592d..04b8e9a 100644 --- a/beautiful-racket-demo/jsonic-demo-3/reader.rkt +++ b/beautiful-racket-demo/jsonic-demo-3/reader.rkt @@ -2,8 +2,8 @@ (require "tokenizer.rkt" "parser.rkt" racket/contract) (define (read-syntax path port) - (define parse-tree (parse path (tokenize port))) - (define module-datum `(module jsonic-module jsonic-demo-3/expander + (define parse-tree (parse path (make-tokenizer port))) + (define module-datum `(module jsonic-module jsonic-demo/expander ,parse-tree)) (datum->syntax #f module-datum)) (provide (contract-out diff --git a/beautiful-racket-demo/jsonic-demo-3/tokenizer.rkt b/beautiful-racket-demo/jsonic-demo-3/tokenizer.rkt index 5887f01..995c833 100644 --- a/beautiful-racket-demo/jsonic-demo-3/tokenizer.rkt +++ b/beautiful-racket-demo/jsonic-demo-3/tokenizer.rkt @@ -13,10 +13,10 @@ (check-true (token? (token 'A-TOKEN-STRUCT "hi"))) (check-false (token? 42))) -(define (tokenize port) +(define (make-tokenizer port) (port-count-lines! port) (define (next-token) - (define our-lexer + (define jsonic-lexer (lexer [(eof) eof] [(from/to "//" "\n") (next-token)] @@ -33,22 +33,24 @@ #:column (col lexeme-start) #:span (- (pos lexeme-end) (pos lexeme-start)))])) - (our-lexer port)) + (jsonic-lexer port)) next-token) (provide (contract-out - [tokenize (input-port? . -> . (-> token?))])) + [make-tokenizer (input-port? . -> . (-> token?))])) (module+ test - (check-equal? (apply-tokenizer tokenize "// comment\n") empty) (check-equal? - (apply-tokenizer tokenize "@$ (+ 6 7) $@") + (apply-tokenizer-maker make-tokenizer "// comment\n") + empty) + (check-equal? + (apply-tokenizer-maker make-tokenizer "@$ (+ 6 7) $@") (list (token 'SEXP-TOK " (+ 6 7) " #:position 3 #:line 1 #:column 2 #:span 9))) (check-equal? - (apply-tokenizer tokenize "hi") + (apply-tokenizer-maker make-tokenizer "hi") (list (token 'CHAR-TOK "h" #:position 1 #:line 1