diff --git a/beautiful-racket/br/demo/jsonic-2/tokenizer.rkt b/beautiful-racket/br/demo/jsonic-2/tokenizer.rkt index 97df881..dcfae04 100644 --- a/beautiful-racket/br/demo/jsonic-2/tokenizer.rkt +++ b/beautiful-racket/br/demo/jsonic-2/tokenizer.rkt @@ -24,40 +24,39 @@ [(from/to "//" "\n") (next-token)] [(from/to "@$" "$@") (token 'SEXP-TOK (trim-ends "@$" lexeme "$@") + #:position (+ (pos lexeme-start) 2) #:line (line lexeme-start) #:column (+ (col lexeme-start) 2) - #:position (+ (pos lexeme-start) 2) #:span (- (pos lexeme-end) (pos lexeme-start) 4))] [any-char (token 'CHAR-TOK lexeme + #:position (pos lexeme-start) #:line (line lexeme-start) #:column (col lexeme-start) - #:position (pos lexeme-start) #:span (- (pos lexeme-end) (pos lexeme-start)))])) - (our-lexer port)) + (our-lexer port)) next-token) (provide tokenize) - (module+ test (check-equal? (apply-tokenizer tokenize "// comment\n") empty) (check-equal? (apply-tokenizer tokenize "@$ (+ 6 7) $@") (list (token 'SEXP-TOK " (+ 6 7) " - #:line 11 - #:column 2 #:position 3 + #:line 1 + #:column 2 #:span 9))) (check-equal? (apply-tokenizer tokenize "hi") (list (token 'CHAR-TOK "h" + #:position 1 #:line 1 #:column 0 - #:position 1 #:span 1) (token 'CHAR-TOK "i" + #:position 2 #:line 1 #:column 1 - #:position 2 #:span 1)))) \ No newline at end of file