You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
beautiful-racket/beautiful-racket-demo/jsonic-demo-2/tokenizer.rkt

64 lines
1.9 KiB
Racket

8 years ago
#lang br/quicklang
8 years ago
(require brag/support racket/contract)
8 years ago
8 years ago
(module+ test
(require rackunit))
8 years ago
(define (jsonic-token? x)
8 years ago
(or (eof-object? x) (string? x) (token-struct? x)))
8 years ago
(module+ test
8 years ago
(check-true (jsonic-token? eof))
(check-true (jsonic-token? "a string"))
(check-true (jsonic-token? (token 'A-TOKEN-STRUCT "hi")))
(check-false (jsonic-token? 42)))
8 years ago
(define (make-tokenizer port)
8 years ago
(port-count-lines! port)
(define (next-token)
(define jsonic-lexer
8 years ago
(lexer
[(eof) eof]
[(from/to "//" "\n") (next-token)]
[(from/to "@$" "$@")
8 years ago
(token 'SEXP-TOK (trim-ends "@$" lexeme "$@")
8 years ago
#:position (+ (pos lexeme-start) 2)
8 years ago
#:line (line lexeme-start)
#:column (+ (col lexeme-start) 2)
#:span (- (pos lexeme-end)
(pos lexeme-start) 4))]
8 years ago
[any-char (token 'CHAR-TOK lexeme
8 years ago
#:position (pos lexeme-start)
8 years ago
#:line (line lexeme-start)
8 years ago
#:column (col lexeme-start)
#:span (- (pos lexeme-end)
(pos lexeme-start)))]))
(jsonic-lexer port))
8 years ago
next-token)
8 years ago
(provide
(contract-out
[make-tokenizer (input-port? . -> . (-> jsonic-token?))]))
8 years ago
(module+ test
(check-equal?
(apply-tokenizer-maker make-tokenizer "// comment\n")
empty)
(check-equal?
(apply-tokenizer-maker make-tokenizer "@$ (+ 6 7) $@")
8 years ago
(list (token 'SEXP-TOK " (+ 6 7) "
#:position 3
8 years ago
#:line 1
#:column 2
8 years ago
#:span 9)))
8 years ago
(check-equal?
(apply-tokenizer-maker make-tokenizer "hi")
8 years ago
(list (token 'CHAR-TOK "h"
8 years ago
#:position 1
8 years ago
#:line 1
#:column 0
#:span 1)
(token 'CHAR-TOK "i"
8 years ago
#:position 2
8 years ago
#:line 1
#:column 1
#:span 1))))