|
|
@ -16,11 +16,11 @@
|
|
|
|
[(eof) eof]
|
|
|
|
[(eof) eof]
|
|
|
|
[(or whitespace
|
|
|
|
[(or whitespace
|
|
|
|
(seq "//" (complement (seq any-string "\n" any-string)) "\n")) (next-token)]
|
|
|
|
(seq "//" (complement (seq any-string "\n" any-string)) "\n")) (next-token)]
|
|
|
|
[(char-set ",:[]{}@#") lexeme]
|
|
|
|
[(char-set ",:[]{}") lexeme]
|
|
|
|
|
|
|
|
[(seq "@" "(" any-string ")") (token 'S-EXP (string-trim lexeme "@"))]
|
|
|
|
[(seq (repetition 0 1 "-") (+ numeric) (repetition 0 1 (seq "." (* numeric))))
|
|
|
|
[(seq (repetition 0 1 "-") (+ numeric) (repetition 0 1 (seq "." (* numeric))))
|
|
|
|
(token 'NUMBER lexeme)] ;; Q: what is grammar for a JS number?
|
|
|
|
(token 'NUMBER lexeme)] ;; Q: what is grammar for a JS number?
|
|
|
|
[(seq "\"" (complement (seq any-string "\"" any-string)) "\"") (token 'STRING (string-trim lexeme "\""))]
|
|
|
|
[(seq "\"" (complement (seq any-string "\"" any-string)) "\"") (token 'STRING (string-trim lexeme "\""))]))
|
|
|
|
[any-char lexeme]))
|
|
|
|
|
|
|
|
(our-lexer port))
|
|
|
|
(our-lexer port))
|
|
|
|
next-token)
|
|
|
|
next-token)
|
|
|
|
|
|
|
|
|
|
|
@ -29,8 +29,3 @@
|
|
|
|
(define token-producer (tokenize ip))
|
|
|
|
(define token-producer (tokenize ip))
|
|
|
|
(for/list ([token (in-producer token-producer eof)])
|
|
|
|
(for/list ([token (in-producer token-producer eof)])
|
|
|
|
token))
|
|
|
|
token))
|
|
|
|
|
|
|
|
|
|
|
|
(test-tokenize #<<HERE
|
|
|
|
|
|
|
|
{"foo": @#(+ 4 2)#}
|
|
|
|
|
|
|
|
HERE
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|