add new token

pull/10/head
Matthew Butterick 7 years ago
parent bf76152db6
commit c35b02c986

@ -8,8 +8,8 @@
(define-lex-abbrev id-kapu
(:or whitespace (char-set "()[]{}\",'`;#|\\")))
(define-lex-abbrev id
(:seq (:~ (:or "-" id-kapu)) (:* (:~ id-kapu))))
(define-lex-abbrev racket-id
(:seq "[" (:~ (:or "-" id-kapu)) (:* (:~ id-kapu)) "]"))
(define basic-lexer
(lexer-srcloc
@ -17,15 +17,17 @@
["\n" (token 'NEWLINE lexeme)]
[whitespace (token lexeme #:skip? #t)]
[(from/stop-before "rem" "\n") (token 'REM lexeme)]
[racket-id (token 'RACKET-ID (string->symbol (trim-ends "[" lexeme "]")))]
[reserved-terms (token lexeme lexeme)]
[(:seq alphabetic (:* (:or alphabetic numeric "$")))
(token 'ID (string->symbol lexeme))]
[digits (token 'INTEGER (string->number lexeme))]
[(:or (:seq (:? digits) "." digits)
(:seq digits "."))
(token 'DECIMAL (string->number lexeme))]
[id (token 'ID (string->symbol lexeme))]
[(:or (from/to "\"" "\"") (from/to "'" "'"))
(token 'STRING
(substring lexeme
1 (sub1 (string-length lexeme))))]))
(provide basic-lexer)
(provide basic-lexer)

@ -22,7 +22,7 @@ b-for : /"for" b-id /"=" b-expr /"to" b-expr [/"step" b-expr]
b-next : /"next" b-id
b-def : /"def" b-id /"(" ID [/"," ID]* /")" /"=" b-expr
b-import : /"import" b-import-name
@b-import-name : ID | STRING
@b-import-name : RACKET-ID | STRING
b-export : /"export" b-export-name
@b-export-name : ID
b-expr : b-or-expr
@ -35,6 +35,6 @@ b-product : [b-product ("*"|"/"|"mod")] b-neg
b-neg : ["-"] b-expt
b-expt : [b-expt ("^")] b-value
@b-value : b-number | b-id | /"(" b-expr /")" | b-func
b-func : ID /"(" b-expr [/"," b-expr]* /")"
b-func : (ID | RACKET-ID) /"(" b-expr [/"," b-expr]* /")"
@b-number : INTEGER | DECIMAL
b-repl : (b-statement | b-expr) (/":" [b-repl])*

Loading…
Cancel
Save