You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
30 lines
1013 B
Racket
30 lines
1013 B
Racket
#lang br
|
|
(require brag/support)
|
|
(provide make-tokenizer)
|
|
|
|
(define-lex-abbrev digit (char-set "0123456789"))
|
|
(define-lex-abbrev hex-digit (:or digit (char-set "ABCDEF")))
|
|
(define-lex-abbrev digits (:+ digit))
|
|
(define-lex-abbrev sign (:? (:or "+" "-")))
|
|
(define-lex-abbrev blackspace (:~ whitespace))
|
|
|
|
(define-lex-abbrev nonreg-char (:seq "#" hex-digit hex-digit))
|
|
|
|
(define (make-tokenizer src port)
|
|
(port-count-lines! port)
|
|
(lexer-file-path src)
|
|
(define lex-once
|
|
(lexer-srcloc
|
|
[(eof) eof]
|
|
[whitespace
|
|
(token 'WHITESPACE lexeme #:skip? #t)]
|
|
[(:or "true" "false") (token 'BOOLEAN (equal? lexeme "true"))]
|
|
[(:seq sign digits) (token 'INT (string->number lexeme))]
|
|
[(:seq sign (:or (:seq digits "." (:? digits))
|
|
(:seq "." digits)))
|
|
(token 'REAL (string->number lexeme))]
|
|
[(:seq "/" (:+ (:or nonreg-char blackspace)))
|
|
(token 'NAME lexeme)]
|
|
["null" (token 'NULL 'null)]
|
|
[any-char (token 'CHAR lexeme)]))
|
|
(λ () (lex-once port))) |