mbutterick
/
br-parser-tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

*** empty log message ***

Browse Source 
original commit: 412287bc9b21f82048af60551f506120e9560b37
tokens
Scott Owens 22 years ago
parent e3434d165f
commit 6d72e6b05d
2 changed files with 108 additions and 120 deletions
Show Stats Download Patch File Download Diff File

20
collects/parser-tools/examples/calc.ss
Unescape Escape View File

@ -13,14 +13,8 @@
(define vars (make-hash-table)) (define vars (make-hash-table))
(define-lex-abbrevs (define-lex-abbrevs
(lower-letter (- a z)) (lower-letter (- "a" "z"))
;; In the following line if we used (- A Z) dr/mzscheme would read it as (- a z) if
;; case-sensitivity is not enabled. (- #\A #\Z) will not be altered because dr/mzscheme
;; reads them as character constants and not as symbols. (- "A" "Z") would work as well
;; since dr/mzscheme would read them as strings. The lexer generator treates single character
;; strings and symbols the same as an actual #\ character. #cs(- A Z) works too because the #cs
;; tells dr/mzscheme to read the next expression with case-sensitivity turned on.
(upper-letter (- #\A #\Z)) (upper-letter (- #\A #\Z))
;; (- 0 9) would not work because the lexer does not understand numbers. (- #\0 #\9) is ok too. ;; (- 0 9) would not work because the lexer does not understand numbers. (- #\0 #\9) is ok too.
@ -34,17 +28,15 @@
[(: #\tab #\space) (calcl input-port)] [(: #\tab #\space) (calcl input-port)]
;; The parser will treat the return of 'newline the same as (token-newline) ;; The parser will treat the return of 'newline the same as (token-newline)
[#\newline 'newline] [#\newline 'newline]
[(: = + - * / ^) (string->symbol lexeme)] [(: "=" "+" "-" "*" "/" "^") (string->symbol lexeme)]
["(" 'OP] ["(" 'OP]
[")" 'CP] [")" 'CP]
[sin (token-FNCT sin)] ["sin" (token-FNCT sin)]
;; It the parens are left off of an "invocation" of an abbreviation, it means the [(+ (: lower-letter upper-letter)) (token-VAR (string->symbol lexeme))]
;; character sequence instead. [(+ digit) (token-NUM (string->number lexeme))]
[(+ (: (lower-letter) (upper-letter))) (token-VAR (string->symbol lexeme))]
[(+ (digit)) (token-NUM (string->number lexeme))]
;; Strings which dr/mzscheme does not think of as symbols (such as . or ,) must be ;; Strings which dr/mzscheme does not think of as symbols (such as . or ,) must be
;; entered as a string or character. "." would also be ok. ;; entered as a string or character. "." would also be ok.
[(@ (+ (digit)) #\. (* (digit))) (token-NUM (string->number lexeme))])) [(@ (+ digit) #\. (* digit)) (token-NUM (string->number lexeme))]))
(define calcp (define calcp

208
collects/parser-tools/examples/read.ss
Unescape Escape View File

@ -17,24 +17,24 @@
(lexer-src-pos (lexer-src-pos
;; Skip comments, without accumulating extra position information ;; Skip comments, without accumulating extra position information
[(: (whitespace) (comment)) (return-without-pos (scheme-lexer input-port))] [(: whitespace comment) (return-without-pos (scheme-lexer input-port))]
["#t" (token-DATUM #t)] ["#t" (token-DATUM #t)]
["#f" (token-DATUM #f)] ["#f" (token-DATUM #f)]
[(@ "#\\" (any)) (token-DATUM (caddr (string->list lexeme)))] [(@ "#\\" any) (token-DATUM (caddr (string->list lexeme)))]
["#\\space" (token-DATUM #\space)] ["#\\space" (token-DATUM #\space)]
["#\\newline" (token-DATUM #\newline)] ["#\\newline" (token-DATUM #\newline)]
[(: (@ (initial) (* (subsequent))) + - "...") (token-DATUM (string->symbol lexeme))] [(: (@ initial (* subsequent)) "+" "-" "...") (token-DATUM (string->symbol lexeme))]
[#\" (token-DATUM (list->string (get-string-token input-port)))] [#\" (token-DATUM (list->string (get-string-token input-port)))]
[#\( 'OP] [#\( 'OP]
[#\) 'CP] [#\) 'CP]
[#\[ 'OP] [#\[ 'OP]
[#\] 'CP] [#\] 'CP]
["#(" 'HASHOP] ["#(" 'HASHOP]
[(num2) (token-DATUM (string->number lexeme 2))] [num2 (token-DATUM (string->number lexeme 2))]
[(num8) (token-DATUM (string->number lexeme 8))] [num8 (token-DATUM (string->number lexeme 8))]
[(num10) (token-DATUM (string->number lexeme 10))] [num10 (token-DATUM (string->number lexeme 10))]
[(num16) (token-DATUM (string->number lexeme 16))] [num16 (token-DATUM (string->number lexeme 16))]
["'" 'QUOTE] ["'" 'QUOTE]
["`" 'QUASIQUOTE] ["`" 'QUASIQUOTE]
["," 'UNQUOTE] ["," 'UNQUOTE]
@ -53,117 +53,113 @@
(define-lex-abbrevs (define-lex-abbrevs
[any (- #\000 #\377)] [any (- #\000 #\377)]
[letter (: (- a z) (- #\A #\Z))] [letter (: (- "a" "z") (- #\A #\Z))]
[digit (- #\0 #\9)] [digit (- #\0 #\9)]
[whitespace (: #\newline #\return #\tab #\space #\vtab)] [whitespace (: #\newline #\return #\tab #\space #\vtab)]
[initial (: (letter) ! $ % & * / : < = > ? ^ _ ~ @)] [initial (: letter "!" "$" "%" "&" "*" "/" ":" "<" "=" ">" "?" "^" "_" "~" "@")]
[subsequent (: (initial) (digit) + - #\. @)] [subsequent (: initial digit "+" "-" #\. "@")]
[comment (@ #\; (* (^ #\newline)) #\newline)] [comment (@ #\; (* (^ #\newline)) #\newline)]
; [numR (@ (prefixR) (complexR))] ; [numR (@ prefixR complexR)]
; [complexR (: (realR) ; [complexR (: realR
; (@ (realR) @ (realR)) ; (@ realR "@" realR)
; (@ (realR) + (urealR) i) ; (@ realR "+" urealR "i")
; (@ (realR) - (urealR) i) ; (@ realR "-" urealR "i")
; (@ (realR) + i) ; (@ realR "+i")
; (@ (realR) - i) ; (@ realR "-i")
; (@ + (urealR) i) ; (@ "+" urealR "i")
; (@ - (urealR) i) ; (@ "-" urealR "i")
; (@ + i) ; (@ "+i")
; (@ - i))] ; (@ "-i"))]
; [realR (@ (sign) (urealR))] ; [realR (@ sign urealR)]
; [urealR (: (uintegerR) (@ (uintegerR) / (uintegerR)) (decimalR))] ; [urealR (: uintegerR (@ uintegerR "/" uintegerR) decimalR)]
; [uintegerR (@ (+ (digitR)) (* #\#))] ; [uintegerR (@ (+ digitR) (* #\#))]
; [prefixR (: (@ (radixR) (exactness)) ; [prefixR (: (@ radixR exactness)
; (@ (exactness) (radixR)))] ; (@ exactness radixR))]
; [numR (@ (prefixR) (complexR))]
; [complexR (: (realR)
; (@ (realR) @ (realR))
[num2 (@ (prefix2) (complex2))]
[complex2 (: (real2)
(@ (real2) @ (real2))
(@ (real2) + (ureal2) i)
(@ (real2) - (ureal2) i)
(@ (real2) + i)
(@ (real2) - i)
(@ + (ureal2) i)
(@ - (ureal2) i)
(@ + i)
(@ - i))]
[real2 (@ (sign) (ureal2))]
[ureal2 (: (uinteger2) (@ (uinteger2) / (uinteger2)))]
[uinteger2 (@ (+ (digit2)) (* #\#))]
[prefix2 (: (@ (radix2) (exactness))
(@ (exactness) (radix2)))]
[radix2 "#b"]
[digit2 (: #\0 #\1)]
[num8 (@ (prefix8) (complex8))] [num2 (@ prefix2 complex2)]
[complex8 (: (real8) [complex2 (: real2
(@ (real8) @ (real8)) (@ real2 "@" real2)
(@ (real8) + (ureal8) i) (@ real2 "+" ureal2 "i")
(@ (real8) - (ureal8) i) (@ real2 "-" ureal2 "i")
(@ (real8) + i) (@ real2 "+i")
(@ (real8) - i) (@ real2 "-i")
(@ + (ureal8) i) (@ "+" ureal2 "i")
(@ - (ureal8) i) (@ "-" ureal2 "i")
(@ + i) (@ "+i")
(@ - i))] (@ "-i"))]
[real8 (@ (sign) (ureal8))] [real2 (@ sign ureal2)]
[ureal8 (: (uinteger8) (@ (uinteger8) / (uinteger8)))] [ureal2 (: uinteger2 (@ uinteger2 "/" uinteger2))]
[uinteger8 (@ (+ (digit8)) (* #\#))] [uinteger2 (@ (+ digit2) (* #\#))]
[prefix8 (: (@ (radix8) (exactness)) [prefix2 (: (@ radix2 exactness)
(@ (exactness) (radix8)))] (@ exactness radix2))]
[radix2 "#b"]
[digit2 (: "0" "1")]
[num8 (@ prefix8 complex8)]
[complex8 (: real8
(@ real8 "@" real8)
(@ real8 "+" ureal8 "i")
(@ real8 "-" ureal8 "i")
(@ real8 "+i")
(@ real8 "-i")
(@ "+" ureal8 "i")
(@ "-" ureal8 "i")
(@ "+i")
(@ "-i"))]
[real8 (@ sign ureal8)]
[ureal8 (: uinteger8 (@ uinteger8 "/" uinteger8))]
[uinteger8 (@ (+ digit8) (* #\#))]
[prefix8 (: (@ radix8 exactness)
(@ exactness radix8))]
[radix8 "#o"] [radix8 "#o"]
[digit8 (- #\0 #\7)] [digit8 (- "0" "7")]
[num10 (@ (prefix10) (complex10))] [num10 (@ prefix10 complex10)]
[complex10 (: (real10) [complex10 (: real10
(@ (real10) @ (real10)) (@ real10 "@" real10)
(@ (real10) + (ureal10) i) (@ real10 "+" ureal10 "i")
(@ (real10) - (ureal10) i) (@ real10 "-" ureal10 "i")
(@ (real10) + i) (@ real10 "+i")
(@ (real10) - i) (@ real10 "-i")
(@ + (ureal10) i) (@ "+" ureal10 "i")
(@ - (ureal10) i) (@ "-" ureal10 "i")
(@ + i) (@ "+i")
(@ - i))] (@ "-i"))]
[real10 (@ (sign) (ureal10))] [real10 (@ sign ureal10)]
[ureal10 (: (uinteger10) (@ (uinteger10) / (uinteger10)) (decimal10))] [ureal10 (: uinteger10 (@ uinteger10 "/" uinteger10) decimal10)]
[uinteger10 (@ (+ (digit10)) (* #\#))] [uinteger10 (@ (+ digit10) (* #\#))]
[prefix10 (: (@ (radix10) (exactness)) [prefix10 (: (@ radix10 exactness)
(@ (exactness) (radix10)))] (@ exactness radix10))]
[radix10 (: (@) "#d")] [radix10 (: (@) "#d")]
[digit10 (digit)] [digit10 digit]
[decimal10 (: (@ (uinteger10) (suffix)) [decimal10 (: (@ uinteger10 suffix)
(@ #\. (+ (digit10)) (* #\#) (suffix)) (@ #\. (+ digit10) (* #\#) suffix)
(@ (+ (digit10)) #\. (* (digit10)) (* #\#) (suffix)) (@ (+ digit10) #\. (* digit10) (* #\#) suffix)
(@ (+ (digit10)) (+ #\#) #\. (* #\#) (suffix)))] (@ (+ digit10) (+ #\#) #\. (* #\#) suffix))]
[num16 (@ (prefix16) (complex16))] [num16 (@ prefix16 complex16)]
[complex16 (: (real16) [complex16 (: real16
(@ (real16) @ (real16)) (@ real16 "@" real16)
(@ (real16) + (ureal16) i) (@ real16 "+" ureal16 "i")
(@ (real16) - (ureal16) i) (@ real16 "-" ureal16 "i")
(@ (real16) + i) (@ real16 "+i")
(@ (real16) - i) (@ real16 "-i")
(@ + (ureal16) i) (@ "+" ureal16 "i")
(@ - (ureal16) i) (@ "-" ureal16 "i")
(@ + i) "+i"
(@ - i))] "-i")]
[real16 (@ (sign) (ureal16))] [real16 (@ sign ureal16)]
[ureal16 (: (uinteger16) (@ (uinteger16) / (uinteger16)))] [ureal16 (: uinteger16 (@ uinteger16 "/" uinteger16))]
[uinteger16 (@ (+ (digit16)) (* #\#))] [uinteger16 (@ (+ digit16) (* #\#))]
[prefix16 (: (@ (radix16) (exactness)) [prefix16 (: (@ radix16 exactness)
(@ (exactness) (radix16)))] (@ exactness radix16))]
[radix16 "#x"] [radix16 "#x"]
[digit16 (: (digit) (- #\a #\f) (- #\A #\F))] [digit16 (: digit (- #\a #\f) (- #\A #\F))]
[suffix (: (@) (@ (exponent-marker) (sign) (+ (digit10))))] [suffix (: (@) (@ exponent-marker sign (+ digit10)))]
[exponent-marker (: e s f d l)] [exponent-marker (: "e" "s" "f" "d" "l")]
[sign (: (@) + -)] [sign (: (@) "+" "-")]
[exactness (: (@) "#i" "#e")]) [exactness (: (@) "#i" "#e")])

Write Preview
Loading…
Cancel
Save