mbutterick
/
br-parser-tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

*** empty log message ***

Browse Source 
original commit: ede718d6c88e4d577ccf1d651f744a69ef8c4cdb
tokens
Scott Owens 20 years ago
parent 5eed9288c9
commit 92a2693f23
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File

18
collects/parser-tools/lex.ss
Unescape Escape View File

@ -205,7 +205,7 @@
(lambda (ip) (lambda (ip)
(let ((first-pos (get-position ip)) (let ((first-pos (get-position ip))
(first-char (peek-char-or-special ip 0))) (first-char (peek-char-or-special ip 0)))
;; (printf "(peek-char-or-special port 0) = ~e~n" first-char) ;(printf "(peek-char-or-special port 0) = ~e~n" first-char)
(cond (cond
((eof-object? first-char) ((eof-object? first-char)
(do-match ip first-pos eof-action (read-char-or-special ip))) (do-match ip first-pos eof-action (read-char-or-special ip)))
@ -236,13 +236,13 @@
;; including a match at the current state ;; including a match at the current state
(longest-match-action (longest-match-action
(vector-ref actions start-state)) (vector-ref actions start-state))
;; how many bytes preceed char
(length-bytes 0)
;; how many characters have been read ;; how many characters have been read
;; including the one just read ;; including the one just read
(length-bytes (char-utf-8-length first-char))
(length-chars 1) (length-chars 1)
;; how many characters are in the longest match ;; how many characters are in the longest match
(longest-match-length 1)) (longest-match-length 1))
;; (printf "(peek-char-or-special port ~e) = ~e~n" (sub1 length-bytes) char)
(let ((next-state (let ((next-state
(cond (cond
((eof-object? char) #f) ((eof-object? char) #f)
@ -261,13 +261,17 @@
length-chars length-chars
(if act act longest-match-action)))) (if act act longest-match-action))))
(else (else
(let ((act (vector-ref actions next-state))) (let* ((act (vector-ref actions next-state))
(next-length-bytes (+ (char-utf-8-length char) length-bytes))
(next-char (peek-char-or-special ip next-length-bytes)))
#;(printf "(peek-char-or-special port ~e) = ~e~n"
next-length-bytes next-char)
(lexer-loop next-state (lexer-loop next-state
(peek-char-or-special ip length-bytes) next-char
(if act (if act
act act
longest-match-action) longest-match-action)
(+ (char-utf-8-length char) length-bytes) next-length-bytes
(add1 length-chars) (add1 length-chars)
(if act (if act
length-chars length-chars
@ -293,7 +297,7 @@
(position-offset first-pos) (position-offset first-pos)
(- (position-offset end-pos) (position-offset first-pos))))) (- (position-offset end-pos) (position-offset first-pos)))))
(let ((match (read-string longest-match-length lb))) (let ((match (read-string longest-match-length lb)))
;; (printf "(read-string ~e port) = ~e~n" longest-match-length match) ;(printf "(read-string ~e port) = ~e~n" longest-match-length match)
(do-match lb first-pos longest-match-action match))) (do-match lb first-pos longest-match-action match)))

Write Preview
Loading…
Cancel
Save