mbutterick
/
br-parser-tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor into racket/base

Browse Source
pull/5/head
Matthew Butterick 7 years ago
parent fc1e00bc2a
commit fd446e6013
28 changed files with 3628 additions and 4148 deletions
Show Stats Download Patch File Download Diff File

391
br-parser-tools-lib/br-parser-tools/cfg-parser.rkt
Unescape Escape View File

@ -47,18 +47,16 @@
(define-struct tasks (active active-back waits multi-waits cache progress?)) (define-struct tasks (active active-back waits multi-waits cache progress?))
(define-for-syntax make-token-identifier-mapping make-hasheq) (define-for-syntax make-token-identifier-mapping make-hasheq)
(define-for-syntax token-identifier-mapping-get (define-for-syntax (token-identifier-mapping-get t tok [fail #f])
(case-lambda (if fail
[(t tok) (hash-ref t (syntax-e tok) fail)
(hash-ref t (syntax-e tok))] (hash-ref t (syntax-e tok))))
[(t tok fail)
(hash-ref t (syntax-e tok) fail)])) (define-for-syntax (token-identifier-mapping-put! t tok v)
(define-for-syntax token-identifier-mapping-put! (hash-set! t (syntax-e tok) v))
(lambda (t tok v)
(hash-set! t (syntax-e tok) v))) (define-for-syntax (token-identifier-mapping-map t f)
(define-for-syntax token-identifier-mapping-map (hash-map t f))
(lambda (t f)
(hash-map t f)))
;; Used to calculate information on the grammar, such as whether ;; Used to calculate information on the grammar, such as whether
;; a particular non-terminal is "simple" instead of recursively defined. ;; a particular non-terminal is "simple" instead of recursively defined.
@ -71,7 +69,7 @@
(cdr as) (cdr bs))])) (cdr as) (cdr bs))]))
(let loop () (let loop ()
(when (ormap-all #f (when (ormap-all #f
(lambda (nt pats) (λ (nt pats)
(let ([old (bound-identifier-mapping-get nts nt)]) (let ([old (bound-identifier-mapping-get nts nt)])
(let ([new (proc nt pats old)]) (let ([new (proc nt pats old)])
(if (equal? old new) (if (equal? old new)
@ -88,149 +86,120 @@
(define (parse-and simple-a? parse-a parse-b (define (parse-and simple-a? parse-a parse-b
stream last-consumed-token depth end success-k fail-k stream last-consumed-token depth end success-k fail-k
max-depth tasks) max-depth tasks)
(letrec ([mk-got-k (define ((mk-got-k success-k fail-k) val stream last-consumed-token depth max-depth tasks next1-k)
(lambda (success-k fail-k)
(lambda (val stream last-consumed-token depth max-depth tasks next1-k)
(if simple-a? (if simple-a?
(parse-b val stream last-consumed-token depth end (parse-b val stream last-consumed-token depth end
(mk-got2-k success-k fail-k next1-k) (mk-got2-k success-k fail-k next1-k)
(mk-fail2-k success-k fail-k next1-k) (mk-fail2-k success-k fail-k next1-k)
max-depth tasks) max-depth tasks)
(parallel-or (parallel-or
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(parse-b val stream last-consumed-token depth end (parse-b val stream last-consumed-token depth end
success-k fail-k success-k fail-k
max-depth tasks)) max-depth tasks))
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(next1-k (mk-got-k success-k fail-k) (next1-k (mk-got-k success-k fail-k)
fail-k max-depth tasks)) fail-k max-depth tasks))
success-k fail-k max-depth tasks))))] success-k fail-k max-depth tasks)))
[mk-got2-k
(lambda (success-k fail-k next1-k) (define ((mk-got2-k success-k fail-k next1-k) val stream last-consumed-token depth max-depth tasks next-k)
(lambda (val stream last-consumed-token depth max-depth tasks next-k)
(success-k val stream last-consumed-token depth max-depth tasks (success-k val stream last-consumed-token depth max-depth tasks
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(next-k (mk-got2-k success-k fail-k next1-k) (next-k (mk-got2-k success-k fail-k next1-k)
(mk-fail2-k success-k fail-k next1-k) (mk-fail2-k success-k fail-k next1-k)
max-depth tasks)))))] max-depth tasks))))
[mk-fail2-k
(lambda (success-k fail-k next1-k) (define ((mk-fail2-k success-k fail-k next1-k) max-depth tasks)
(lambda (max-depth tasks) (next1-k (mk-got-k success-k fail-k) fail-k max-depth tasks))
(next1-k (mk-got-k success-k fail-k)
fail-k
max-depth
tasks)))])
(parse-a stream last-consumed-token depth end (parse-a stream last-consumed-token depth end
(mk-got-k success-k fail-k) (mk-got-k success-k fail-k)
fail-k fail-k
max-depth tasks))) max-depth tasks))
;; Parallel or for non-terminal alternatives ;; Parallel or for non-terminal alternatives
(define (parse-parallel-or parse-a parse-b stream last-consumed-token depth end success-k fail-k max-depth tasks) (define (parse-parallel-or parse-a parse-b stream last-consumed-token depth end success-k fail-k max-depth tasks)
(parallel-or (lambda (success-k fail-k max-depth tasks) (parallel-or (λ (success-k fail-k max-depth tasks)
(parse-a stream last-consumed-token depth end success-k fail-k max-depth tasks)) (parse-a stream last-consumed-token depth end success-k fail-k max-depth tasks))
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(parse-b stream last-consumed-token depth end success-k fail-k max-depth tasks)) (parse-b stream last-consumed-token depth end success-k fail-k max-depth tasks))
success-k fail-k max-depth tasks)) success-k fail-k max-depth tasks))
;; Generic parallel-or ;; Generic parallel-or
(define (parallel-or parse-a parse-b success-k fail-k max-depth tasks) (define (parallel-or parse-a parse-b success-k fail-k max-depth tasks)
(define answer-key (gensym)) (define answer-key (gensym))
(letrec ([gota-k (define (gota-k val stream last-consumed-token depth max-depth tasks next-k)
(lambda (val stream last-consumed-token depth max-depth tasks next-k)
(report-answer answer-key (report-answer answer-key
max-depth max-depth
tasks tasks
(list val stream last-consumed-token depth next-k)))] (list val stream last-consumed-token depth next-k)))
[faila-k (define (faila-k max-depth tasks)
(lambda (max-depth tasks)
(report-answer answer-key (report-answer answer-key
max-depth max-depth
tasks tasks
null))]) null))
(let* ([tasks (queue-task (let* ([tasks (queue-task tasks (λ (max-depth tasks)
tasks (parse-a gota-k faila-k max-depth tasks)))]
(lambda (max-depth tasks) [tasks (queue-task tasks (λ (max-depth tasks)
(parse-a gota-k (parse-b gota-k faila-k max-depth tasks)))]
faila-k [queue-next (λ (next-k tasks)
max-depth tasks)))] (queue-task tasks (λ (max-depth tasks)
[tasks (queue-task (next-k gota-k faila-k max-depth tasks))))])
tasks (define ((mk-got-one immediate-next? get-nth success-k) val stream last-consumed-token depth max-depth tasks next-k)
(lambda (max-depth tasks)
(parse-b gota-k
faila-k
max-depth tasks)))]
[queue-next (lambda (next-k tasks)
(queue-task tasks
(lambda (max-depth tasks)
(next-k gota-k
faila-k
max-depth tasks))))])
(letrec ([mk-got-one
(lambda (immediate-next? get-nth success-k)
(lambda (val stream last-consumed-token depth max-depth tasks next-k)
(let ([tasks (if immediate-next? (let ([tasks (if immediate-next?
(queue-next next-k tasks) (queue-next next-k tasks)
tasks)]) tasks)])
(success-k val stream last-consumed-token depth max-depth (success-k val stream last-consumed-token depth max-depth
tasks tasks
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(let ([tasks (if immediate-next? (let ([tasks (if immediate-next?
tasks tasks
(queue-next next-k tasks))]) (queue-next next-k tasks))])
(get-nth max-depth tasks success-k fail-k)))))))] (get-nth max-depth tasks success-k fail-k))))))
[get-first (define (get-first max-depth tasks success-k fail-k)
(lambda (max-depth tasks success-k fail-k)
(wait-for-answer #f max-depth tasks answer-key (wait-for-answer #f max-depth tasks answer-key
(mk-got-one #t get-first success-k) (mk-got-one #t get-first success-k)
(lambda (max-depth tasks) (λ (max-depth tasks)
(get-second max-depth tasks success-k fail-k)) (get-second max-depth tasks success-k fail-k))
#f))] #f))
[get-second (define (get-second max-depth tasks success-k fail-k)
(lambda (max-depth tasks success-k fail-k)
(wait-for-answer #f max-depth tasks answer-key (wait-for-answer #f max-depth tasks answer-key
(mk-got-one #f get-second success-k) (mk-got-one #f get-second success-k)
fail-k #f))]) fail-k #f))
(get-first max-depth tasks success-k fail-k))))) (get-first max-depth tasks success-k fail-k)))
;; Non-terminal alternatives where the first is "simple" can be done ;; Non-terminal alternatives where the first is "simple" can be done
;; sequentially, which is simpler ;; sequentially, which is simpler
(define (parse-or parse-a parse-b (define (parse-or parse-a parse-b
stream last-consumed-token depth end success-k fail-k max-depth tasks) stream last-consumed-token depth end success-k fail-k max-depth tasks)
(letrec ([mk-got-k (define ((mk-got-k success-k fail-k) val stream last-consumed-token depth max-depth tasks next-k)
(lambda (success-k fail-k)
(lambda (val stream last-consumed-token depth max-depth tasks next-k)
(success-k val stream last-consumed-token depth (success-k val stream last-consumed-token depth
max-depth tasks max-depth tasks
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(next-k (mk-got-k success-k fail-k) (next-k (mk-got-k success-k fail-k)
(mk-fail-k success-k fail-k) (mk-fail-k success-k fail-k)
max-depth tasks)))))] max-depth tasks))))
[mk-fail-k (define ((mk-fail-k success-k fail-k) max-depth tasks)
(lambda (success-k fail-k) (parse-b stream last-consumed-token depth end success-k fail-k max-depth tasks))
(lambda (max-depth tasks)
(parse-b stream last-consumed-token depth end success-k fail-k max-depth tasks)))])
(parse-a stream last-consumed-token depth end (parse-a stream last-consumed-token depth end
(mk-got-k success-k fail-k) (mk-got-k success-k fail-k)
(mk-fail-k success-k fail-k) (mk-fail-k success-k fail-k)
max-depth tasks))) max-depth tasks))
;; Starts a thread ;; Starts a thread
(define queue-task (define (queue-task tasks t [progress? #t])
(lambda (tasks t [progress? #t])
(make-tasks (tasks-active tasks) (make-tasks (tasks-active tasks)
(cons t (tasks-active-back tasks)) (cons t (tasks-active-back tasks))
(tasks-waits tasks) (tasks-waits tasks)
(tasks-multi-waits tasks) (tasks-multi-waits tasks)
(tasks-cache tasks) (tasks-cache tasks)
(or progress? (tasks-progress? tasks))))) (or progress? (tasks-progress? tasks))))
;; Reports an answer to a waiting thread: ;; Reports an answer to a waiting thread:
(define (report-answer answer-key max-depth tasks val) (define (report-answer answer-key max-depth tasks val)
(let ([v (hash-ref (tasks-waits tasks) answer-key (lambda () #f))]) (define v (hash-ref (tasks-waits tasks) answer-key (λ () #f)))
(if v (if v
(let ([tasks (make-tasks (cons (v val) (let ([tasks (make-tasks (cons (v val) (tasks-active tasks))
(tasks-active tasks))
(tasks-active-back tasks) (tasks-active-back tasks)
(tasks-waits tasks) (tasks-waits tasks)
(tasks-multi-waits tasks) (tasks-multi-waits tasks)
@ -241,29 +210,29 @@
;; We have an answer ready too fast; wait ;; We have an answer ready too fast; wait
(swap-task max-depth (swap-task max-depth
(queue-task tasks (queue-task tasks
(lambda (max-depth tasks) (λ (max-depth tasks)
(report-answer answer-key max-depth tasks val)) (report-answer answer-key max-depth tasks val))
#f))))) #f))))
;; Reports an answer to multiple waiting threads: ;; Reports an answer to multiple waiting threads:
(define (report-answer-all answer-key max-depth tasks val k) (define (report-answer-all answer-key max-depth tasks val k)
(let ([v (hash-ref (tasks-multi-waits tasks) answer-key (lambda () null))]) (define v (hash-ref (tasks-multi-waits tasks) answer-key (λ () null)))
(hash-remove! (tasks-multi-waits tasks) answer-key) (hash-remove! (tasks-multi-waits tasks) answer-key)
(let ([tasks (make-tasks (append (map (lambda (a) (a val)) v) (let ([tasks (make-tasks (append (map (λ (a) (a val)) v)
(tasks-active tasks)) (tasks-active tasks))
(tasks-active-back tasks) (tasks-active-back tasks)
(tasks-waits tasks) (tasks-waits tasks)
(tasks-multi-waits tasks) (tasks-multi-waits tasks)
(tasks-cache tasks) (tasks-cache tasks)
#t)]) #t)])
(k max-depth tasks)))) (k max-depth tasks)))
;; Waits for an answer; if `multi?' is #f, this is sole waiter, otherwise ;; Waits for an answer; if `multi?' is #f, this is sole waiter, otherwise
;; there might be many. Use wither #t or #f (and `report-answer' or ;; there might be many. Use wither #t or #f (and `report-answer' or
;; `report-answer-all', resptively) consistently for a particular answer key. ;; `report-answer-all', resptively) consistently for a particular answer key.
(define (wait-for-answer multi? max-depth tasks answer-key success-k fail-k deadlock-k) (define (wait-for-answer multi? max-depth tasks answer-key success-k fail-k deadlock-k)
(let ([wait (lambda (val) (let ([wait (λ (val)
(lambda (max-depth tasks) (λ (max-depth tasks)
(if val (if val
(if (null? val) (if (null? val)
(fail-k max-depth tasks) (fail-k max-depth tasks)
@ -273,7 +242,7 @@
(if multi? (if multi?
(hash-set! (tasks-multi-waits tasks) answer-key (hash-set! (tasks-multi-waits tasks) answer-key
(cons wait (hash-ref (tasks-multi-waits tasks) answer-key (cons wait (hash-ref (tasks-multi-waits tasks) answer-key
(lambda () null)))) (λ () null))))
(hash-set! (tasks-waits tasks) answer-key wait)) (hash-set! (tasks-waits tasks) answer-key wait))
(let ([tasks (make-tasks (tasks-active tasks) (let ([tasks (make-tasks (tasks-active tasks)
(tasks-active-back tasks) (tasks-active-back tasks)
@ -302,8 +271,8 @@
(make-tasks (apply (make-tasks (apply
append append
(hash-map (tasks-multi-waits tasks) (hash-map (tasks-multi-waits tasks)
(lambda (k l) (λ (k l)
(map (lambda (v) (v #f)) l)))) (map (λ (v) (v #f)) l))))
(tasks-active-back tasks) (tasks-active-back tasks)
(tasks-waits tasks) (tasks-waits tasks)
(make-hasheq) (make-hasheq)
@ -325,11 +294,9 @@
(define no-pos-val (make-position #f #f #f)) (define no-pos-val (make-position #f #f #f))
(define-for-syntax no-pos (define-for-syntax no-pos
(let ([npv ((syntax-local-certifier) #'no-pos-val)]) (let ([npv ((syntax-local-certifier) #'no-pos-val)])
(lambda (stx) npv))) (λ (stx) npv)))
(define-for-syntax at-tok-pos (define-for-syntax ((at-tok-pos sel expr) stx)
(lambda (sel expr) #`(let ([v #,expr]) (if v (#,sel v) no-pos-val)))
(lambda (stx)
#`(let ([v #,expr]) (if v (#,sel v) no-pos-val)))))
;; Builds a matcher for a particular alternative ;; Builds a matcher for a particular alternative
(define-for-syntax (build-match nts toks pat handle $ctx) (define-for-syntax (build-match nts toks pat handle $ctx)
@ -337,27 +304,23 @@
[pos 1]) [pos 1])
(if (null? pat) (if (null? pat)
#`(success-k #,handle stream last-consumed-token depth max-depth tasks #`(success-k #,handle stream last-consumed-token depth max-depth tasks
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(fail-k max-depth tasks))) (fail-k max-depth tasks)))
(let ([id (datum->syntax (car pat) (let ([id (datum->syntax (car pat) (string->symbol (format "$~a" pos)))]
(string->symbol (format "$~a" pos)))] [id-start-pos (datum->syntax (car pat) (string->symbol (format "$~a-start-pos" pos)))]
[id-start-pos (datum->syntax (car pat) [id-end-pos (datum->syntax (car pat) (string->symbol (format "$~a-end-pos" pos)))]
(string->symbol (format "$~a-start-pos" pos)))] [n-end-pos (and (null? (cdr pat)) (datum->syntax (car pat) '$n-end-pos))])
[id-end-pos (datum->syntax (car pat)
(string->symbol (format "$~a-end-pos" pos)))]
[n-end-pos (and (null? (cdr pat))
(datum->syntax (car pat) '$n-end-pos))])
(cond (cond
[(bound-identifier-mapping-get nts (car pat) (lambda () #f)) [(bound-identifier-mapping-get nts (car pat) (λ () #f))
;; Match non-termimal ;; Match non-termimal
#`(parse-and #`(parse-and
;; First part is simple? (If so, we don't have to parallelize the `and'.) ;; First part is simple? (If so, we don't have to parallelize the `and'.)
#,(let ([l (bound-identifier-mapping-get nts (car pat) (lambda () #f))]) #,(let ([l (bound-identifier-mapping-get nts (car pat) (λ () #f))])
(or (not l) (or (not l)
(andmap values (caddr l)))) (andmap values (caddr l))))
#,(car pat) #,(car pat)
(let ([original-stream stream]) (let ([original-stream stream])
(lambda (#,id stream last-consumed-token depth end success-k fail-k max-depth tasks) (λ (#,id stream last-consumed-token depth end success-k fail-k max-depth tasks)
(let-syntax ([#,id-start-pos (at-tok-pos #'(if (eq? original-stream stream) (let-syntax ([#,id-start-pos (at-tok-pos #'(if (eq? original-stream stream)
tok-end tok-end
tok-start) tok-start)
@ -372,10 +335,10 @@
#,(loop (cdr pat) (add1 pos))))) #,(loop (cdr pat) (add1 pos)))))
stream last-consumed-token depth stream last-consumed-token depth
#,(let ([cnt (apply + #,(let ([cnt (apply +
(map (lambda (item) (map (λ (item)
(cond (cond
[(bound-identifier-mapping-get nts item (lambda () #f)) [(bound-identifier-mapping-get nts item (λ () #f))
=> (lambda (l) (car l))] => (λ (l) (car l))]
[else 1])) [else 1]))
(cdr pat)))]) (cdr pat)))])
#`(- end #,cnt)) #`(- end #,cnt))
@ -419,37 +382,36 @@
[max-depth max-depth] [max-depth max-depth]
[tasks tasks] [tasks tasks]
[k k]) [k k])
(let ([answer-key (gensym)] (define answer-key (gensym))
[table-key (vector key depth n)] (define table-key (vector key depth n))
[old-depth depth] (define old-depth depth)
[old-stream stream]) (define old-stream stream)
#;(printf "Loop ~a\n" table-key) #;(printf "Loop ~a\n" table-key)
(cond (cond
[(hash-ref (tasks-cache tasks) table-key (lambda () #f)) [(hash-ref (tasks-cache tasks) table-key (λ () #f))
=> (lambda (result) => (λ (result)
#;(printf "Reuse ~a\n" table-key) #;(printf "Reuse ~a\n" table-key)
(result success-k fail-k max-depth tasks))] (result success-k fail-k max-depth tasks))]
[else [else
#;(printf "Try ~a ~a\n" table-key (map tok-name stream)) #;(printf "Try ~a ~a\n" table-key (map tok-name stream))
(hash-set! (tasks-cache tasks) table-key (hash-set! (tasks-cache tasks) table-key
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
#;(printf "Wait ~a ~a\n" table-key answer-key) #;(printf "Wait ~a ~a\n" table-key answer-key)
(wait-for-answer #t max-depth tasks answer-key success-k fail-k (wait-for-answer #t max-depth tasks answer-key success-k fail-k
(lambda (max-depth tasks) (λ (max-depth tasks)
#;(printf "Deadlock ~a ~a\n" table-key answer-key) #;(printf "Deadlock ~a ~a\n" table-key answer-key)
(fail-k max-depth tasks))))) (fail-k max-depth tasks)))))
(let result-loop ([max-depth max-depth][tasks tasks][k k]) (let result-loop ([max-depth max-depth][tasks tasks][k k])
(letrec ([orig-stream stream] (define orig-stream stream)
[new-got-k (define (new-got-k val stream last-consumed-token depth max-depth tasks next-k)
(lambda (val stream last-consumed-token depth max-depth tasks next-k)
;; Check whether we already have a result that consumed the same amount: ;; Check whether we already have a result that consumed the same amount:
(let ([result-key (vector #f key old-depth depth)]) (define result-key (vector #f key old-depth depth))
(cond (cond
[(hash-ref (tasks-cache tasks) result-key (lambda () #f)) [(hash-ref (tasks-cache tasks) result-key (λ () #f))
;; Go for the next-result ;; Go for the next-result
(result-loop max-depth (result-loop max-depth
tasks tasks
(lambda (end max-depth tasks success-k fail-k) (λ (end max-depth tasks success-k fail-k)
(next-k success-k fail-k max-depth tasks)))] (next-k success-k fail-k max-depth tasks)))]
[else [else
#;(printf "Success ~a ~a\n" table-key #;(printf "Success ~a ~a\n" table-key
@ -457,37 +419,36 @@
(if (= d depth) (if (= d depth)
null null
(cons (car s) (loop (add1 d) (cdr s))))))) (cons (car s) (loop (add1 d) (cdr s)))))))
(let ([next-k (lambda (success-k fail-k max-depth tasks) (let ([next-k (λ (success-k fail-k max-depth tasks)
(loop (add1 n) (loop (add1 n)
success-k success-k
fail-k fail-k
max-depth max-depth
tasks tasks
(lambda (end max-depth tasks success-k fail-k) (λ (end max-depth tasks success-k fail-k)
(next-k success-k fail-k max-depth tasks))))]) (next-k success-k fail-k max-depth tasks))))])
(hash-set! (tasks-cache tasks) result-key #t) (hash-set! (tasks-cache tasks) result-key #t)
(hash-set! (tasks-cache tasks) table-key (hash-set! (tasks-cache tasks) table-key
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(success-k val stream last-consumed-token depth max-depth tasks next-k))) (success-k val stream last-consumed-token depth max-depth tasks next-k)))
(report-answer-all answer-key (report-answer-all answer-key
max-depth max-depth
tasks tasks
(list val stream last-consumed-token depth next-k) (list val stream last-consumed-token depth next-k)
(lambda (max-depth tasks) (λ (max-depth tasks)
(success-k val stream last-consumed-token depth max-depth tasks next-k))))])))] (success-k val stream last-consumed-token depth max-depth tasks next-k))))]))
[new-fail-k (define (new-fail-k max-depth tasks)
(lambda (max-depth tasks)
#;(printf "Failure ~a\n" table-key) #;(printf "Failure ~a\n" table-key)
(hash-set! (tasks-cache tasks) table-key (hash-set! (tasks-cache tasks) table-key
(lambda (success-k fail-k max-depth tasks) (λ (success-k fail-k max-depth tasks)
(fail-k max-depth tasks))) (fail-k max-depth tasks)))
(report-answer-all answer-key (report-answer-all answer-key
max-depth max-depth
tasks tasks
null null
(lambda (max-depth tasks) (λ (max-depth tasks)
(fail-k max-depth tasks))))]) (fail-k max-depth tasks))))
(k end max-depth tasks new-got-k new-fail-k)))]))))) (k end max-depth tasks new-got-k new-fail-k))]))))
;; These temp identifiers can't be `gensym` or `generate-temporary` ;; These temp identifiers can't be `gensym` or `generate-temporary`
;; because they have to be consistent between module loads ;; because they have to be consistent between module loads
@ -497,39 +458,34 @@
(define-for-syntax atok-id-temp 'atok_wrutdjgecmybyfipiwsgjlvsveryodlgassuzcargiuznzgdghrykfqfbwcjgzdhdoeqxcucmtjkuyucskzethozhqkasphdwbht) (define-for-syntax atok-id-temp 'atok_wrutdjgecmybyfipiwsgjlvsveryodlgassuzcargiuznzgdghrykfqfbwcjgzdhdoeqxcucmtjkuyucskzethozhqkasphdwbht)
(define-syntax (cfg-parser stx) (define-syntax (cfg-parser stx)
(syntax-case stx () (syntax-case stx ()
[(_ clause ...) [(_ CLAUSE ...)
(let ([clauses (syntax->list #'(clause ...))]) (let ([clauses (syntax->list #'(CLAUSE ...))])
(let-values ([(start grammar cfg-error parser-clauses src-pos?) (let-values ([(start grammar cfg-error parser-clauses src-pos?)
(let ([all-toks (apply (let ([all-toks (apply
append append
(map (lambda (clause) (for/list ([clause (in-list clauses)])
(syntax-case clause (tokens) (syntax-case clause (tokens)
[(tokens t ...) [(tokens T ...)
(apply (apply
append append
(map (lambda (t) (for/list ([t (in-list (syntax->list #'(T ...)))])
(let ([v (syntax-local-value t (lambda () #f))]) (define v (syntax-local-value t (λ () #f)))
(cond (cond
[(terminals-def? v) [(terminals-def? v)
(map (lambda (v) (for/list ([v (in-list (syntax->list (terminals-def-t v)))])
(cons v #f)) (cons v #f))]
(syntax->list (terminals-def-t v)))]
[(e-terminals-def? v) [(e-terminals-def? v)
(map (lambda (v) (for/list ([v (in-list (syntax->list (e-terminals-def-t v)))])
(cons v #t)) (cons v #t))]
(syntax->list (e-terminals-def-t v)))] [else null])))]
[else null]))) [_else null])))]
(syntax->list #'(t ...))))]
[_else null]))
clauses))]
[all-end-toks (apply [all-end-toks (apply
append append
(map (lambda (clause) (for/list ([clause (in-list clauses)])
(syntax-case clause (end) (syntax-case clause (end)
[(end t ...) [(end T ...)
(syntax->list #'(t ...))] (syntax->list #'(T ...))]
[_else null])) [_else null])))])
clauses))])
(let loop ([clauses clauses] (let loop ([clauses clauses]
[cfg-start #f] [cfg-start #f]
[cfg-grammar #f] [cfg-grammar #f]
@ -543,47 +499,35 @@
(reverse parser-clauses) (reverse parser-clauses)
src-pos?) src-pos?)
(syntax-case (car clauses) (start error grammar src-pos) (syntax-case (car clauses) (start error grammar src-pos)
[(start tok) [(start TOK)
(loop (cdr clauses) #'tok cfg-grammar cfg-error src-pos? parser-clauses)] (loop (cdr clauses) #'TOK cfg-grammar cfg-error src-pos? parser-clauses)]
[(error expr) [(error EXPR)
(loop (cdr clauses) cfg-start cfg-grammar #'expr src-pos? parser-clauses)] (loop (cdr clauses) cfg-start cfg-grammar #'EXPR src-pos? parser-clauses)]
[(grammar [nt [pat handle0 handle ...] ...] ...) [(grammar [NT [PAT HANDLE0 HANDLE ...] ...] ...)
(let ([nts (make-bound-identifier-mapping)] (let ([nts (make-bound-identifier-mapping)]
[toks (make-token-identifier-mapping)] [toks (make-token-identifier-mapping)]
[end-toks (make-token-identifier-mapping)] [end-toks (make-token-identifier-mapping)]
[nt-ids (syntax->list #'(nt ...))] [nt-ids (syntax->list #'(NT ...))]
[patss (map (lambda (stx) [patss (map (λ (stx)
(map syntax->list (syntax->list stx))) (map syntax->list (syntax->list stx)))
(syntax->list #'((pat ...) ...)))]) (syntax->list #'((PAT ...) ...)))])
(for-each (lambda (nt) (for ([nt (in-list nt-ids)])
(bound-identifier-mapping-put! nts nt (list 0))) (bound-identifier-mapping-put! nts nt (list 0)))
nt-ids) (for ([t (in-list all-end-toks)])
(for-each (lambda (t)
(token-identifier-mapping-put! end-toks t #t)) (token-identifier-mapping-put! end-toks t #t))
all-end-toks) (for ([t (in-list all-toks)]
(for-each (lambda (t) #:unless (token-identifier-mapping-get end-toks (car t) (λ () #f)))
(unless (token-identifier-mapping-get end-toks (car t) (lambda () #f)) (define id (gensym (syntax-e (car t))))
(let ([id (gensym (syntax-e (car t)))]) (token-identifier-mapping-put! toks (car t) (cons id (cdr t))))
(token-identifier-mapping-put! toks (car t)
(cons id (cdr t))))))
all-toks)
;; Compute min max size for each non-term: ;; Compute min max size for each non-term:
(nt-fixpoint (nt-fixpoint
nts nts
(lambda (nt pats old-list) (λ (nt pats old-list)
(let ([new-cnt (let ([new-cnt
(apply (apply min (for/list ([pat (in-list pats)])
min (for/sum ([elem (in-list pat)])
(map (lambda (pat) (car (bound-identifier-mapping-get
(apply nts elem (λ () (list 1)))))))])
+
(map (lambda (elem)
(car
(bound-identifier-mapping-get nts
elem
(lambda () (list 1)))))
pat)))
pats))])
(if (new-cnt . > . (car old-list)) (if (new-cnt . > . (car old-list))
(cons new-cnt (cdr old-list)) (cons new-cnt (cdr old-list))
old-list))) old-list)))
@ -592,17 +536,17 @@
;; for a non-terminal ;; for a non-terminal
(nt-fixpoint (nt-fixpoint
nts nts
(lambda (nt pats old-list) (λ (nt pats old-list)
(let ([new-list (let ([new-list
(apply (apply
append append
(map (lambda (pat) (for/list ([pat (in-list pats)])
(let loop ([pat pat]) (let loop ([pat pat])
(if (pair? pat) (if (pair? pat)
(let ([l (bound-identifier-mapping-get (let ([l (bound-identifier-mapping-get
nts nts
(car pat) (car pat)
(lambda () (λ ()
(list 1 (map-token toks (car pat)))))]) (list 1 (map-token toks (car pat)))))])
;; If the non-terminal can match 0 things, ;; If the non-terminal can match 0 things,
;; then it might match something from the ;; then it might match something from the
@ -611,10 +555,9 @@
(if (zero? (car l)) (if (zero? (car l))
(append (cdr l) (loop (cdr pat))) (append (cdr l) (loop (cdr pat)))
(cdr l))) (cdr l)))
null))) null))))])
pats))]) (let ([new (filter (λ (id)
(let ([new (filter (lambda (id) (andmap (λ (id2)
(andmap (lambda (id2)
(not (eq? id id2))) (not (eq? id id2)))
(cdr old-list))) (cdr old-list)))
new-list)]) new-list)])
@ -623,7 +566,7 @@
(let ([new (let loop ([new new]) (let ([new (let loop ([new new])
(if (null? (cdr new)) (if (null? (cdr new))
new new
(if (ormap (lambda (id) (if (ormap (λ (id)
(eq? (car new) id)) (eq? (car new) id))
(cdr new)) (cdr new))
(loop (cdr new)) (loop (cdr new))
@ -632,26 +575,26 @@
old-list)))) old-list))))
nt-ids patss) nt-ids patss)
;; Determine left-recursive clauses: ;; Determine left-recursive clauses:
(for-each (lambda (nt pats) (for-each (λ (nt pats)
(let ([l (bound-identifier-mapping-get nts nt)]) (let ([l (bound-identifier-mapping-get nts nt)])
(bound-identifier-mapping-put! nts nt (list (car l) (bound-identifier-mapping-put! nts nt (list (car l)
(cdr l) (cdr l)
(map (lambda (x) #f) pats))))) (map (λ (x) #f) pats)))))
nt-ids patss) nt-ids patss)
(nt-fixpoint (nt-fixpoint
nts nts
(lambda (nt pats old-list) (λ (nt pats old-list)
(list (car old-list) (list (car old-list)
(cadr old-list) (cadr old-list)
(map (lambda (pat simple?) (map (λ (pat simple?)
(or simple? (or simple?
(let ([l (map (lambda (elem) (let ([l (map (λ (elem)
(bound-identifier-mapping-get (bound-identifier-mapping-get
nts nts
elem elem
(lambda () #f))) (λ () #f)))
pat)]) pat)])
(andmap (lambda (i) (andmap (λ (i)
(or (not i) (or (not i)
(andmap values (caddr i)))) (andmap values (caddr i))))
l)))) l))))
@ -660,16 +603,16 @@
;; Build a definition for each non-term: ;; Build a definition for each non-term:
(loop (cdr clauses) (loop (cdr clauses)
cfg-start cfg-start
(map (lambda (nt pats handles $ctxs) (map (λ (nt pats handles $ctxs)
(define info (bound-identifier-mapping-get nts nt)) (define info (bound-identifier-mapping-get nts nt))
(list nt (list nt
#`(let ([key (gensym '#,nt)]) #`(let ([key (gensym '#,nt)])
(lambda (stream last-consumed-token depth end success-k fail-k max-depth tasks) (λ (stream last-consumed-token depth end success-k fail-k max-depth tasks)
(parse-nt/share (parse-nt/share
key #,(car info) '#,(cadr info) stream last-consumed-token depth end key #,(car info) '#,(cadr info) stream last-consumed-token depth end
max-depth tasks max-depth tasks
success-k fail-k success-k fail-k
(lambda (end max-depth tasks success-k fail-k) (λ (end max-depth tasks success-k fail-k)
#,(let loop ([pats pats] #,(let loop ([pats pats]
[handles (syntax->list handles)] [handles (syntax->list handles)]
[$ctxs (syntax->list $ctxs)] [$ctxs (syntax->list $ctxs)]
@ -680,13 +623,13 @@
(car simple?s)) (car simple?s))
#'parse-or #'parse-or
#'parse-parallel-or) #'parse-parallel-or)
(lambda (stream last-consumed-token depth end success-k fail-k max-depth tasks) (λ (stream last-consumed-token depth end success-k fail-k max-depth tasks)
#,(build-match nts #,(build-match nts
toks toks
(car pats) (car pats)
(car handles) (car handles)
(car $ctxs))) (car $ctxs)))
(lambda (stream last-consumed-token depth end success-k fail-k max-depth tasks) (λ (stream last-consumed-token depth end success-k fail-k max-depth tasks)
#,(loop (cdr pats) #,(loop (cdr pats)
(cdr handles) (cdr handles)
(cdr $ctxs) (cdr $ctxs)
@ -694,14 +637,14 @@
stream last-consumed-token depth end success-k fail-k max-depth tasks))))))))) stream last-consumed-token depth end success-k fail-k max-depth tasks)))))))))
nt-ids nt-ids
patss patss
(syntax->list #'(((begin handle0 handle ...) ...) ...)) (syntax->list #'(((begin HANDLE0 HANDLE ...) ...) ...))
(syntax->list #'((handle0 ...) ...))) (syntax->list #'((HANDLE0 ...) ...)))
cfg-error cfg-error
src-pos? src-pos?
(list* (list*
(with-syntax ([((tok tok-id . $e) ...) (with-syntax ([((tok tok-id . $e) ...)
(token-identifier-mapping-map toks (token-identifier-mapping-map toks
(lambda (k v) (λ (k v)
(list* k (list* k
(car v) (car v)
(if (cdr v) (if (cdr v)
@ -743,19 +686,19 @@
src-pos? src-pos?
(cons (car clauses) parser-clauses))]))))]) (cons (car clauses) parser-clauses))]))))])
#`(let ([orig-parse (parser #`(let ([orig-parse (parser
[error (lambda (a b c) [error (λ (a b c)
(error 'cfg-parser "unexpected ~a token: ~a" b c))] (error 'cfg-parser "unexpected ~a token: ~a" b c))]
. #,parser-clauses)] . #,parser-clauses)]
[error-proc #,cfg-error]) [error-proc #,cfg-error])
(letrec #,grammar (letrec #,grammar
(lambda (get-tok) (λ (get-tok)
(let ([tok-list (orig-parse get-tok)]) (let ([tok-list (orig-parse get-tok)])
(letrec ([success-k (letrec ([success-k
(lambda (val stream last-consumed-token depth max-depth tasks next) (λ (val stream last-consumed-token depth max-depth tasks next)
(if (null? stream) (if (null? stream)
val val
(next success-k fail-k max-depth tasks)))] (next success-k fail-k max-depth tasks)))]
[fail-k (lambda (max-depth tasks) [fail-k (λ (max-depth tasks)
(cond (cond
[(null? tok-list) [(null? tok-list)
(if error-proc (if error-proc
@ -847,7 +790,7 @@
(define (parse s) (define (parse s)
(define ip (open-input-string s)) (define ip (open-input-string s))
(port-count-lines! ip) (port-count-lines! ip)
(-parse (lambda () (lex ip)))) (-parse (λ () (lex ip))))
(check-equal? (parse "abc") (check-equal? (parse "abc")
'(unanchored (lit "abc" 1 4) 1 4)) '(unanchored (lit "abc" 1 4) 1 4))
@ -881,7 +824,7 @@
(tokens non-terminals) (tokens non-terminals)
(start <program>) (start <program>)
(end EOF) (end EOF)
(error (lambda (a b stx) (error (λ (a b stx)
(error 'parse "failed at ~s" stx))) (error 'parse "failed at ~s" stx)))
(grammar [<program> [(PLUS) "plus"] (grammar [<program> [(PLUS) "plus"]
[(<minus-program> BAR <minus-program>) (list $1 $2 $3)] [(<minus-program> BAR <minus-program>) (list $1 $2 $3)]
@ -903,7 +846,7 @@
-|-*|-|-**|-|-*|-|-***|-|-*|-|-**|-|-*|-|-*****" -|-*|-|-**|-|-*|-|-***|-|-*|-|-**|-|-*|-|-*****"
;; This one fails: ;; This one fails:
#;"+*")]) #;"+*")])
(check-equal? (parse (lambda () (lex p))) (check-equal? (parse (λ () (lex p)))
'((((((((((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *) || (((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *)) . *) '((((((((((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *) || (((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *)) . *)
|| ||
(((((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *) || (((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *)) . *)) (((((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *) || (((("minus" || "minus") . *) || (("minus" || "minus") . *)) . *)) . *))

25
br-parser-tools-lib/br-parser-tools/examples/calc.rkt
Unescape Escape View File

@ -1,4 +1,4 @@
#lang scheme #lang racket/base
;; An interactive calculator inspired by the calculator example in the bison manual. ;; An interactive calculator inspired by the calculator example in the bison manual.
@ -22,12 +22,12 @@
;; (:/ 0 9) would not work because the lexer does not understand numbers. (:/ #\0 #\9) is ok too. ;; (:/ 0 9) would not work because the lexer does not understand numbers. (:/ #\0 #\9) is ok too.
(digit (:/ "0" "9"))) (digit (:/ "0" "9")))
(define calcl (define calc-lex
(lexer (lexer
[(eof) 'EOF] [(eof) 'EOF]
;; recursively call the lexer on the remaining input after a tab or space. Returning the ;; recursively call the lexer on the remaining input after a tab or space. Returning the
;; result of that operation. This effectively skips all whitespace. ;; result of that operation. This effectively skips all whitespace.
[(:or #\tab #\space) (calcl input-port)] [(:or #\tab #\space) (calc-lex input-port)]
;; (token-newline) returns 'newline ;; (token-newline) returns 'newline
[#\newline (token-newline)] [#\newline (token-newline)]
;; Since (token-=) returns '=, just return the symbol directly ;; Since (token-=) returns '=, just return the symbol directly
@ -40,7 +40,7 @@
[(:: (:+ digit) #\. (:* digit)) (token-NUM (string->number lexeme))])) [(:: (:+ digit) #\. (:* digit)) (token-NUM (string->number lexeme))]))
(define calcp (define calc-parse
(parser (parser
(start start) (start start)
@ -78,12 +78,15 @@
;; run the calculator on the given input-port ;; run the calculator on the given input-port
(define (calc ip) (define (calc ip)
(port-count-lines! ip) (port-count-lines! ip)
(letrec ((one-line (let loop ()
(lambda () (define result (calc-parse (λ () (calc-lex ip))))
(let ((result (calcp (lambda () (calcl ip)))))
(when result (when result
(printf "~a\n" result) (printf "~a\n" result)
(one-line)))))) (loop))))
(one-line)))
(module+ test
(calc (open-input-string "x=1\n(x + 2 * 3) - (1+2)*3")) (require rackunit)
(check-equal? (let ([o (open-output-string)])
(parameterize ([current-output-port o])
(calc (open-input-string "x=1\n(x + 2 * 3) - (1+2)*3")))
(get-output-string o)) "1\n-2\n"))

52
br-parser-tools-lib/br-parser-tools/examples/read.rkt
Unescape Escape View File

@ -1,18 +1,18 @@
#lang racket/base
;; This implements the equivalent of racket's read-syntax for R5RS scheme. ;; This implements the equivalent of racket's read-syntax for R5RS scheme.
;; It has not been thoroughly tested. Also it will read an entire file into a ;; It has not been thoroughly tested. Also it will read an entire file into a
;; list of syntax objects, instead of returning one syntax object at a time ;; list of syntax objects, instead of returning one syntax object at a time
(module read mzscheme (require (for-syntax racket/base)
br-parser-tools/lex
(require br-parser-tools/lex (prefix-in : br-parser-tools/lex-sre)
(prefix : br-parser-tools/lex-sre)
br-parser-tools/yacc br-parser-tools/yacc
syntax/readerr) syntax/readerr)
(define-tokens data (DATUM)) (define-tokens data (DATUM))
(define-empty-tokens delim (OP CP HASHOP QUOTE QUASIQUOTE UNQUOTE UNQUOTE-SPLICING DOT EOF)) (define-empty-tokens delim (OP CP HASHOP QUOTE QUASIQUOTE UNQUOTE UNQUOTE-SPLICING DOT EOF))
(define scheme-lexer (define scheme-lexer
(lexer-src-pos (lexer-src-pos
;; Skip comments, without accumulating extra position information ;; Skip comments, without accumulating extra position information
@ -41,7 +41,7 @@
["." 'DOT] ["." 'DOT]
[(eof) 'EOF])) [(eof) 'EOF]))
(define get-string-token (define get-string-token
(lexer (lexer
[(:~ #\" #\\) (cons (car (string->list lexeme)) [(:~ #\" #\\) (cons (car (string->list lexeme))
(get-string-token input-port))] (get-string-token input-port))]
@ -50,7 +50,7 @@
[#\" null])) [#\" null]))
(define-lex-abbrevs (define-lex-abbrevs
[letter (:or (:/ "a" "z") (:/ #\A #\Z))] [letter (:or (:/ "a" "z") (:/ #\A #\Z))]
[digit (:/ #\0 #\9)] [digit (:/ #\0 #\9)]
[scheme-whitespace (:or #\newline #\return #\tab #\space #\vtab)] [scheme-whitespace (:or #\newline #\return #\tab #\space #\vtab)]
@ -164,27 +164,27 @@
[exactness (:or "" "#i" "#e")]) [exactness (:or "" "#i" "#e")])
(define stx-for-original-property (read-syntax #f (open-input-string "original"))) (define stx-for-original-property (read-syntax #f (open-input-string "original")))
;; A macro to build the syntax object ;; A macro to build the syntax object
(define-syntax (build-so stx) (define-syntax (build-so stx)
(syntax-case stx () (syntax-case stx ()
((_ value start end) ((_ value start end)
(with-syntax ((start-pos (datum->syntax-object (with-syntax ((start-pos (datum->syntax
(syntax end) #'end
(string->symbol (string->symbol
(format "$~a-start-pos" (format "$~a-start-pos"
(syntax-object->datum (syntax start)))))) (syntax->datum #'start)))))
(end-pos (datum->syntax-object (end-pos (datum->syntax
(syntax end) #'end
(string->symbol (string->symbol
(format "$~a-end-pos" (format "$~a-end-pos"
(syntax-object->datum (syntax end)))))) (syntax->datum #'end)))))
(source (datum->syntax-object (source (datum->syntax
(syntax end) #'end
'source-name))) 'source-name)))
(syntax (syntax
(datum->syntax-object (datum->syntax
#f #f
value value
(list source (list source
@ -195,7 +195,7 @@
(position-offset start-pos))) (position-offset start-pos)))
stx-for-original-property)))))) stx-for-original-property))))))
(define (scheme-parser source-name) (define (scheme-parser source-name)
(parser (parser
(src-pos) (src-pos)
@ -229,14 +229,12 @@
(sexp-list [() null] (sexp-list [() null]
[(sexp-list sexp) (cons $2 $1)])))) [(sexp-list sexp) (cons $2 $1)]))))
(define (rs sn ip) (define (rs sn ip)
(port-count-lines! ip) (port-count-lines! ip)
((scheme-parser sn) (lambda () (scheme-lexer ip)))) ((scheme-parser sn) (lambda () (scheme-lexer ip))))
(define readsyntax (define readsyntax
(case-lambda ((sn) (rs sn (current-input-port))) (case-lambda ((sn) (rs sn (current-input-port)))
((sn ip) (rs sn ip)))) ((sn ip) (rs sn ip))))
(provide (rename readsyntax read-syntax)) (provide (rename-out [readsyntax read-syntax]))
)

39
br-parser-tools-lib/br-parser-tools/lex-plt-v200.rkt
Unescape Escape View File

@ -1,24 +1,23 @@
(module lex-plt-v200 mzscheme #lang racket/base
(require br-parser-tools/lex (require (for-syntax racket/base)
(prefix : br-parser-tools/lex-sre)) br-parser-tools/lex
(prefix-in : br-parser-tools/lex-sre))
(provide epsilon (provide epsilon ~
~ (rename-out [:* *]
(rename :* *) [:+ +]
(rename :+ +) [:? ?]
(rename :? ?) [:or :]
(rename :or :) [:& &]
(rename :& &) [:: @]
(rename :: @) [:~ ^]
(rename :~ ^) [:/ -]))
(rename :/ -))
(define-lex-trans epsilon (define-lex-trans (epsilon stx)
(syntax-rules () (syntax-case stx ()
((_) ""))) [(_) #'""]))
(define-lex-trans ~
(syntax-rules ()
((_ re) (complement re)))))
(define-lex-trans (~ stx)
(syntax-case stx ()
[(_ RE) #'(complement RE)]))

204
br-parser-tools-lib/br-parser-tools/lex-sre.rkt
Unescape Escape View File

@ -1,119 +1,103 @@
(module lex-sre mzscheme #lang racket/base
(require br-parser-tools/lex) (require (for-syntax racket/base)
br-parser-tools/lex)
(provide (rename sre-* *)
(rename sre-+ +) (provide (rename-out [sre-* *]
? [sre-+ +]
(rename sre-= =) [sre-= =]
(rename sre->= >=) [sre->= >=]
** [sre-or or]
(rename sre-or or) [sre-- -]
: [sre-/ /])
seq ? ** : seq & ~ /-only-chars)
&
~ (define-lex-trans (sre-* stx)
(rename sre-- -)
(rename sre-/ /)
/-only-chars)
(define-lex-trans sre-*
(syntax-rules ()
((_ re ...)
(repetition 0 +inf.0 (union re ...)))))
(define-lex-trans sre-+
(syntax-rules ()
((_ re ...)
(repetition 1 +inf.0 (union re ...)))))
(define-lex-trans ?
(syntax-rules ()
((_ re ...)
(repetition 0 1 (union re ...)))))
(define-lex-trans sre-=
(syntax-rules ()
((_ n re ...)
(repetition n n (union re ...)))))
(define-lex-trans sre->=
(syntax-rules ()
((_ n re ...)
(repetition n +inf.0 (union re ...)))))
(define-lex-trans **
(syntax-rules ()
((_ low #f re ...)
(** low +inf.0 re ...))
((_ low high re ...)
(repetition low high (union re ...)))))
(define-lex-trans sre-or
(syntax-rules ()
((_ re ...)
(union re ...))))
(define-lex-trans :
(syntax-rules ()
((_ re ...)
(concatenation re ...))))
(define-lex-trans seq
(syntax-rules ()
((_ re ...)
(concatenation re ...))))
(define-lex-trans &
(syntax-rules ()
((_ re ...)
(intersection re ...))))
(define-lex-trans ~
(syntax-rules ()
((_ re ...)
(char-complement (union re ...)))))
;; set difference
(define-lex-trans (sre-- stx)
(syntax-case stx () (syntax-case stx ()
((_) [(_ RE ...)
#'(repetition 0 +inf.0 (union RE ...))]))
(define-lex-trans (sre-+ stx)
(syntax-case stx ()
[(_ RE ...)
#'(repetition 1 +inf.0 (union RE ...))]))
(define-lex-trans (? stx)
(syntax-case stx ()
[(_ RE ...)
#'(repetition 0 1 (union RE ...))]))
(define-lex-trans (sre-= stx)
(syntax-case stx ()
[(_ N RE ...)
#'(repetition N N (union RE ...))]))
(define-lex-trans (sre->= stx)
(syntax-case stx ()
[(_ N RE ...)
#'(repetition N +inf.0 (union RE ...))]))
(define-lex-trans (** stx)
(syntax-case stx ()
[(_ LOW #f RE ...)
#'(** LOW +inf.0 RE ...)]
[(_ LOW HIGH RE ...)
#'(repetition LOW HIGH (union RE ...))]))
(define-lex-trans (sre-or stx)
(syntax-case stx ()
[(_ RE ...)
#'(union RE ...)]))
(define-lex-trans (: stx)
(syntax-case stx ()
[(_ RE ...)
#'(concatenation RE ...)]))
(define-lex-trans (seq stx)
(syntax-case stx ()
[(_ RE ...)
#'(concatenation RE ...)]))
(define-lex-trans (& stx)
(syntax-case stx ()
[(_ RE ...)
#'(intersection RE ...)]))
(define-lex-trans (~ stx)
(syntax-case stx ()
[(_ RE ...)
#'(char-complement (union RE ...))]))
;; set difference
(define-lex-trans (sre-- stx)
(syntax-case stx ()
[(_)
(raise-syntax-error #f (raise-syntax-error #f
"must have at least one argument" "must have at least one argument"
stx)) stx)]
((_ big-re re ...) [(_ BIG-RE RE ...)
(syntax (& big-re (complement (union re ...))))))) #'(& BIG-RE (complement (union RE ...)))]))
(define-lex-trans (sre-/ stx) (define-lex-trans (sre-/ stx)
(syntax-case stx () (syntax-case stx ()
((_ range ...) [(_ RANGE ...)
(let ((chars (let ([chars
(apply append (map (lambda (r) (apply append (for/list ([r (in-list (syntax->list #'(RANGE ...)))])
(let ((x (syntax-e r))) (let ([x (syntax-e r)])
(cond (cond
((char? x) (list x)) [(char? x) (list x)]
((string? x) (string->list x)) [(string? x) (string->list x)]
(else [else
(raise-syntax-error (raise-syntax-error #f "not a char or string" stx r)]))))])
#f
"not a char or string"
stx
r)))))
(syntax->list (syntax (range ...)))))))
(unless (even? (length chars)) (unless (even? (length chars))
(raise-syntax-error (raise-syntax-error #f "not given an even number of characters" stx))
#f #`(/-only-chars #,@chars))]))
"not given an even number of characters"
stx)) (define-lex-trans (/-only-chars stx)
#`(/-only-chars #,@chars))))) (syntax-case stx ()
[(_ C1 C2)
(define-lex-trans /-only-chars #'(char-range C1 C2)]
(syntax-rules () [(_ C1 C2 C ...)
((_ c1 c2) #'(union (char-range C1 C2) (/-only-chars C ...))]))
(char-range c1 c2))
((_ c1 c2 c ...)
(union (char-range c1 c2)
(/-only-chars c ...)))))
)

60
br-parser-tools-lib/br-parser-tools/lex.rkt
Unescape Escape View File

@ -3,7 +3,7 @@
;; Provides the syntax used to create lexers and the functions needed to ;; Provides the syntax used to create lexers and the functions needed to
;; create and use the buffer that the lexer reads from. See docs. ;; create and use the buffer that the lexer reads from. See docs.
(require (for-syntax mzlib/list (require (for-syntax racket/list
syntax/stx syntax/stx
syntax/define syntax/define
syntax/boundmap syntax/boundmap
@ -14,7 +14,7 @@
racket/base racket/base
racket/promise)) racket/promise))
(require mzlib/stxparam (require racket/stxparam
syntax/readerr syntax/readerr
"private-lex/token.rkt") "private-lex/token.rkt")
@ -30,7 +30,7 @@
file-path file-path
lexer-file-path ;; alternate name lexer-file-path ;; alternate name
;; Lex abbrevs for unicode char sets. See mzscheme manual section 3.4. ;; Lex abbrevs for unicode char sets.
any-char any-string nothing alphabetic lower-case upper-case title-case any-char any-string nothing alphabetic lower-case upper-case title-case
numeric symbolic punctuation graphic whitespace blank iso-control numeric symbolic punctuation graphic whitespace blank iso-control
@ -212,12 +212,11 @@
(define (get-next-state char table) (define (get-next-state char table)
(and table (get-next-state-helper char 0 (vector-length table) table))) (and table (get-next-state-helper char 0 (vector-length table) table)))
(define (lexer-body start-state trans-table actions no-lookahead special-action (define ((lexer-body start-state trans-table actions no-lookahead special-action
has-special-comment-action? special-comment-action eof-action) has-special-comment-action? special-comment-action eof-action) ip)
(letrec ([lexer (define (lexer ip)
(λ (ip) (define first-pos (get-position ip))
(let ((first-pos (get-position ip)) (define first-char (peek-char-or-special ip 0))
(first-char (peek-char-or-special ip 0)))
;(printf "(peek-char-or-special port 0) = ~e\n" first-char) ;(printf "(peek-char-or-special port 0) = ~e\n" first-char)
(cond (cond
[(eof-object? first-char) [(eof-object? first-char)
@ -233,40 +232,40 @@
[else [else
(let lexer-loop ( (let lexer-loop (
;; current-state ;; current-state
(state start-state) [state start-state]
;; the character to transition on ;; the character to transition on
(char first-char) [char first-char]
;; action for the longest match seen thus far ;; action for the longest match seen thus far
;; including a match at the current state ;; including a match at the current state
(longest-match-action [longest-match-action
(vector-ref actions start-state)) (vector-ref actions start-state)]
;; how many bytes precede char ;; how many bytes precede char
(length-bytes 0) [length-bytes 0]
;; how many characters have been read ;; how many characters have been read
;; including the one just read ;; including the one just read
(length-chars 1) [length-chars 1]
;; how many characters are in the longest match ;; how many characters are in the longest match
(longest-match-length 0)) [longest-match-length 0])
(let ([next-state (define next-state
(cond (cond
[(not (char? char)) #f] [(not (char? char)) #f]
[else (get-next-state (char->integer char) [else (get-next-state (char->integer char)
(vector-ref trans-table state))])]) (vector-ref trans-table state))]))
(cond (cond
[(not next-state) [(not next-state)
(check-match ip first-pos longest-match-length (check-match ip first-pos longest-match-length
length-chars longest-match-action)] length-chars longest-match-action)]
[(vector-ref no-lookahead next-state) [(vector-ref no-lookahead next-state)
(let ((act (vector-ref actions next-state))) (define act (vector-ref actions next-state))
(check-match ip (check-match ip
first-pos first-pos
(if act length-chars longest-match-length) (if act length-chars longest-match-length)
length-chars length-chars
(if act act longest-match-action)))] (if act act longest-match-action))]
[else [else
(let* ([act (vector-ref actions next-state)] (define act (vector-ref actions next-state))
[next-length-bytes (+ (char-utf-8-length char) length-bytes)] (define next-length-bytes (+ (char-utf-8-length char) length-bytes))
[next-char (peek-char-or-special ip next-length-bytes)]) (define next-char (peek-char-or-special ip next-length-bytes))
#;(printf "(peek-char-or-special port ~e) = ~e\n" #;(printf "(peek-char-or-special port ~e) = ~e\n"
next-length-bytes next-char) next-length-bytes next-char)
(lexer-loop next-state (lexer-loop next-state
@ -278,26 +277,25 @@
(add1 length-chars) (add1 length-chars)
(if act (if act
length-chars length-chars
longest-match-length)))])))])))]) longest-match-length))]))]))
(λ (ip)
(unless (input-port? ip) (unless (input-port? ip)
(raise-argument-error 'lexer "input-port?" 0 ip)) (raise-argument-error 'lexer "input-port?" 0 ip))
(lexer ip)))) (lexer ip))
(define (check-match lb first-pos longest-match-length length longest-match-action) (define (check-match lb first-pos longest-match-length length longest-match-action)
(unless longest-match-action (unless longest-match-action
(let* ([match (read-string length lb)] (define match (read-string length lb))
[end-pos (get-position lb)]) (define end-pos (get-position lb))
(raise-read-error (raise-read-error
(format "lexer: No match found in input starting with: ~a" match) (format "lexer: No match found in input starting with: ~a" match)
(file-path) (file-path)
(position-line first-pos) (position-line first-pos)
(position-col first-pos) (position-col first-pos)
(position-offset first-pos) (position-offset first-pos)
(- (position-offset end-pos) (position-offset first-pos))))) (- (position-offset end-pos) (position-offset first-pos))))
(let ([match (read-string longest-match-length lb)]) (define match (read-string longest-match-length lb))
;(printf "(read-string ~e port) = ~e\n" longest-match-length match) ;(printf "(read-string ~e port) = ~e\n" longest-match-length match)
(do-match lb first-pos longest-match-action match))) (do-match lb first-pos longest-match-action match))
(define file-path (make-parameter #f)) (define file-path (make-parameter #f))
(define lexer-file-path file-path) (define lexer-file-path file-path)

9
br-parser-tools-lib/br-parser-tools/private-lex/actions.rkt
Unescape Escape View File

@ -1,5 +1,4 @@
#lang racket/base #lang racket/base
(provide (all-defined-out)) (provide (all-defined-out))
(require syntax/stx) (require syntax/stx)
@ -7,10 +6,10 @@
;; Returns the first action from a rule of the form ((which-special) action) ;; Returns the first action from a rule of the form ((which-special) action)
(define (get-special-action rules which-special none) (define (get-special-action rules which-special none)
(cond (cond
((null? rules) none) [(null? rules) none]
(else [else
(syntax-case (car rules) () (syntax-case (car rules) ()
[((special) ACT) [((special) ACT)
(and (identifier? #'special) (module-or-top-identifier=? (syntax special) which-special)) (and (identifier? #'special) (module-or-top-identifier=? #'special which-special))
#'ACT] #'ACT]
[_ (get-special-action (cdr rules) which-special none)])))) [_ (get-special-action (cdr rules) which-special none)])]))

286
br-parser-tools-lib/br-parser-tools/private-lex/deriv.rkt
Unescape Escape View File

@ -1,43 +1,42 @@
(module deriv mzscheme #lang racket/base
(require racket/list
(require mzlib/list (prefix-in is: data/integer-set)
(prefix is: mzlib/integer-set)
"re.rkt" "re.rkt"
"util.rkt") "util.rkt")
(provide build-dfa print-dfa (struct dfa (num-states start-state final-states/actions transitions))) (provide build-dfa print-dfa (struct-out dfa))
(define e (build-epsilon)) (define e (build-epsilon))
(define z (build-zero)) (define z (build-zero))
;; Don't do anything with this one but extract the chars ;; Don't do anything with this one but extract the chars
(define all-chars (->re `(char-complement (union)) (make-cache))) (define all-chars (->re `(char-complement (union)) (make-cache)))
;; get-char-groups : re bool -> (list-of char-setR?) ;; get-char-groups : re bool -> (list-of char-setR?)
;; Collects the char-setRs in r that could be used in ;; Collects the char-setRs in r that could be used in
;; taking the derivative of r. ;; taking the derivative of r.
(define (get-char-groups r found-negation) (define (get-char-groups r found-negation)
(cond (cond
((or (eq? r e) (eq? r z)) null) [(or (eq? r e) (eq? r z)) null]
((char-setR? r) (list r)) [(char-setR? r) (list r)]
((concatR? r) [(concatR? r)
(if (re-nullable? (concatR-re1 r)) (if (re-nullable? (concatR-re1 r))
(append (get-char-groups (concatR-re1 r) found-negation) (append (get-char-groups (concatR-re1 r) found-negation)
(get-char-groups (concatR-re2 r) found-negation)) (get-char-groups (concatR-re2 r) found-negation))
(get-char-groups (concatR-re1 r) found-negation))) (get-char-groups (concatR-re1 r) found-negation))]
((repeatR? r) [(repeatR? r)
(get-char-groups (repeatR-re r) found-negation)) (get-char-groups (repeatR-re r) found-negation)]
((orR? r) [(orR? r)
(apply append (map (lambda (x) (get-char-groups x found-negation)) (orR-res r)))) (apply append (map (λ (x) (get-char-groups x found-negation)) (orR-res r)))]
((andR? r) [(andR? r)
(apply append (map (lambda (x) (get-char-groups x found-negation)) (andR-res r)))) (apply append (map (λ (x) (get-char-groups x found-negation)) (andR-res r)))]
((negR? r) [(negR? r)
(if found-negation (if found-negation
(get-char-groups (negR-re r) #t) (get-char-groups (negR-re r) #t)
(cons all-chars (get-char-groups (negR-re r) #t)))))) (cons all-chars (get-char-groups (negR-re r) #t)))]))
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(r1 (->re #\1 c)) (r1 (->re #\1 c))
(r2 (->re #\2 c))) (r2 (->re #\2 c)))
((get-char-groups e #f) null) ((get-char-groups e #f) null)
@ -62,39 +61,39 @@
(concatenation (repetition 0 +inf.0 ,r2) "3") "4") c) #f) (concatenation (repetition 0 +inf.0 ,r2) "3") "4") c) #f)
(list r1 r2 (->re "3" c) (->re "4" c))) (list r1 r2 (->re "3" c) (->re "4" c)))
) )
(define loc:member? is:member?) (define loc:member? is:member?)
;; deriveR : re char cache -> re ;; deriveR : re char cache -> re
(define (deriveR r c cache) (define (deriveR r c cache)
(cond (cond
((or (eq? r e) (eq? r z)) z) [(or (eq? r e) (eq? r z)) z]
((char-setR? r) [(char-setR? r)
(if (loc:member? c (char-setR-chars r)) e z)) (if (loc:member? c (char-setR-chars r)) e z)]
((concatR? r) [(concatR? r)
(let* ((r1 (concatR-re1 r)) (define r1 (concatR-re1 r))
(r2 (concatR-re2 r)) (define r2 (concatR-re2 r))
(d (build-concat (deriveR r1 c cache) r2 cache))) (define d (build-concat (deriveR r1 c cache) r2 cache))
(if (re-nullable? r1) (if (re-nullable? r1)
(build-or (list d (deriveR r2 c cache)) cache) (build-or (list d (deriveR r2 c cache)) cache)
d))) d)]
((repeatR? r) [(repeatR? r)
(build-concat (deriveR (repeatR-re r) c cache) (build-concat (deriveR (repeatR-re r) c cache)
(build-repeat (sub1 (repeatR-low r)) (build-repeat (sub1 (repeatR-low r))
(sub1 (repeatR-high r)) (sub1 (repeatR-high r))
(repeatR-re r) cache) (repeatR-re r) cache)
cache)) cache)]
((orR? r) [(orR? r)
(build-or (map (lambda (x) (deriveR x c cache)) (build-or (map (λ (x) (deriveR x c cache))
(orR-res r)) (orR-res r))
cache)) cache)]
((andR? r) [(andR? r)
(build-and (map (lambda (x) (deriveR x c cache)) (build-and (map (λ (x) (deriveR x c cache))
(andR-res r)) (andR-res r))
cache)) cache)]
((negR? r) [(negR? r)
(build-neg (deriveR (negR-re r) c cache) cache)))) (build-neg (deriveR (negR-re r) c cache) cache)]))
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(a (char->integer #\a)) (a (char->integer #\a))
(b (char->integer #\b)) (b (char->integer #\b))
(r1 (->re #\a c)) (r1 (->re #\a c))
@ -129,21 +128,19 @@
((deriveR (->re `(repetition 1 2 "ab") c) a c) ((deriveR (->re `(repetition 1 2 "ab") c) a c)
(->re `(concatenation "b" (repetition 0 1 "ab")) c))) (->re `(concatenation "b" (repetition 0 1 "ab")) c)))
;; An re-action is (cons re action) ;; An re-action is (cons re action)
;; derive : (list-of re-action) char cache -> (union (list-of re-action) #f) ;; derive : (list-of re-action) char cache -> (union (list-of re-action) #f)
;; applies deriveR to all the re-actions's re parts. ;; applies deriveR to all the re-actions's re parts.
;; Returns #f if the derived state is equivalent to z. ;; Returns #f if the derived state is equivalent to z.
(define (derive r c cache) (define (derive r c cache)
(let ((new-r (map (lambda (ra) (define new-r (for/list ([ra (in-list r)])
(cons (deriveR (car ra) c cache) (cdr ra))) (cons (deriveR (car ra) c cache) (cdr ra))))
r))) (if (andmap (λ (x) (eq? z (car x))) new-r)
(if (andmap (lambda (x) (eq? z (car x)))
new-r)
#f #f
new-r))) new-r))
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(r1 (->re #\1 c)) (r1 (->re #\1 c))
(r2 (->re #\2 c))) (r2 (->re #\2 c)))
((derive null (char->integer #\1) c) #f) ((derive null (char->integer #\1) c) #f)
@ -152,16 +149,16 @@
((derive (list (cons r1 1) (cons r2 2)) (char->integer #\3) c) #f)) ((derive (list (cons r1 1) (cons r2 2)) (char->integer #\3) c) #f))
;; get-final : (list-of re-action) -> (union #f syntax-object) ;; get-final : (list-of re-action) -> (union #f syntax-object)
;; An re that accepts e represents a final state. Return the ;; An re that accepts e represents a final state. Return the
;; action from the first final state or #f if there is none. ;; action from the first final state or #f if there is none.
(define (get-final res) (define (get-final res)
(cond (cond
((null? res) #f) [(null? res) #f]
((re-nullable? (caar res)) (cdar res)) [(re-nullable? (caar res)) (cdar res)]
(else (get-final (cdr res))))) [else (get-final (cdr res))]))
(test-block ((c->i char->integer) (test-block ((c->i char->integer)
(c (make-cache)) (c (make-cache))
(r1 (->re #\a c)) (r1 (->re #\a c))
(r2 (->re #\b c)) (r2 (->re #\b c))
@ -183,28 +180,28 @@
((get-final b) 4)) ((get-final b) 4))
;; A state is (make-state (list-of re-action) nat) ;; A state is (make-state (list-of re-action) nat)
(define-struct state (spec index)) (define-struct state (spec index))
;; get->key : re-action -> (list-of nat) ;; get->key : re-action -> (list-of nat)
;; states are indexed by the list of indexes of their res ;; states are indexed by the list of indexes of their res
(define (get-key s) (define (get-key s)
(map (lambda (x) (re-index (car x))) s)) (map (λ (x) (re-index (car x))) s))
(define loc:partition is:partition) (define loc:partition is:partition)
;; compute-chars : (list-of state) -> (list-of char-set) ;; compute-chars : (list-of state) -> (list-of char-set)
;; Computed the sets of equivalent characters for taking the ;; Computed the sets of equivalent characters for taking the
;; derivative of the car of st. Only one derivative per set need to be taken. ;; derivative of the car of st. Only one derivative per set need to be taken.
(define (compute-chars st) (define (compute-chars st)
(cond (cond
((null? st) null) [(null? st) null]
(else [else
(loc:partition (map char-setR-chars (loc:partition (map char-setR-chars
(apply append (map (lambda (x) (get-char-groups (car x) #f)) (apply append (map (λ (x) (get-char-groups (car x) #f))
(state-spec (car st))))))))) (state-spec (car st))))))]))
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(c->i char->integer) (c->i char->integer)
(r1 (->re `(char-range #\1 #\4) c)) (r1 (->re `(char-range #\1 #\4) c))
(r2 (->re `(char-range #\2 #\3) c))) (r2 (->re `(char-range #\2 #\3) c)))
@ -217,87 +214,84 @@
(is:make-range (c->i #\4))))))) (is:make-range (c->i #\4)))))))
;; A dfa is (make-dfa int int ;; A dfa is (make-dfa int int
;; (list-of (cons int syntax-object)) ;; (list-of (cons int syntax-object))
;; (list-of (cons int (list-of (cons char-set int))))) ;; (list-of (cons int (list-of (cons char-set int)))))
;; Each transitions is a state and a list of chars with the state to transition to. ;; Each transitions is a state and a list of chars with the state to transition to.
;; The finals and transitions are sorted by state number, and duplicate free. ;; The finals and transitions are sorted by state number, and duplicate free.
(define-struct dfa (num-states start-state final-states/actions transitions) (make-inspector)) (define-struct dfa (num-states start-state final-states/actions transitions) #:inspector (make-inspector))
(define loc:get-integer is:get-integer) (define loc:get-integer is:get-integer)
;; build-dfa : (list-of re-action) cache -> dfa ;; build-dfa : (list-of re-action) cache -> dfa
(define (build-dfa rs cache) (define (build-dfa rs cache)
(let* ((transitions (make-hash-table)) (let* ([transitions (make-hash)]
(get-state-number (make-counter)) [get-state-number (make-counter)]
(start (make-state rs (get-state-number)))) [start (make-state rs (get-state-number))])
(cache (cons 'state (get-key rs)) (lambda () start)) (cache (cons 'state (get-key rs)) (λ () start))
(let loop ((old-states (list start)) (let loop ([old-states (list start)]
(new-states null) [new-states null]
(all-states (list start)) [all-states (list start)]
(cs (compute-chars (list start)))) [cs (compute-chars (list start))])
(cond (cond
((and (null? old-states) (null? new-states)) [(and (null? old-states) (null? new-states))
(make-dfa (get-state-number) (state-index start) (make-dfa (get-state-number) (state-index start)
(sort (filter (lambda (x) (cdr x)) (sort (for*/list ([state (in-list all-states)]
(map (lambda (state) [val (in-value (cons (state-index state) (get-final (state-spec state))))]
(cons (state-index state) (get-final (state-spec state)))) #:when (cdr val))
all-states)) val)
(lambda (a b) (< (car a) (car b)))) < #:key car)
(sort (hash-table-map transitions (sort (hash-map transitions
(lambda (state trans) (λ (state trans)
(cons (state-index state) (cons (state-index state)
(map (lambda (t) (for/list ([t (in-list trans)])
(cons (car t) (cons (car t)
(state-index (cdr t)))) (state-index (cdr t)))))))
trans)))) < #:key car))]
(lambda (a b) (< (car a) (car b)))))) [(null? old-states)
((null? old-states) (loop new-states null all-states (compute-chars new-states))]
(loop new-states null all-states (compute-chars new-states))) [(null? cs)
((null? cs) (loop (cdr old-states) new-states all-states (compute-chars (cdr old-states)))]
(loop (cdr old-states) new-states all-states (compute-chars (cdr old-states)))) [else
(else (define state (car old-states))
(let* ((state (car old-states)) (define c (car cs))
(c (car cs)) (define new-re (derive (state-spec state) (loc:get-integer c) cache))
(new-re (derive (state-spec state) (loc:get-integer c) cache)))
(cond (cond
(new-re [new-re
(let* ((new-state? #f) (let* ([new-state? #f]
(new-state (cache (cons 'state (get-key new-re)) [new-state (cache (cons 'state (get-key new-re))
(lambda () (λ ()
(set! new-state? #t) (set! new-state? #t)
(make-state new-re (get-state-number))))) (make-state new-re (get-state-number))))]
(new-all-states (if new-state? (cons new-state all-states) all-states))) [new-all-states (if new-state? (cons new-state all-states) all-states)])
(hash-table-put! transitions (hash-set! transitions
state state
(cons (cons c new-state) (cons (cons c new-state)
(hash-table-get transitions state (hash-ref transitions state
(lambda () null)))) (λ () null))))
(cond (cond
(new-state? [new-state?
(loop old-states (cons new-state new-states) new-all-states (cdr cs))) (loop old-states (cons new-state new-states) new-all-states (cdr cs))]
(else [else
(loop old-states new-states new-all-states (cdr cs)))))) (loop old-states new-states new-all-states (cdr cs))]))]
(else (loop old-states new-states all-states (cdr cs)))))))))) [else (loop old-states new-states all-states (cdr cs))])]))))
(define (print-dfa x) (define (print-dfa x)
(printf "number of states: ~a\n" (dfa-num-states x)) (printf "number of states: ~a\n" (dfa-num-states x))
(printf "start state: ~a\n" (dfa-start-state x)) (printf "start state: ~a\n" (dfa-start-state x))
(printf "final states: ~a\n" (map car (dfa-final-states/actions x))) (printf "final states: ~a\n" (map car (dfa-final-states/actions x)))
(for-each (lambda (trans) (for-each (λ (trans)
(printf "state: ~a\n" (car trans)) (printf "state: ~a\n" (car trans))
(for-each (lambda (rule) (for-each (λ (rule)
(printf " -~a-> ~a\n" (printf " -~a-> ~a\n"
(is:integer-set-contents (car rule)) (is:integer-set-contents (car rule))
(cdr rule))) (cdr rule)))
(cdr trans))) (cdr trans)))
(dfa-transitions x))) (dfa-transitions x)))
(define (build-test-dfa rs) (define (build-test-dfa rs)
(let ((c (make-cache))) (define c (make-cache))
(build-dfa (map (lambda (x) (cons (->re x c) 'action)) (build-dfa (map (λ (x) (cons (->re x c) 'action)) rs) c))
rs)
c)))
#| #|
@ -334,6 +328,6 @@
(define t13 (build-test-dfa `((intersection (concatenation (intersection) "111" (intersection)) (define t13 (build-test-dfa `((intersection (concatenation (intersection) "111" (intersection))
(complement (union (concatenation (intersection) "01") (complement (union (concatenation (intersection) "01")
(repetition 1 +inf.0 "1"))))))) (repetition 1 +inf.0 "1")))))))
(define t14 (build-test-dfa `((complement "1")))) (define t14 (build-test-dfa `((complement "1")))))
|#
) |#

4
br-parser-tools-lib/br-parser-tools/private-lex/error-tests.rkt
Unescape Escape View File

@ -1,5 +1,5 @@
#lang scheme/base #lang racket/base
(require (for-syntax scheme/base) (require (for-syntax racket/base)
"../lex.rkt" "../lex.rkt"
rackunit) rackunit)

168
br-parser-tools-lib/br-parser-tools/private-lex/front.rkt
Unescape Escape View File

@ -1,109 +1,96 @@
(module front mzscheme #lang racket/base
(require (prefix is: mzlib/integer-set) (require racket/base
mzlib/list racket/match
(prefix-in is: data/integer-set)
racket/list
syntax/stx syntax/stx
"util.rkt" "util.rkt"
"stx.rkt" "stx.rkt"
"re.rkt" "re.rkt"
"deriv.rkt") "deriv.rkt")
(provide build-lexer) (provide build-lexer)
(define-syntax time-label (define-syntax time-label
(syntax-rules () (syntax-rules ()
((_ l e ...) ((_ l e ...)
(begin (begin
(printf "~a: " l) (printf "~a: " l)
(time (begin e ...)))))) (time (begin e ...))))))
;; A table is either ;; A table is either
;; - (vector-of (union #f nat)) ;; - (vector-of (union #f nat))
;; - (vector-of (vector-of (vector nat nat nat))) ;; - (vector-of (vector-of (vector nat nat nat)))
(define loc:integer-set-contents is:integer-set-contents) (define loc:integer-set-contents is:integer-set-contents)
;; dfa->1d-table : dfa -> (same as build-lexer) ;; dfa->1d-table : dfa -> (same as build-lexer)
(define (dfa->1d-table dfa) (define (dfa->1d-table dfa)
(let ((state-table (make-vector (dfa-num-states dfa) #f)) (define state-table (make-vector (dfa-num-states dfa) #f))
(transition-cache (make-hash-table 'equal))) (define transition-cache (make-hasheq))
(for-each (for ([trans (in-list (dfa-transitions dfa))])
(lambda (trans) (match-define (cons from-state all-chars/to) trans)
(let* ((from-state (car trans)) (define flat-all-chars/to
(all-chars/to (cdr trans))
(flat-all-chars/to
(sort (sort
(apply append (for*/list ([chars/to (in-list all-chars/to)]
(map (lambda (chars/to) [char-ranges (in-value (loc:integer-set-contents (car chars/to)))]
(let ((char-ranges (loc:integer-set-contents (car chars/to))) [to (in-value (cdr chars/to))]
(to (cdr chars/to))) [char-range (in-list char-ranges)])
(map (lambda (char-range) (define entry (vector (car char-range) (cdr char-range) to))
(let ((entry (vector (car char-range) (cdr char-range) to))) (hash-ref transition-cache entry (λ ()
(hash-table-get transition-cache entry (hash-set! transition-cache
(lambda ()
(hash-table-put! transition-cache
entry entry
entry) entry)
entry)))) entry)))
char-ranges))) < #:key (λ (v) (vector-ref v 0))))
all-chars/to)) (vector-set! state-table from-state (list->vector flat-all-chars/to)))
(lambda (a b) state-table)
(< (vector-ref a 0) (vector-ref b 0))))))
(vector-set! state-table from-state (list->vector flat-all-chars/to))))
(dfa-transitions dfa))
state-table))
(define loc:foldr is:foldr) (define loc:foldr is:foldr)
;; dfa->2d-table : dfa -> (same as build-lexer) ;; dfa->2d-table : dfa -> (same as build-lexer)
(define (dfa->2d-table dfa) (define (dfa->2d-table dfa)
(let (
;; char-table : (vector-of (union #f nat)) ;; char-table : (vector-of (union #f nat))
;; The lexer table, one entry per state per char. ;; The lexer table, one entry per state per char.
;; Each entry specifies a state to transition to. ;; Each entry specifies a state to transition to.
;; #f indicates no transition ;; #f indicates no transition
(char-table (make-vector (* 256 (dfa-num-states dfa)) #f))) (define char-table (make-vector (* 256 (dfa-num-states dfa)) #f))
;; Fill the char-table vector ;; Fill the char-table vector
(for-each (for* ([trans (in-list (dfa-transitions dfa))]
(lambda (trans) [chars/to (in-list (cdr trans))])
(let ((from-state (car trans))) (define from-state (car trans))
(for-each (lambda (chars/to) (define to-state (cdr chars/to))
(let ((to-state (cdr chars/to))) (loc:foldr (λ (char _)
(loc:foldr (lambda (char _)
(vector-set! char-table (vector-set! char-table
(bitwise-ior (bitwise-ior
char char
(arithmetic-shift from-state 8)) (arithmetic-shift from-state 8))
to-state)) to-state))
(void) (void)
(car chars/to)))) (car chars/to)))
(cdr trans)))) char-table)
(dfa-transitions dfa))
char-table))
;; dfa->actions : dfa -> (vector-of (union #f syntax-object)) ;; dfa->actions : dfa -> (vector-of (union #f syntax-object))
;; The action for each final state, #f if the state isn't final ;; The action for each final state, #f if the state isn't final
(define (dfa->actions dfa) (define (dfa->actions dfa)
(let ((actions (make-vector (dfa-num-states dfa) #f))) (define actions (make-vector (dfa-num-states dfa) #f))
(for-each (lambda (state/action) (for ([state/action (in-list (dfa-final-states/actions dfa))])
(vector-set! actions (car state/action) (cdr state/action))) (vector-set! actions (car state/action) (cdr state/action)))
(dfa-final-states/actions dfa)) actions)
actions))
;; dfa->no-look : dfa -> (vector-of bool)
;; dfa->no-look : dfa -> (vector-of bool) ;; For each state whether the lexer can ignore the next input.
;; For each state whether the lexer can ignore the next input. ;; It can do this only if there are no transitions out of the
;; It can do this only if there are no transitions out of the ;; current state.
;; current state. (define (dfa->no-look dfa)
(define (dfa->no-look dfa) (define no-look (make-vector (dfa-num-states dfa) #t))
(let ((no-look (make-vector (dfa-num-states dfa) #t))) (for ([trans (in-list (dfa-transitions dfa))])
(for-each (lambda (trans)
(vector-set! no-look (car trans) #f)) (vector-set! no-look (car trans) #f))
(dfa-transitions dfa)) no-look)
no-look))
(test-block ((d1 (make-dfa 1 1 (list) (list))) (test-block ((d1 (make-dfa 1 1 (list) (list)))
(d2 (make-dfa 4 1 (list (cons 2 2) (cons 3 3)) (d2 (make-dfa 4 1 (list (cons 2 2) (cons 3 3))
(list (cons 1 (list (cons (is:make-range 49 50) 1) (list (cons 1 (list (cons (is:make-range 49 50) 1)
(cons (is:make-range 51) 2))) (cons (is:make-range 51) 2)))
@ -134,39 +121,34 @@
((dfa->no-look d1) (vector #t)) ((dfa->no-look d1) (vector #t))
((dfa->no-look d2) (vector #t #f #f #t))) ((dfa->no-look d2) (vector #t #f #f #t)))
;; build-lexer : syntax-object list -> ;; build-lexer : syntax-object list ->
;; (values table nat (vector-of (union #f syntax-object)) (vector-of bool) (list-of syntax-object)) ;; (values table nat (vector-of (union #f syntax-object)) (vector-of bool) (list-of syntax-object))
;; each syntax object has the form (re action) ;; each syntax object has the form (re action)
(define (build-lexer sos) (define (build-lexer sos)
(let* ((disappeared-uses (box null)) (define disappeared-uses (box null))
(s-re-acts (map (lambda (so) (define s-re-acts (for/list ([so (in-list sos)])
(cons (parse (stx-car so) disappeared-uses) (cons (parse (stx-car so) disappeared-uses)
(stx-car (stx-cdr so)))) (stx-car (stx-cdr so)))))
sos)) (define cache (make-cache))
(define re-acts (for/list ([s-re-act (in-list s-re-acts)])
(cache (make-cache))
(re-acts (map (lambda (s-re-act)
(cons (->re (car s-re-act) cache) (cons (->re (car s-re-act) cache)
(cdr s-re-act))) (cdr s-re-act))))
s-re-acts)) (define dfa (build-dfa re-acts cache))
(define table (dfa->1d-table dfa))
(dfa (build-dfa re-acts cache))
(table (dfa->1d-table dfa)))
;(print-dfa dfa) ;(print-dfa dfa)
#;(let ((num-states (vector-length table)) #;(let ((num-states (vector-length table))
(num-vectors (length (filter values (vector->list table)))) (num-vectors (length (filter values (vector->list table))))
(num-entries (apply + (map (num-entries (apply + (map
(lambda (x) (if x (vector-length x) 0)) (λ (x) (if x (vector-length x) 0))
(vector->list table)))) (vector->list table))))
(num-different-entries (num-different-entries
(let ((ht (make-hash-table))) (let ((ht (make-hash)))
(for-each (for-each
(lambda (x) (λ (x)
(when x (when x
(for-each (for-each
(lambda (y) (λ (y)
(hash-table-put! ht y #t)) (hash-set! ht y #t))
(vector->list x)))) (vector->list x))))
(vector->list table)) (vector->list table))
(length (hash-table-map ht cons))))) (length (hash-table-map ht cons)))))
@ -175,5 +157,5 @@
(/ (* 4.0 (+ 2 num-states (* 2 num-vectors) num-entries (/ (* 4.0 (+ 2 num-states (* 2 num-vectors) num-entries
(* 5 num-different-entries))) 1024))) (* 5 num-different-entries))) 1024)))
(values table (dfa-start-state dfa) (dfa->actions dfa) (dfa->no-look dfa) (values table (dfa-start-state dfa) (dfa->actions dfa) (dfa->no-look dfa)
(unbox disappeared-uses)))) (unbox disappeared-uses)))
)

323
br-parser-tools-lib/br-parser-tools/private-lex/re.rkt
Unescape Escape View File

@ -1,228 +1,227 @@
(module re mzscheme #lang racket/base
(require mzlib/list (require racket/list
scheme/match racket/match
(prefix is: mzlib/integer-set) (prefix-in is: data/integer-set)
"util.rkt") "util.rkt")
(provide ->re build-epsilon build-zero build-char-set build-concat (provide ->re build-epsilon build-zero build-char-set build-concat
build-repeat build-or build-and build-neg build-repeat build-or build-and build-neg
epsilonR? zeroR? char-setR? concatR? repeatR? orR? andR? negR? epsilonR? zeroR? char-setR? concatR? repeatR? orR? andR? negR?
char-setR-chars concatR-re1 concatR-re2 repeatR-re repeatR-low repeatR-high char-setR-chars concatR-re1 concatR-re2 repeatR-re repeatR-low repeatR-high
orR-res andR-res negR-re orR-res andR-res negR-re
re-nullable? re-index) re-nullable? re-index)
;; get-index : -> nat ;; get-index : -> nat
(define get-index (make-counter)) (define get-index (make-counter))
;; An re is either ;; An re is either
;; - (make-epsilonR bool nat) ;; - (make-epsilonR bool nat)
;; - (make-zeroR bool nat) ;; - (make-zeroR bool nat)
;; - (make-char-setR bool nat char-set) ;; - (make-char-setR bool nat char-set)
;; - (make-concatR bool nat re re) ;; - (make-concatR bool nat re re)
;; - (make-repeatR bool nat nat nat-or-+inf.0 re) ;; - (make-repeatR bool nat nat nat-or-+inf.0 re)
;; - (make-orR bool nat (list-of re)) Must not directly contain any orRs ;; - (make-orR bool nat (list-of re)) Must not directly contain any orRs
;; - (make-andR bool nat (list-of re)) Must not directly contain any andRs ;; - (make-andR bool nat (list-of re)) Must not directly contain any andRs
;; - (make-negR bool nat re) ;; - (make-negR bool nat re)
;; ;;
;; Every re must have an index field globally different from all ;; Every re must have an index field globally different from all
;; other re index fields. ;; other re index fields.
(define-struct re (nullable? index) (make-inspector)) (define-struct re (nullable? index) #:inspector (make-inspector))
(define-struct (epsilonR re) () (make-inspector)) (define-struct (epsilonR re) () #:inspector (make-inspector))
(define-struct (zeroR re) () (make-inspector)) (define-struct (zeroR re) () #:inspector (make-inspector))
(define-struct (char-setR re) (chars) (make-inspector)) (define-struct (char-setR re) (chars) #:inspector (make-inspector))
(define-struct (concatR re) (re1 re2) (make-inspector)) (define-struct (concatR re) (re1 re2) #:inspector (make-inspector))
(define-struct (repeatR re) (low high re) (make-inspector)) (define-struct (repeatR re) (low high re) #:inspector (make-inspector))
(define-struct (orR re) (res) (make-inspector)) (define-struct (orR re) (res) #:inspector (make-inspector))
(define-struct (andR re) (res) (make-inspector)) (define-struct (andR re) (res) #:inspector (make-inspector))
(define-struct (negR re) (re) (make-inspector)) (define-struct (negR re) (re) #:inspector (make-inspector))
;; e : re ;; e : re
;; The unique epsilon re ;; The unique epsilon re
(define e (make-epsilonR #t (get-index))) (define e (make-epsilonR #t (get-index)))
;; z : re ;; z : re
;; The unique zero re ;; The unique zero re
(define z (make-zeroR #f (get-index))) (define z (make-zeroR #f (get-index)))
;; s-re = char constant ;; s-re = char constant
;; | string constant (sequence of characters) ;; | string constant (sequence of characters)
;; | re a precompiled re ;; | re a precompiled re
;; | (repetition low high s-re) repetition between low and high times (inclusive) ;; | (repetition low high s-re) repetition between low and high times (inclusive)
;; | (union s-re ...) ;; | (union s-re ...)
;; | (intersection s-re ...) ;; | (intersection s-re ...)
;; | (complement s-re) ;; | (complement s-re)
;; | (concatenation s-re ...) ;; | (concatenation s-re ...)
;; | (char-range rng rng) match any character between two (inclusive) ;; | (char-range rng rng) match any character between two (inclusive)
;; | (char-complement char-set) match any character not listed ;; | (char-complement char-set) match any character not listed
;; low = natural-number ;; low = natural-number
;; high = natural-number or +inf.0 ;; high = natural-number or +inf.0
;; rng = char or string with length 1 ;; rng = char or string with length 1
;; (concatenation) (repetition 0 0 x), and "" match the empty string. ;; (concatenation) (repetition 0 0 x), and "" match the empty string.
;; (union) matches no strings. ;; (union) matches no strings.
;; (intersection) matches any string. ;; (intersection) matches any string.
(define loc:make-range is:make-range) (define loc:make-range is:make-range)
(define loc:union is:union) (define loc:union is:union)
(define loc:split is:split) (define loc:split is:split)
(define loc:complement is:complement) (define loc:complement is:complement)
;; ->re : s-re cache -> re ;; ->re : s-re cache -> re
(define (->re exp cache) (define (->re exp cache)
(match exp (match exp
((? char?) (build-char-set (loc:make-range (char->integer exp)) cache)) [(? char?) (build-char-set (loc:make-range (char->integer exp)) cache)]
((? string?) (->re `(concatenation ,@(string->list exp)) cache)) [(? string?) (->re `(concatenation ,@(string->list exp)) cache)]
((? re?) exp) [(? re?) exp]
(`(repetition ,low ,high ,r) [`(repetition ,low ,high ,r)
(build-repeat low high (->re r cache) cache)) (build-repeat low high (->re r cache) cache)]
(`(union ,rs ...) [`(union ,rs ...)
(build-or (flatten-res (map (lambda (r) (->re r cache)) rs) (build-or (flatten-res (map (λ (r) (->re r cache)) rs)
orR? orR-res loc:union cache) orR? orR-res loc:union cache)
cache)) cache)]
(`(intersection ,rs ...) [`(intersection ,rs ...)
(build-and (flatten-res (map (lambda (r) (->re r cache)) rs) (build-and (flatten-res (map (λ (r) (->re r cache)) rs)
andR? andR-res (lambda (a b) andR? andR-res (λ (a b)
(let-values (((i _ __) (loc:split a b))) i)) (let-values (((i _ __) (loc:split a b))) i))
cache) cache)
cache)) cache)]
(`(complement ,r) [`(complement ,r) (build-neg (->re r cache) cache)]
(build-neg (->re r cache) cache)) [`(concatenation ,rs ...)
(`(concatenation ,rs ...) (foldr (λ (x y)
(foldr (lambda (x y)
(build-concat (->re x cache) y cache)) (build-concat (->re x cache) y cache))
e e
rs)) rs)]
(`(char-range ,c1 ,c2) [`(char-range ,c1 ,c2)
(let ((i1 (char->integer (if (string? c1) (string-ref c1 0) c1))) (let ([i1 (char->integer (if (string? c1) (string-ref c1 0) c1))]
(i2 (char->integer (if (string? c2) (string-ref c2 0) c2)))) [i2 (char->integer (if (string? c2) (string-ref c2 0) c2))])
(if (<= i1 i2) (if (<= i1 i2)
(build-char-set (loc:make-range i1 i2) cache) (build-char-set (loc:make-range i1 i2) cache)
z))) z))]
(`(char-complement ,crs ...) [`(char-complement ,crs ...)
(let ((cs (->re `(union ,@crs) cache))) (let ([cs (->re `(union ,@crs) cache)])
(cond (cond
((zeroR? cs) (build-char-set (loc:make-range 0 max-char-num) cache)) [(zeroR? cs) (build-char-set (loc:make-range 0 max-char-num) cache)]
((char-setR? cs) [(char-setR? cs)
(build-char-set (loc:complement (char-setR-chars cs) 0 max-char-num) cache)) (build-char-set (loc:complement (char-setR-chars cs) 0 max-char-num) cache)]
(else z)))))) [else z]))]))
;; flatten-res: (list-of re) (re -> bool) (re -> (list-of re)) ;; flatten-res: (list-of re) (re -> bool) (re -> (list-of re))
;; (char-set char-set -> char-set) cache -> (list-of re) ;; (char-set char-set -> char-set) cache -> (list-of re)
;; Takes all the char-sets in l and combines them into one char-set using the combine function. ;; Takes all the char-sets in l and combines them into one char-set using the combine function.
;; Flattens out the values of type?. get-res only needs to function on things type? returns ;; Flattens out the values of type?. get-res only needs to function on things type? returns
;; true for. ;; true for.
(define (flatten-res l type? get-res combine cache) (define (flatten-res l type? get-res combine cache)
(let loop ((res l) (let loop ([res l]
;; chars : (union #f char-set) ;; chars : (union #f char-set)
(chars #f) [chars #f]
(no-chars null)) [no-chars null])
(cond (cond
((null? res) [(null? res)
(if chars (if chars
(cons (build-char-set chars cache) no-chars) (cons (build-char-set chars cache) no-chars)
no-chars)) no-chars)]
((char-setR? (car res)) [(char-setR? (car res))
(if chars (if chars
(loop (cdr res) (combine (char-setR-chars (car res)) chars) no-chars) (loop (cdr res) (combine (char-setR-chars (car res)) chars) no-chars)
(loop (cdr res) (char-setR-chars (car res)) no-chars))) (loop (cdr res) (char-setR-chars (car res)) no-chars))]
((type? (car res)) [(type? (car res))
(loop (append (get-res (car res)) (cdr res)) chars no-chars)) (loop (append (get-res (car res)) (cdr res)) chars no-chars)]
(else (loop (cdr res) chars (cons (car res) no-chars)))))) [else (loop (cdr res) chars (cons (car res) no-chars))])))
;; build-epsilon : -> re ;; build-epsilon : -> re
(define (build-epsilon) e) (define (build-epsilon) e)
(define (build-zero) z) (define (build-zero) z)
(define loc:integer-set-contents is:integer-set-contents) (define loc:integer-set-contents is:integer-set-contents)
;; build-char-set : char-set cache -> re ;; build-char-set : char-set cache -> re
(define (build-char-set cs cache) (define (build-char-set cs cache)
(let ((l (loc:integer-set-contents cs))) (define l (loc:integer-set-contents cs))
(cond (cond
((null? l) z) [(null? l) z]
(else [else
(cache l (cache l
(lambda () (λ ()
(make-char-setR #f (get-index) cs))))))) (make-char-setR #f (get-index) cs)))]))
;; build-concat : re re cache -> re ;; build-concat : re re cache -> re
(define (build-concat r1 r2 cache) (define (build-concat r1 r2 cache)
(cond (cond
((eq? e r1) r2) [(eq? e r1) r2]
((eq? e r2) r1) [(eq? e r2) r1]
((or (eq? z r1) (eq? z r2)) z) [(or (eq? z r1) (eq? z r2)) z]
(else [else
(cache (cons 'concat (cons (re-index r1) (re-index r2))) (cache (cons 'concat (cons (re-index r1) (re-index r2)))
(lambda () (λ ()
(make-concatR (and (re-nullable? r1) (re-nullable? r2)) (make-concatR (and (re-nullable? r1) (re-nullable? r2))
(get-index) (get-index)
r1 r2)))))) r1 r2)))]))
;; build-repeat : nat nat-or-+inf.0 re cache -> re ;; build-repeat : nat nat-or-+inf.0 re cache -> re
(define (build-repeat low high r cache) (define (build-repeat low high r cache)
(let ((low (if (< low 0) 0 low))) (let ([low (if (< low 0) 0 low)])
(cond (cond
((eq? r e) e) [(eq? r e) e]
((and (= 0 low) (or (= 0 high) (eq? z r))) e) [(and (= 0 low) (or (= 0 high) (eq? z r))) e]
((and (= 1 low) (= 1 high)) r) [(and (= 1 low) (= 1 high)) r]
((and (repeatR? r) [(and (repeatR? r)
(eqv? (repeatR-high r) +inf.0) (eqv? (repeatR-high r) +inf.0)
(or (= 0 (repeatR-low r)) (or (= 0 (repeatR-low r))
(= 1 (repeatR-low r)))) (= 1 (repeatR-low r))))
(build-repeat (* low (repeatR-low r)) (build-repeat (* low (repeatR-low r))
+inf.0 +inf.0
(repeatR-re r) (repeatR-re r)
cache)) cache)]
(else [else
(cache (cons 'repeat (cons low (cons high (re-index r)))) (cache (cons 'repeat (cons low (cons high (re-index r))))
(lambda () (λ ()
(make-repeatR (or (re-nullable? r) (= 0 low)) (get-index) low high r))))))) (make-repeatR (or (re-nullable? r) (= 0 low)) (get-index) low high r)))])))
;; build-or : (list-of re) cache -> re ;; build-or : (list-of re) cache -> re
(define (build-or rs cache) (define (build-or rs cache)
(let ((rs (let ([rs
(filter (filter
(lambda (x) (not (eq? x z))) (λ (x) (not (eq? x z)))
(do-simple-equiv (replace rs orR? orR-res null) re-index)))) (do-simple-equiv (replace rs orR? orR-res null) re-index))])
(cond (cond
((null? rs) z) [(null? rs) z]
((null? (cdr rs)) (car rs)) [(null? (cdr rs)) (car rs)]
((memq (build-neg z cache) rs) (build-neg z cache)) [(memq (build-neg z cache) rs) (build-neg z cache)]
(else [else
(cache (cons 'or (map re-index rs)) (cache (cons 'or (map re-index rs))
(lambda () (λ ()
(make-orR (ormap re-nullable? rs) (get-index) rs))))))) (make-orR (ormap re-nullable? rs) (get-index) rs)))])))
;; build-and : (list-of re) cache -> re ;; build-and : (list-of re) cache -> re
(define (build-and rs cache) (define (build-and rs cache)
(let ((rs (do-simple-equiv (replace rs andR? andR-res null) re-index))) (let ([rs (do-simple-equiv (replace rs andR? andR-res null) re-index)])
(cond (cond
((null? rs) (build-neg z cache)) [(null? rs) (build-neg z cache)]
((null? (cdr rs)) (car rs)) [(null? (cdr rs)) (car rs)]
((memq z rs) z) [(memq z rs) z]
(else [else
(cache (cons 'and (map re-index rs)) (cache (cons 'and (map re-index rs))
(lambda () (λ ()
(make-andR (andmap re-nullable? rs) (get-index) rs))))))) (make-andR (andmap re-nullable? rs) (get-index) rs)))])))
;; build-neg : re cache -> re ;; build-neg : re cache -> re
(define (build-neg r cache) (define (build-neg r cache)
(cond (cond
((negR? r) (negR-re r)) [(negR? r) (negR-re r)]
(else [else
(cache (cons 'neg (re-index r)) (cache (cons 'neg (re-index r))
(lambda () (λ ()
(make-negR (not (re-nullable? r)) (get-index) r)))))) (make-negR (not (re-nullable? r)) (get-index) r)))]))
;; Tests for the build-functions ;; Tests for the build-functions
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(isc is:integer-set-contents) (isc is:integer-set-contents)
(r1 (build-char-set (is:make-range (char->integer #\1)) c)) (r1 (build-char-set (is:make-range (char->integer #\1)) c))
(r2 (build-char-set (is:make-range (char->integer #\2)) c)) (r2 (build-char-set (is:make-range (char->integer #\2)) c))
@ -301,7 +300,7 @@
((re-nullable? (build-neg r1 c)) #t) ((re-nullable? (build-neg r1 c)) #t)
((re-nullable? (build-neg rr c)) #f)) ((re-nullable? (build-neg rr c)) #f))
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(isc is:integer-set-contents) (isc is:integer-set-contents)
(r1 (->re #\1 c)) (r1 (->re #\1 c))
(r2 (->re #\2 c)) (r2 (->re #\2 c))
@ -317,15 +316,15 @@
((isc (char-setR-chars (car (flatten-res `(,r6 ,r5 ,r4 ,r3-5 ,r2 ,r1) ((isc (char-setR-chars (car (flatten-res `(,r6 ,r5 ,r4 ,r3-5 ,r2 ,r1)
orR? orR-res is:union c)))) orR? orR-res is:union c))))
(isc (is:make-range (char->integer #\1) (char->integer #\7)))) (isc (is:make-range (char->integer #\1) (char->integer #\7))))
((flatten-res `(,r1 ,r2) andR? andR-res (lambda (x y) ((flatten-res `(,r1 ,r2) andR? andR-res (λ (x y)
(let-values (((i _ __) (let-values (((i _ __)
(is:split x y))) (is:split x y)))
i)) i))
c) c)
(list z))) (list z)))
;; ->re ;; ->re
(test-block ((c (make-cache)) (test-block ((c (make-cache))
(isc is:integer-set-contents) (isc is:integer-set-contents)
(r (->re #\a c)) (r (->re #\a c))
(rr (->re `(concatenation ,r ,r) c)) (rr (->re `(concatenation ,r ,r) c))
@ -382,4 +381,4 @@
(isc (is:make-range 0))) (isc (is:make-range 0)))
) )
)

195
br-parser-tools-lib/br-parser-tools/private-lex/stx.rkt
Unescape Escape View File

@ -1,30 +1,25 @@
#lang racket #lang racket/base
(require "util.rkt" syntax/id-table)
(require "util.rkt"
syntax/id-table)
(provide parse) (provide parse)
(define (bad-args stx num) (define (bad-args stx num)
(raise-syntax-error (raise-syntax-error #f (format "incorrect number of arguments (should have ~a)" num) stx))
#f
(format "incorrect number of arguments (should have ~a)" num)
stx))
;; char-range-arg: syntax-object syntax-object -> nat ;; char-range-arg: syntax-object syntax-object -> nat
;; If c contains is a character or length 1 string, returns the integer ;; If c contains is a character or length 1 string, returns the integer
;; for the character. Otherwise raises a syntax error. ;; for the character. Otherwise raises a syntax error.
(define (char-range-arg stx containing-stx) (define (char-range-arg stx containing-stx)
(let ((c (syntax-e stx))) (define c (syntax-e stx))
(cond (cond
((char? c) (char->integer c)) [(char? c) (char->integer c)]
((and (string? c) (= (string-length c) 1)) [(and (string? c) (= (string-length c) 1))
(char->integer (string-ref c 0))) (char->integer (string-ref c 0))]
(else [else
(raise-syntax-error (raise-syntax-error
#f #f
"not a char or single-char string" "not a char or single-char string"
containing-stx stx))))) containing-stx stx)]))
(module+ test (module+ test
(check-equal? (char-range-arg #'#\1 #'here) (char->integer #\1)) (check-equal? (char-range-arg #'#\1 #'here) (char->integer #\1))
(check-equal? (char-range-arg #'"1" #'here) (char->integer #\1))) (check-equal? (char-range-arg #'"1" #'here) (char->integer #\1)))
@ -34,147 +29,118 @@
(define (disarm stx) (define (disarm stx)
(syntax-disarm stx orig-insp)) (syntax-disarm stx orig-insp))
;; parse : syntax-object (box (list-of syntax-object)) -> s-re (see re.rkt) ;; parse : syntax-object (box (list-of syntax-object)) -> s-re (see re.rkt)
;; checks for errors and generates the plain s-exp form for s ;; checks for errors and generates the plain s-exp form for s
;; Expands lex-abbrevs and applies lex-trans. ;; Expands lex-abbrevs and applies lex-trans.
(define (parse stx disappeared-uses) (define (parse stx disappeared-uses)
(let loop ([stx stx] (let loop ([stx stx]
[disappeared-uses disappeared-uses] [disappeared-uses disappeared-uses]
;; seen-lex-abbrevs: id-table ;; seen-lex-abbrevs: id-table
[seen-lex-abbrevs (make-immutable-free-id-table)]) [seen-lex-abbrevs (make-immutable-free-id-table)])
(let ([recur (lambda (s) (let ([recur (λ (s)
(loop (syntax-rearm s stx) (loop (syntax-rearm s stx)
disappeared-uses disappeared-uses
seen-lex-abbrevs))] seen-lex-abbrevs))]
[recur/abbrev (lambda (s id) [recur/abbrev (λ (s id)
(loop (syntax-rearm s stx) (loop (syntax-rearm s stx)
disappeared-uses disappeared-uses
(free-id-table-set seen-lex-abbrevs id id)))]) (free-id-table-set seen-lex-abbrevs id id)))])
(syntax-case (disarm stx) (repetition union intersection complement concatenation (syntax-case (disarm stx) (repetition union intersection complement concatenation
char-range char-complement) char-range char-complement)
(_ [_
(identifier? stx) (identifier? stx)
(let ((expansion (syntax-local-value stx (lambda () #f)))) (let ([expansion (syntax-local-value stx (λ () #f))])
(unless (lex-abbrev? expansion) (unless (lex-abbrev? expansion)
(raise-syntax-error 'regular-expression (raise-syntax-error 'regular-expression
"undefined abbreviation" "undefined abbreviation"
stx)) stx))
;; Check for cycles. ;; Check for cycles.
(when (free-id-table-ref seen-lex-abbrevs stx (lambda () #f)) (when (free-id-table-ref seen-lex-abbrevs stx (λ () #f))
(raise-syntax-error 'regular-expression (raise-syntax-error 'regular-expression
"illegal lex-abbrev cycle detected" "illegal lex-abbrev cycle detected"
stx stx
#f #f
(list (free-id-table-ref seen-lex-abbrevs stx)))) (list (free-id-table-ref seen-lex-abbrevs stx))))
(set-box! disappeared-uses (cons stx (unbox disappeared-uses))) (set-box! disappeared-uses (cons stx (unbox disappeared-uses)))
(recur/abbrev ((lex-abbrev-get-abbrev expansion)) stx))) (recur/abbrev ((lex-abbrev-get-abbrev expansion)) stx))]
(_ [_
(or (char? (syntax-e stx)) (string? (syntax-e stx))) (or (char? (syntax-e stx)) (string? (syntax-e stx)))
(syntax-e stx)) (syntax-e stx)]
((repetition arg ...) [(repetition ARG ...)
(let ((arg-list (syntax->list (syntax (arg ...))))) (let ([arg-list (syntax->list #'(ARG ...))])
(unless (= 3 (length arg-list)) (unless (= 3 (length arg-list))
(bad-args stx 2)) (bad-args stx 2))
(let ((low (syntax-e (car arg-list))) (define low (syntax-e (car arg-list)))
(high (syntax-e (cadr arg-list))) (define high (syntax-e (cadr arg-list)))
(re (caddr arg-list))) (define re (caddr arg-list))
(unless (and (number? low) (exact? low) (integer? low) (>= low 0)) (unless (and (number? low) (exact? low) (integer? low) (>= low 0))
(raise-syntax-error #f (raise-syntax-error #f "not a non-negative exact integer" stx (car arg-list)))
"not a non-negative exact integer"
stx
(car arg-list)))
(unless (or (and (number? high) (exact? high) (integer? high) (>= high 0)) (unless (or (and (number? high) (exact? high) (integer? high) (>= high 0))
(eqv? high +inf.0)) (eqv? high +inf.0))
(raise-syntax-error #f (raise-syntax-error #f "not a non-negative exact integer or +inf.0" stx (cadr arg-list)))
"not a non-negative exact integer or +inf.0"
stx
(cadr arg-list)))
(unless (<= low high) (unless (<= low high)
(raise-syntax-error (raise-syntax-error #f "the first argument is not less than or equal to the second argument" stx))
#f `(repetition ,low ,high ,(recur re)))]
"the first argument is not less than or equal to the second argument" [(union RE ...)
stx)) `(union ,@(map recur (syntax->list #'(RE ...))))]
`(repetition ,low ,high ,(recur re))))) [(intersection RE ...)
((union re ...) `(intersection ,@(map recur (syntax->list #'(RE ...))))]
`(union ,@(map recur (syntax->list (syntax (re ...)))))) [(complement RE ...)
((intersection re ...) (let ([re-list (syntax->list #'(RE ...))])
`(intersection ,@(map recur (syntax->list (syntax (re ...))))))
((complement re ...)
(let ((re-list (syntax->list (syntax (re ...)))))
(unless (= 1 (length re-list)) (unless (= 1 (length re-list))
(bad-args stx 1)) (bad-args stx 1))
`(complement ,(recur (car re-list))))) `(complement ,(recur (car re-list))))]
((concatenation re ...) [(concatenation RE ...)
`(concatenation ,@(map recur (syntax->list (syntax (re ...)))))) `(concatenation ,@(map recur (syntax->list #'(RE ...))))]
((char-range arg ...) [(char-range ARG ...)
(let ((arg-list (syntax->list (syntax (arg ...))))) (let ((arg-list (syntax->list #'(ARG ...))))
(unless (= 2 (length arg-list)) (unless (= 2 (length arg-list))
(bad-args stx 2)) (bad-args stx 2))
(let ((i1 (char-range-arg (car arg-list) stx)) (let ([i1 (char-range-arg (car arg-list) stx)]
(i2 (char-range-arg (cadr arg-list) stx))) [i2 (char-range-arg (cadr arg-list) stx)])
(if (<= i1 i2) (if (<= i1 i2)
`(char-range ,(integer->char i1) ,(integer->char i2)) `(char-range ,(integer->char i1) ,(integer->char i2))
(raise-syntax-error (raise-syntax-error #f "the first argument does not precede or equal second argument" stx))))]
#f [(char-complement ARG ...)
"the first argument does not precede or equal second argument" (let ([arg-list (syntax->list #'(ARG ...))])
stx)))))
((char-complement arg ...)
(let ((arg-list (syntax->list (syntax (arg ...)))))
(unless (= 1 (length arg-list)) (unless (= 1 (length arg-list))
(bad-args stx 1)) (bad-args stx 1))
(let ((parsed (recur (car arg-list)))) (define parsed (recur (car arg-list)))
(unless (char-set? parsed) (unless (char-set? parsed)
(raise-syntax-error #f (raise-syntax-error #f "not a character set" stx (car arg-list)))
"not a character set" `(char-complement ,parsed))]
stx ((OP form ...)
(car arg-list))) (identifier? #'OP)
`(char-complement ,parsed)))) (let* ([expansion (syntax-local-value #'OP (λ () #f))])
((op form ...) (set-box! disappeared-uses (cons #'OP (unbox disappeared-uses)))
(identifier? (syntax op))
(let* ((o (syntax op))
(expansion (syntax-local-value o (lambda () #f))))
(set-box! disappeared-uses (cons o (unbox disappeared-uses)))
(cond (cond
((lex-trans? expansion) [(lex-trans? expansion)
(recur ((lex-trans-f expansion) (disarm stx)))) (recur ((lex-trans-f expansion) (disarm stx)))]
(expansion [expansion
(raise-syntax-error 'regular-expression (raise-syntax-error 'regular-expression "not a lex-trans" stx)]
"not a lex-trans" [else
stx)) (raise-syntax-error 'regular-expression "undefined operator" stx)])))
(else [_ (raise-syntax-error 'regular-expression "not a char, string, identifier, or (op args ...)" stx)]))))
(raise-syntax-error 'regular-expression
"undefined operator"
stx)))))
(_
(raise-syntax-error
'regular-expression
"not a char, string, identifier, or (op args ...)"
stx))))))
;; char-set? : s-re -> bool ;; char-set? : s-re -> bool
;; A char-set is an re that matches only strings of length 1. ;; A char-set is an re that matches only strings of length 1.
;; char-set? is conservative. ;; char-set? is conservative.
(define (char-set? s-re) (define (char-set? s-re)
(cond (cond
((char? s-re) #t) [(char? s-re)]
((string? s-re) (= (string-length s-re) 1)) [(string? s-re) (= (string-length s-re) 1)]
((list? s-re) [(list? s-re) (case (car s-re)
(let ((op (car s-re))) [(union intersection) (andmap char-set? (cdr s-re))]
(case op [(char-range char-complement) #t]
((union intersection) (andmap char-set? (cdr s-re))) [(repetition) (and (= (cadr s-re) (caddr s-re)) (char-set? (cadddr s-re)))]
((char-range char-complement) #t) [(concatenation) (and (= 2 (length s-re)) (char-set? (cadr s-re)))]
((repetition) (else #f))]
(and (= (cadr s-re) (caddr s-re)) (char-set? (cadddr s-re)))) [else #f]))
((concatenation)
(and (= 2 (length s-re)) (char-set? (cadr s-re))))
(else #f))))
(else #f)))
(module+ test (module+ test
(require rackunit)) (require rackunit)
(module+ test
(check-equal? (char-set? #\a) #t) (check-equal? (char-set? #\a) #t)
(check-equal? (char-set? "12") #f) (check-equal? (char-set? "12") #f)
(check-equal? (char-set? "1") #t) (check-equal? (char-set? "1") #t)
@ -193,10 +159,10 @@
(check-equal? (char-set? '(char-range #\1 #\2)) #t) (check-equal? (char-set? '(char-range #\1 #\2)) #t)
(check-equal? (char-set? '(char-complement #\1)) #t)) (check-equal? (char-set? '(char-complement #\1)) #t))
;; yikes... these test cases all have the wrong arity, now. ;; yikes... these test cases all have the wrong arity, now.
;; and by "now", I mean it's been broken since before we ;; and by "now", I mean it's been broken since before we
;; moved to git. ;; moved to git.
(module+ test (module+ test
(check-equal? (parse #'#\a null) #\a) (check-equal? (parse #'#\a null) #\a)
(check-equal? (parse #'"1" null) "1") (check-equal? (parse #'"1" null) "1")
(check-equal? (parse #'(repetition 1 1 #\1) null) (check-equal? (parse #'(repetition 1 1 #\1) null)
@ -217,4 +183,3 @@
(check-equal? (parse #'(char-range "1" "3") null) '(char-range #\1 #\3)) (check-equal? (parse #'(char-range "1" "3") null) '(char-range #\1 #\3))
(check-equal? (parse #'(char-complement (union "1" "2")) null) (check-equal? (parse #'(char-complement (union "1" "2")) null)
'(char-complement (union "1" "2")))) '(char-complement (union "1" "2"))))
; )

14
br-parser-tools-lib/br-parser-tools/private-lex/token-syntax.rkt
Unescape Escape View File

@ -1,9 +1,7 @@
(module token-syntax mzscheme #lang racket/base
(provide make-terminals-def terminals-def-t terminals-def?
;; The things needed at compile time to handle definition of tokens
(provide make-terminals-def terminals-def-t terminals-def?
make-e-terminals-def e-terminals-def-t e-terminals-def?) make-e-terminals-def e-terminals-def-t e-terminals-def?)
(define-struct terminals-def (t))
(define-struct e-terminals-def (t)) ;; The things needed at compile time to handle definition of tokens
) (define-struct terminals-def (t))
(define-struct e-terminals-def (t))

98
br-parser-tools-lib/br-parser-tools/private-lex/token.rkt
Unescape Escape View File

@ -1,68 +1,57 @@
(module token mzscheme #lang racket/base
(require (for-syntax racket/base "token-syntax.rkt"))
(require-for-syntax "token-syntax.rkt") ;; Defining tokens
;; Defining tokens (provide define-tokens define-empty-tokens make-token token?
(protect-out (rename-out [token-name real-token-name]))
(protect-out (rename-out [token-value real-token-value]))
(rename-out [token-name* token-name][token-value* token-value])
(struct-out position)
(struct-out position-token)
(struct-out srcloc-token))
(provide define-tokens define-empty-tokens make-token token?
(protect (rename token-name real-token-name))
(protect (rename token-value real-token-value))
(rename token-name* token-name)
(rename token-value* token-value)
(struct position (offset line col))
(struct position-token (token start-pos end-pos))
(struct srcloc-token (token srcloc)))
;; A token is either
;; - symbol
;; - (make-token symbol any)
(define-struct token (name value) #:inspector (make-inspector))
;; A token is either ;; token-name*: token -> symbol
;; - symbol (define (token-name* t)
;; - (make-token symbol any)
(define-struct token (name value) (make-inspector))
;; token-name*: token -> symbol
(define (token-name* t)
(cond (cond
((symbol? t) t) [(symbol? t) t]
((token? t) (token-name t)) [(token? t) (token-name t)]
(else (raise-type-error [else (raise-type-error 'token-name "symbol or struct:token" 0 t)]))
'token-name
"symbol or struct:token"
0
t))))
;; token-value*: token -> any ;; token-value*: token -> any
(define (token-value* t) (define (token-value* t)
(cond (cond
((symbol? t) #f) [(symbol? t) #f]
((token? t) (token-value t)) [(token? t) (token-value t)]
(else (raise-type-error [else (raise-type-error 'token-value "symbol or struct:token" 0 t)]))
'token-value
"symbol or struct:token"
0
t))))
(define-for-syntax (make-ctor-name n) (define-for-syntax (make-ctor-name n)
(datum->syntax-object n (datum->syntax n
(string->symbol (format "token-~a" (syntax-e n))) (string->symbol (format "token-~a" (syntax-e n)))
n n
n)) n))
(define-for-syntax (make-define-tokens empty?) (define-for-syntax ((make-define-tokens empty?) stx)
(lambda (stx)
(syntax-case stx () (syntax-case stx ()
((_ name (token ...)) [(_ NAME (TOKEN ...))
(andmap identifier? (syntax->list (syntax (token ...)))) (andmap identifier? (syntax->list #'(TOKEN ...)))
(with-syntax (((marked-token ...) (with-syntax (((marked-token ...)
(map values #;(make-syntax-introducer) (map values #;(make-syntax-introducer)
(syntax->list (syntax (token ...)))))) (syntax->list #'(TOKEN ...)))))
(quasisyntax/loc stx (quasisyntax/loc stx
(begin (begin
(define-syntax name (define-syntax NAME
#,(if empty? #,(if empty?
#'(make-e-terminals-def (quote-syntax (marked-token ...))) #'(make-e-terminals-def (quote-syntax (marked-token ...)))
#'(make-terminals-def (quote-syntax (marked-token ...))))) #'(make-terminals-def (quote-syntax (marked-token ...)))))
#,@(map #,@(map
(lambda (n) (λ (n)
(when (eq? (syntax-e n) 'error) (when (eq? (syntax-e n) 'error)
(raise-syntax-error (raise-syntax-error
#f #f
@ -73,20 +62,19 @@
'#,n) '#,n)
#`(define (#,(make-ctor-name n) x) #`(define (#,(make-ctor-name n) x)
(make-token '#,n x)))) (make-token '#,n x))))
(syntax->list (syntax (token ...)))) (syntax->list #'(TOKEN ...)))
#;(define marked-token #f) #;...)))) #;(define marked-token #f) #;...)))]
((_ ...) [(_ ...)
(raise-syntax-error (raise-syntax-error #f
#f
"must have the form (define-tokens name (identifier ...)) or (define-empty-tokens name (identifier ...))" "must have the form (define-tokens name (identifier ...)) or (define-empty-tokens name (identifier ...))"
stx))))) stx)]))
(define-syntax define-tokens (make-define-tokens #f))
(define-syntax define-empty-tokens (make-define-tokens #t))
(define-syntax define-tokens (make-define-tokens #f)) (define-struct position (offset line col) #:inspector #f)
(define-syntax define-empty-tokens (make-define-tokens #t)) (define-struct position-token (token start-pos end-pos) #:inspector #f)
(define-struct position (offset line col) #f) (define-struct srcloc-token (token srcloc) #:inspector #f)
(define-struct position-token (token start-pos end-pos) #f)
(define-struct srcloc-token (token srcloc) #f)
)

50
br-parser-tools-lib/br-parser-tools/private-lex/unicode-chars.rkt
Unescape Escape View File

@ -1,6 +1,5 @@
#lang racket #lang racket/base
(require racket/promise "util.rkt")
(require "util.rkt")
(provide (all-defined-out)) (provide (all-defined-out))
@ -10,36 +9,33 @@
;; get-chars-for-x : (nat -> bool) (listof (list nat nat bool)) -> (listof (cons nat nat)) ;; get-chars-for-x : (nat -> bool) (listof (list nat nat bool)) -> (listof (cons nat nat))
(define (get-chars-for char-x? mapped-chars) (define (get-chars-for char-x? mapped-chars)
(cond (cond
((null? mapped-chars) null) [(null? mapped-chars) null]
(else [else
(let* ((range (car mapped-chars)) (define range (car mapped-chars))
(low (car range)) (define low (car range))
(high (cadr range)) (define high (cadr range))
(x (char-x? low))) (define x (char-x? low))
(cond (cond
((caddr range) [(caddr range)
(if x (if x
(cons (cons low high) (cons (cons low high) (get-chars-for char-x? (cdr mapped-chars)))
(get-chars-for char-x? (cdr mapped-chars))) (get-chars-for char-x? (cdr mapped-chars)))]
(get-chars-for char-x? (cdr mapped-chars)))) [else
(else (let loop ([range-start low]
(let loop ((range-start low) [i (car range)]
(i (car range)) [parity x])
(parity x))
(cond (cond
((> i high) [(> i high)
(if parity (if parity
(cons (cons range-start high) (get-chars-for char-x? (cdr mapped-chars))) (cons (cons range-start high) (get-chars-for char-x? (cdr mapped-chars)))
(get-chars-for char-x? (cdr mapped-chars)))) (get-chars-for char-x? (cdr mapped-chars)))]
((eq? parity (char-x? i)) [(eq? parity (char-x? i))
(loop range-start (add1 i) parity)) (loop range-start (add1 i) parity)]
(parity [parity (cons (cons range-start (sub1 i)) (loop i (add1 i) #f))]
(cons (cons range-start (sub1 i)) (loop i (add1 i) #f))) [else (loop i (add1 i) #t)]))])]))
(else
(loop i (add1 i) #t))))))))))
(define (compute-ranges x?) (define (compute-ranges x?)
(delay (get-chars-for (lambda (x) (x? (integer->char x))) mapped-chars))) (delay (get-chars-for (λ (x) (x? (integer->char x))) mapped-chars)))
(define alphabetic-ranges (compute-ranges char-alphabetic?)) ;; 325 (define alphabetic-ranges (compute-ranges char-alphabetic?)) ;; 325
(define lower-case-ranges (compute-ranges char-lower-case?)) ;; 405 (define lower-case-ranges (compute-ranges char-lower-case?)) ;; 405
@ -61,7 +57,7 @@
(check-equal? (get-chars-for odd? '()) '()) (check-equal? (get-chars-for odd? '()) '())
(check-equal? (get-chars-for odd? '((1 4 #f) (8 13 #f))) (check-equal? (get-chars-for odd? '((1 4 #f) (8 13 #f)))
'((1 . 1) (3 . 3) (9 . 9) (11 . 11) (13 . 13))) '((1 . 1) (3 . 3) (9 . 9) (11 . 11) (13 . 13)))
(check-equal? (get-chars-for (lambda (x) (check-equal? (get-chars-for (λ (x)
(odd? (quotient x 10))) (odd? (quotient x 10)))
'((1 5 #t) (17 19 #t) (21 51 #f))) '((1 5 #t) (17 19 #t) (21 51 #f)))
'((17 . 19) (30 . 39) (50 . 51)))) '((17 . 19) (30 . 39) (50 . 51))))

70
br-parser-tools-lib/br-parser-tools/private-lex/util.rkt
Unescape Escape View File

@ -1,4 +1,5 @@
#lang racket #lang racket/base
(require (for-syntax racket/base))
(provide (all-defined-out)) (provide (all-defined-out))
@ -10,16 +11,16 @@
(module+ test (module+ test
(require rackunit)) (require rackunit))
#;(define-syntax test-block (define-syntax (test-block stx)
(syntax-rules () (syntax-case stx ()
((_ defs (code right-ans) ...) [(_ defs (code right-ans) ...)
#'(module+ test
(require rackunit)
(let* defs (let* defs
(let ((real-ans code)) (let ([real-ans code])
(unless (equal? real-ans right-ans) (check-equal? real-ans right-ans)) ...))]))
(printf "Test failed: ~e gave ~e. Expected ~e\n"
'code real-ans 'right-ans))) ...))))
(define-syntax test-block #;(define-syntax test-block
(syntax-rules () (syntax-rules ()
((_ x ...) (void)))) ((_ x ...) (void))))
@ -31,23 +32,22 @@
;; returned. ;; returned.
;; Xs are compared with equal? ;; Xs are compared with equal?
(define (make-cache) (define (make-cache)
(let ((table (make-hash))) (let ([table (make-hash)])
(lambda (key build) (λ (key build)
(hash-ref table key (hash-ref table key (λ ()
(lambda () (let ([new (build)])
(let ((new (build)))
(hash-set! table key new) (hash-set! table key new)
new)))))) new))))))
(module+ test (module+ test
(define cache (make-cache)) (define cache (make-cache))
(check-equal? (cache '(s 1 2) (lambda () 9)) 9) (check-equal? (cache '(s 1 2) (λ () 9)) 9)
(check-equal? (cache '(s 2 1) (lambda () 8)) 8) (check-equal? (cache '(s 2 1) (λ () 8)) 8)
(check-equal? (cache '(s 1 2) (lambda () 1)) 9) (check-equal? (cache '(s 1 2) (λ () 1)) 9)
(check-equal? (cache (cons 's (cons 0 (cons +inf.0 10))) (check-equal? (cache (cons 's (cons 0 (cons +inf.0 10)))
(lambda () 22)) 22) (λ () 22)) 22)
(check-equal? (cache (cons 's (cons 0 (cons +inf.0 10))) (check-equal? (cache (cons 's (cons 0 (cons +inf.0 10)))
(lambda () 1)) 22)) (λ () 1)) 22))
@ -55,8 +55,8 @@
;; makes a function that returns a higher number by 1, each time ;; makes a function that returns a higher number by 1, each time
;; it is called. ;; it is called.
(define (make-counter) (define (make-counter)
(let ((counter 0)) (let ([counter 0])
(lambda () (λ ()
(begin0 (begin0
counter counter
(set! counter (add1 counter)))))) (set! counter (add1 counter))))))
@ -76,12 +76,12 @@
;; previous entry. l must be grouped by indexes. ;; previous entry. l must be grouped by indexes.
(define (remove-dups l index acc) (define (remove-dups l index acc)
(cond (cond
((null? l) (reverse acc)) [(null? l) (reverse acc)]
((null? acc) (remove-dups (cdr l) index (cons (car l) acc))) [(null? acc) (remove-dups (cdr l) index (cons (car l) acc))]
((= (index (car acc)) (index (car l))) [(= (index (car acc)) (index (car l)))
(remove-dups (cdr l) index acc)) (remove-dups (cdr l) index acc)]
(else [else
(remove-dups (cdr l) index (cons (car l) acc))))) (remove-dups (cdr l) index (cons (car l) acc))]))
(module+ test (module+ test
@ -94,8 +94,8 @@
;; Sorts l according to index and removes the entries with duplicate ;; Sorts l according to index and removes the entries with duplicate
;; indexes. ;; indexes.
(define (do-simple-equiv l index) (define (do-simple-equiv l index)
(let ((ordered (sort l (lambda (a b) (< (index a) (index b)))))) (define ordered (sort l (λ (a b) (< (index a) (index b)))))
(remove-dups ordered index null))) (remove-dups ordered index null))
(module+ test (module+ test
(check-equal? (do-simple-equiv '((2 2) (1 4) (1 2) (check-equal? (do-simple-equiv '((2 2) (1 4) (1 2)
@ -110,16 +110,16 @@
;; list. ;; list.
(define (replace l pred? get acc) (define (replace l pred? get acc)
(cond (cond
((null? l) acc) [(null? l) acc]
((pred? (car l)) (replace (cdr l) pred? get (append (get (car l)) acc))) [(pred? (car l)) (replace (cdr l) pred? get (append (get (car l)) acc))]
(else (replace (cdr l) pred? get (cons (car l) acc))))) [else (replace (cdr l) pred? get (cons (car l) acc))]))
(module+ test (module+ test
(check-equal? (replace null void (lambda () (list 1)) null) null) (check-equal? (replace null void (λ () (list 1)) null) null)
(check-equal? (replace '(1 2 3 4 3 5) (check-equal? (replace '(1 2 3 4 3 5)
(lambda (x) (= x 3)) (λ (x) (= x 3))
(lambda (x) (list 1 2 3)) (λ (x) (list 1 2 3))
null) null)
'(5 1 2 3 4 1 2 3 2 1))) '(5 1 2 3 4 1 2 3 2 1)))

272
br-parser-tools-lib/br-parser-tools/private-yacc/grammar.rkt
Unescape Escape View File

@ -1,41 +1,38 @@
#lang racket/base
;; Constructs to create and access grammars, the internal ;; Constructs to create and access grammars, the internal
;; representation of the input to the parser generator. ;; representation of the input to the parser generator.
(module grammar mzscheme (require racket/class
(except-in racket/list remove-duplicates)
(require mzlib/class
mzlib/list
"yacc-helper.rkt" "yacc-helper.rkt"
racket/contract) racket/contract)
;; Each production has a unique index 0 <= index <= number of productions ;; Each production has a unique index 0 <= index <= number of productions
(define-struct prod (lhs rhs index prec action) (make-inspector)) (define-struct prod (lhs rhs index prec action) #:inspector (make-inspector) #:mutable)
;; The dot-pos field is the index of the element in the rhs
;; of prod that the dot immediately precedes.
;; Thus 0 <= dot-pos <= (vector-length rhs).
(define-struct item (prod dot-pos) (make-inspector))
;; gram-sym = (union term? non-term?)
;; Each term has a unique index 0 <= index < number of terms
;; Each non-term has a unique index 0 <= index < number of non-terms
(define-struct term (sym index prec) (make-inspector))
(define-struct non-term (sym index) (make-inspector))
;; a precedence declaration. ;; The dot-pos field is the index of the element in the rhs
(define-struct prec (num assoc) (make-inspector)) ;; of prod that the dot immediately precedes.
;; Thus 0 <= dot-pos <= (vector-length rhs).
(define-struct item (prod dot-pos) #:inspector (make-inspector))
(provide/contract ;; gram-sym = (union term? non-term?)
(make-item (prod? (or/c #f natural-number/c) . -> . item?)) ;; Each term has a unique index 0 <= index < number of terms
(make-term (symbol? (or/c #f natural-number/c) (or/c prec? #f) . -> . term?)) ;; Each non-term has a unique index 0 <= index < number of non-terms
(make-non-term (symbol? (or/c #f natural-number/c) . -> . non-term?)) (define-struct term (sym index prec) #:inspector (make-inspector) #:mutable)
(make-prec (natural-number/c (or/c 'left 'right 'nonassoc) . -> . prec?)) (define-struct non-term (sym index) #:inspector (make-inspector) #:mutable)
(make-prod (non-term? (vectorof (or/c non-term? term?))
(or/c #f natural-number/c) (or/c #f prec?) syntax? . -> . prod?)))
(provide ;; a precedence declaration.
(define-struct prec (num assoc) #:inspector (make-inspector))
(provide/contract
[make-item (prod? (or/c #f natural-number/c) . -> . item?)]
[make-term (symbol? (or/c #f natural-number/c) (or/c prec? #f) . -> . term?)]
[make-non-term (symbol? (or/c #f natural-number/c) . -> . non-term?)]
[make-prec (natural-number/c (or/c 'left 'right 'nonassoc) . -> . prec?)]
[make-prod (non-term? (vectorof (or/c non-term? term?))
(or/c #f natural-number/c) (or/c #f prec?) syntax? . -> . prod?)])
(provide
;; Things that work on items ;; Things that work on items
start-item? item-prod item->string start-item? item-prod item->string
sym-at-dot move-dot-right item<? item-dot-pos sym-at-dot move-dot-right item<? item-dot-pos
@ -54,97 +51,95 @@
prod-index prod-prec prod-rhs prod-lhs prod-action) prod-index prod-prec prod-rhs prod-lhs prod-action)
;;---------------------- LR items -------------------------- ;;---------------------- LR items --------------------------
;; item<?: LR-item * LR-item -> bool ;; item<?: LR-item * LR-item -> bool
;; Lexicographic comparison on two items. ;; Lexicographic comparison on two items.
(define (item<? i1 i2) (define (item<? i1 i2)
(let ((p1 (prod-index (item-prod i1))) (define p1 (prod-index (item-prod i1)))
(p2 (prod-index (item-prod i2)))) (define p2 (prod-index (item-prod i2)))
(or (< p1 p2) (or (< p1 p2)
(and (= p1 p2) (and (= p1 p2)
(let ((d1 (item-dot-pos i1)) (< (item-dot-pos i1) (item-dot-pos i2)))))
(d2 (item-dot-pos i2)))
(< d1 d2))))))
;; start-item?: LR-item -> bool ;; start-item?: LR-item -> bool
;; The start production always has index 0 ;; The start production always has index 0
(define (start-item? i) (define (start-item? i)
(= 0 (non-term-index (prod-lhs (item-prod i))))) (zero? (non-term-index (prod-lhs (item-prod i)))))
;; move-dot-right: LR-item -> LR-item | #f ;; move-dot-right: LR-item -> LR-item | #f
;; moves the dot to the right in the item, unless it is at its ;; moves the dot to the right in the item, unless it is at its
;; rightmost, then it returns false ;; rightmost, then it returns false
(define (move-dot-right i) (define (move-dot-right i)
(cond (cond
((= (item-dot-pos i) (vector-length (prod-rhs (item-prod i)))) #f) [(= (item-dot-pos i) (vector-length (prod-rhs (item-prod i)))) #f]
(else (make-item (item-prod i) [else (make-item (item-prod i)
(add1 (item-dot-pos i)))))) (add1 (item-dot-pos i)))]))
;; sym-at-dot: LR-item -> gram-sym | #f ;; sym-at-dot: LR-item -> gram-sym | #f
;; returns the symbol after the dot in the item or #f if there is none ;; returns the symbol after the dot in the item or #f if there is none
(define (sym-at-dot i) (define (sym-at-dot i)
(let ((dp (item-dot-pos i)) (define dp (item-dot-pos i))
(rhs (prod-rhs (item-prod i)))) (define rhs (prod-rhs (item-prod i)))
(cond (cond
((= dp (vector-length rhs)) #f) [(= dp (vector-length rhs)) #f]
(else (vector-ref rhs dp))))) [else (vector-ref rhs dp)]))
;; print-item: LR-item -> ;; print-item: LR-item ->
(define (item->string it) (define (item->string it)
(let ((print-sym (lambda (i) (define print-sym (λ (i)
(let ((gs (vector-ref (prod-rhs (item-prod it)) i))) (let ((gs (vector-ref (prod-rhs (item-prod it)) i)))
(cond (cond
((term? gs) (format "~a " (term-sym gs))) ((term? gs) (format "~a " (term-sym gs)))
(else (format "~a " (non-term-sym gs)))))))) (else (format "~a " (non-term-sym gs)))))))
(string-append (string-append
(format "~a -> " (non-term-sym (prod-lhs (item-prod it)))) (format "~a -> " (non-term-sym (prod-lhs (item-prod it))))
(let loop ((i 0)) (let loop ((i 0))
(cond (cond
((= i (vector-length (prod-rhs (item-prod it)))) [(= i (vector-length (prod-rhs (item-prod it))))
(if (= i (item-dot-pos it)) (if (= i (item-dot-pos it))
". " ". "
"")) "")]
((= i (item-dot-pos it)) [(= i (item-dot-pos it))
(string-append ". " (print-sym i) (loop (add1 i)))) (string-append ". " (print-sym i) (loop (add1 i)))]
(else (string-append (print-sym i) (loop (add1 i))))))))) [else (string-append (print-sym i) (loop (add1 i)))]))))
;; --------------------- Grammar Symbols -------------------------- ;; --------------------- Grammar Symbols --------------------------
(define (non-term<? nt1 nt2) (define (non-term<? nt1 nt2)
(< (non-term-index nt1) (non-term-index nt2))) (< (non-term-index nt1) (non-term-index nt2)))
(define (term<? nt1 nt2) (define (term<? nt1 nt2)
(< (term-index nt1) (term-index nt2))) (< (term-index nt1) (term-index nt2)))
(define (gram-sym-index gs) (define (gram-sym-index gs)
(cond (if (term? gs)
((term? gs) (term-index gs)) (term-index gs)
(else (non-term-index gs)))) (non-term-index gs)))
(define (gram-sym-symbol gs) (define (gram-sym-symbol gs)
(cond (if (term? gs)
((term? gs) (term-sym gs)) (term-sym gs)
(else (non-term-sym gs)))) (non-term-sym gs)))
(define (gram-sym->string gs) (define (gram-sym->string gs)
(symbol->string (gram-sym-symbol gs))) (symbol->string (gram-sym-symbol gs)))
;; term-list->bit-vector: term list -> int ;; term-list->bit-vector: term list -> int
;; Creates a number where the nth bit is 1 if the term with index n is in ;; Creates a number where the nth bit is 1 if the term with index n is in
;; the list, and whose nth bit is 0 otherwise ;; the list, and whose nth bit is 0 otherwise
(define (term-list->bit-vector terms) (define (term-list->bit-vector terms)
(cond (if (null? terms)
((null? terms) 0) 0
(else (bitwise-ior (arithmetic-shift 1 (term-index (car terms)))
(bitwise-ior (arithmetic-shift 1 (term-index (car terms))) (term-list->bit-vector (cdr terms)))))) (term-list->bit-vector (cdr terms)))))
;; ------------------------- Grammar ------------------------------ ;; ------------------------- Grammar ------------------------------
(define grammar% (define grammar%
(class object% (class object%
(super-instantiate ()) (super-instantiate ())
;; prods: production list list ;; prods: production list list
@ -161,33 +156,20 @@
(define num-terms (length terms)) (define num-terms (length terms))
(define num-non-terms (length non-terms)) (define num-non-terms (length non-terms))
(let ((count 0)) (for ([(nt count) (in-indexed non-terms)])
(for-each (set-non-term-index! nt count))
(lambda (nt)
(set-non-term-index! nt count) (for ([(t count) (in-indexed terms)])
(set! count (add1 count))) (set-term-index! t count))
non-terms))
(for ([(prod count) (in-indexed all-prods)])
(let ((count 0)) (set-prod-index! prod count))
(for-each
(lambda (t)
(set-term-index! t count)
(set! count (add1 count)))
terms))
(let ((count 0))
(for-each
(lambda (prod)
(set-prod-index! prod count)
(set! count (add1 count)))
all-prods))
;; indexed by the index of the non-term - contains the list of productions for that non-term ;; indexed by the index of the non-term - contains the list of productions for that non-term
(define nt->prods (define nt->prods
(let ((v (make-vector (length prods) #f))) (let ((v (make-vector (length prods) #f)))
(for-each (lambda (prods) (for ([prods (in-list prods)])
(vector-set! v (non-term-index (prod-lhs (car prods))) prods)) (vector-set! v (non-term-index (prod-lhs (car prods))) prods))
prods)
v)) v))
(define nullable-non-terms (define nullable-non-terms
@ -211,70 +193,58 @@
(vector-ref nullable-non-terms (non-term-index nt))) (vector-ref nullable-non-terms (non-term-index nt)))
(define/public (nullable-after-dot? item) (define/public (nullable-after-dot? item)
(let* ((rhs (prod-rhs (item-prod item))) (define rhs (prod-rhs (item-prod item)))
(prod-length (vector-length rhs))) (define prod-length (vector-length rhs))
(let loop ((i (item-dot-pos item))) (let loop ((i (item-dot-pos item)))
(cond (cond
((< i prod-length) [(< i prod-length)
(if (and (non-term? (vector-ref rhs i)) (nullable-non-term? (vector-ref rhs i))) (and (non-term? (vector-ref rhs i))
(loop (add1 i)) (nullable-non-term? (vector-ref rhs i))
#f)) (loop (add1 i)))]
((= i prod-length) #t))))) [(= i prod-length)])))
(define/public (nullable-non-term-thunk) (define/public (nullable-non-term-thunk)
(lambda (nt) (λ (nt) (nullable-non-term? nt)))
(nullable-non-term? nt)))
(define/public (nullable-after-dot?-thunk) (define/public (nullable-after-dot?-thunk)
(lambda (item) (λ (item) (nullable-after-dot? item)))))
(nullable-after-dot? item)))))
;; nullable: production list * int -> non-term set ;; nullable: production list * int -> non-term set
;; determines which non-terminals can derive epsilon ;; determines which non-terminals can derive epsilon
(define (nullable prods num-nts) (define (nullable prods num-nts)
(letrec ((nullable (make-vector num-nts #f)) (define nullable (make-vector num-nts #f))
(added #f) (define added #f)
;; possible-nullable: producion list -> production list ;; possible-nullable: producion list -> production list
;; Removes all productions that have a terminal ;; Removes all productions that have a terminal
(possible-nullable (define (possible-nullable prods)
(lambda (prods) (for/list ([prod (in-list prods)]
(filter (lambda (prod) #:when (vector-andmap non-term? (prod-rhs prod)))
(vector-andmap non-term? (prod-rhs prod))) prod))
prods)))
;; set-nullables: production list -> production list ;; set-nullables: production list -> production list
;; makes one pass through the productions, adding the ones ;; makes one pass through the productions, adding the ones
;; known to be nullable now to nullable and returning a list ;; known to be nullable now to nullable and returning a list
;; of productions that we don't know about yet. ;; of productions that we don't know about yet.
(set-nullables (define (set-nullables prods)
(lambda (prods)
(cond (cond
((null? prods) null) [(null? prods) null]
((vector-ref nullable [(vector-ref nullable (gram-sym-index (prod-lhs (car prods))))
(gram-sym-index (prod-lhs (car prods)))) (set-nullables (cdr prods))]
(set-nullables (cdr prods))) [(vector-andmap (λ (nt) (vector-ref nullable (gram-sym-index nt))) (prod-rhs (car prods)))
((vector-andmap (lambda (nt) (vector-set! nullable (gram-sym-index (prod-lhs (car prods))) #t)
(vector-ref nullable (gram-sym-index nt)))
(prod-rhs (car prods)))
(vector-set! nullable
(gram-sym-index (prod-lhs (car prods)))
#t)
(set! added #t) (set! added #t)
(set-nullables (cdr prods))) (set-nullables (cdr prods))]
(else [else (cons (car prods) (set-nullables (cdr prods)))]))
(cons (car prods)
(set-nullables (cdr prods))))))))
(let loop ((P (possible-nullable prods))) (let loop ((P (possible-nullable prods)))
(cond (cond
((null? P) nullable) [(null? P) nullable]
(else [else
(set! added #f) (set! added #f)
(let ((new-P (set-nullables P))) (define new-P (set-nullables P))
(if added (if added
(loop new-P) (loop new-P)
nullable))))))) nullable)])))
)

82
br-parser-tools-lib/br-parser-tools/private-yacc/graph.rkt
Unescape Escape View File

@ -1,61 +1,53 @@
(module graph mzscheme #lang racket/base
(provide digraph)
(provide digraph) (define (zero-thunk) 0)
(define (zero-thunk) 0) ;; digraph:
;; ('a list) * ('a -> 'a list) * ('a -> 'b) * ('b * 'b -> 'b) * (-> 'b)
;; digraph: ;; -> ('a -> 'b)
;; ('a list) * ('a -> 'a list) * ('a -> 'b) * ('b * 'b -> 'b) * (-> 'b) ;; DeRemer and Pennello 1982
;; -> ('a -> 'b) ;; Computes (f x) = (f- x) union Union{(f y) | y in (edges x)}
;; DeRemer and Pennello 1982 ;; We use a hash-table to represent the result function 'a -> 'b set, so
;; Computes (f x) = (f- x) union Union{(f y) | y in (edges x)} ;; the values of type 'a must be comparable with eq?.
;; We use a hash-table to represent the result function 'a -> 'b set, so
;; the values of type 'a must be comparable with eq?.
(define (digraph nodes edges f- union fail)
(letrec [
;; Will map elements of 'a to 'b sets
(results (make-hash-table))
(f (lambda (x) (hash-table-get results x fail)))
(define (digraph nodes edges f- union fail)
(define results (make-hasheq))
(define (f x) (hash-ref results x fail))
;; Maps elements of 'a to integers. ;; Maps elements of 'a to integers.
(N (make-hash-table)) (define N (make-hasheq))
(get-N (lambda (x) (hash-table-get N x zero-thunk))) (define (get-N x) (hash-ref N x zero-thunk))
(set-N (lambda (x d) (hash-table-put! N x d))) (define (set-N x d) (hash-set! N x d))
(define stack null)
(stack null) (define (push x) (set! stack (cons x stack)))
(push (lambda (x) (define (pop) (begin0
(set! stack (cons x stack))))
(pop (lambda ()
(begin0
(car stack) (car stack)
(set! stack (cdr stack))))) (set! stack (cdr stack))))
(depth (lambda () (length stack))) (define (depth) (length stack))
;; traverse: 'a -> ;; traverse: 'a ->
(traverse (define (traverse x)
(lambda (x)
(push x) (push x)
(let ((d (depth))) (define d (depth))
(set-N x d) (set-N x d)
(hash-table-put! results x (f- x)) (hash-set! results x (f- x))
(for-each (lambda (y) (for-each (λ (y)
(if (= 0 (get-N y)) (when (= 0 (get-N y))
(traverse y)) (traverse y))
(hash-table-put! results (hash-set! results
x x
(union (f x) (f y))) (union (f x) (f y)))
(set-N x (min (get-N x) (get-N y)))) (set-N x (min (get-N x) (get-N y))))
(edges x)) (edges x))
(if (= d (get-N x)) (when (= d (get-N x))
(let loop ((p (pop))) (let loop ([p (pop)])
(set-N p +inf.0) (set-N p +inf.0)
(hash-table-put! results p (f x)) (hash-set! results p (f x))
(if (not (eq? x p)) (when (not (eq? x p))
(loop (pop))))))))] (loop (pop))))))
(for-each (lambda (x) ;; Will map elements of 'a to 'b sets
(if (= 0 (get-N x)) (for ([x (in-list nodes)]
(traverse x))) #:when (zero? (get-N x)))
nodes) (traverse x))
f)) f)
)

445
br-parser-tools-lib/br-parser-tools/private-yacc/input-file-parser.rkt
Unescape Escape View File

@ -1,374 +1,297 @@
(module input-file-parser mzscheme #lang racket/base
(require "yacc-helper.rkt"
;; routines for parsing the input to the parser generator and producing a
;; grammar (See grammar.rkt)
(require "yacc-helper.rkt"
"../private-lex/token-syntax.rkt" "../private-lex/token-syntax.rkt"
"grammar.rkt" "grammar.rkt"
mzlib/class racket/class
racket/contract) racket/contract
(require-for-template mzscheme) (for-template racket/base))
;; routines for parsing the input to the parser generator and producing a
;; grammar (See grammar.rkt)
(define (is-a-grammar%? x) (is-a? x grammar%)) (define (is-a-grammar%? x) (is-a? x grammar%))
(provide/contract (provide/contract
(parse-input ((listof identifier?) (listof identifier?) (listof identifier?) [parse-input ((listof identifier?) (listof identifier?) (listof identifier?)
(or/c #f syntax?) syntax? any/c . -> . is-a-grammar%?)) (or/c #f syntax?) syntax? any/c . -> . is-a-grammar%?)]
(get-term-list ((listof identifier?) . -> . (listof identifier?)))) [get-term-list ((listof identifier?) . -> . (listof identifier?))])
(define stx-for-original-property (read-syntax #f (open-input-string "original"))) (define stx-for-original-property (read-syntax #f (open-input-string "original")))
;; get-args: ??? -> (values (listof syntax) (or/c #f (cons integer? stx))) ;; get-args: ??? -> (values (listof syntax) (or/c #f (cons integer? stx)))
(define (get-args i rhs src-pos term-defs) (define (get-args i rhs src-pos term-defs)
(let ((empty-table (make-hash-table)) (define empty-table (make-hasheq))
(biggest-pos #f)) (define biggest-pos #f)
(hash-table-put! empty-table 'error #t) (hash-set! empty-table 'error #t)
(for-each (lambda (td) (for* ([td (in-list term-defs)]
(let ((v (syntax-local-value td))) [v (in-value (syntax-local-value td))]
(if (e-terminals-def? v) #:when (e-terminals-def? v)
(for-each (lambda (s) [s (in-list (syntax->list (e-terminals-def-t v)))])
(hash-table-put! empty-table (syntax-object->datum s) #t)) (hash-set! empty-table (syntax->datum s) #t))
(syntax->list (e-terminals-def-t v)))))) (define args
term-defs) (let get-args ([i i][rhs rhs])
(let ([args
(let get-args ((i i)
(rhs rhs))
(cond (cond
((null? rhs) null) [(null? rhs) null]
(else [else
(let ((b (car rhs)) (define b (car rhs))
(name (if (hash-table-get empty-table (syntax-object->datum (car rhs)) (lambda () #f)) (define name (if (hash-ref empty-table (syntax->datum (car rhs)) #f)
(gensym) (gensym)
(string->symbol (format "$~a" i))))) (string->symbol (format "$~a" i))))
(cond (cond
(src-pos [src-pos
(let ([start-pos-id (define start-pos-id
(datum->syntax-object b (string->symbol (format "$~a-start-pos" i)) b stx-for-original-property)] (datum->syntax b (string->symbol (format "$~a-start-pos" i)) b stx-for-original-property))
[end-pos-id (define end-pos-id
(datum->syntax-object b (string->symbol (format "$~a-end-pos" i)) b stx-for-original-property)]) (datum->syntax b (string->symbol (format "$~a-end-pos" i)) b stx-for-original-property))
(set! biggest-pos (cons start-pos-id end-pos-id)) (set! biggest-pos (cons start-pos-id end-pos-id))
`(,(datum->syntax-object b name b stx-for-original-property) (list* (datum->syntax b name b stx-for-original-property)
,start-pos-id start-pos-id
,end-pos-id end-pos-id
,@(get-args (add1 i) (cdr rhs))))) (get-args (add1 i) (cdr rhs)))]
(else [else
`(,(datum->syntax-object b name b stx-for-original-property) (list* (datum->syntax b name b stx-for-original-property)
,@(get-args (add1 i) (cdr rhs)))))))))]) (get-args (add1 i) (cdr rhs)))])])))
(values args biggest-pos)))) (values args biggest-pos))
;; Given the list of terminal symbols and the precedence/associativity definitions,
;; builds terminal structures (See grammar.rkt)
;; build-terms: symbol list * symbol list list -> term list
(define (build-terms term-list precs)
(let ((counter 0)
;; Given the list of terminal symbols and the precedence/associativity definitions,
;; builds terminal structures (See grammar.rkt)
;; build-terms: symbol list * symbol list list -> term list
(define (build-terms term-list precs)
(define counter 0)
;;(term-list (cons (gensym) term-list)) ;;(term-list (cons (gensym) term-list))
;; Will map a terminal symbol to its precedence/associativity ;; Will map a terminal symbol to its precedence/associativity
(prec-table (make-hash-table))) (define prec-table (make-hasheq))
;; Fill the prec table ;; Fill the prec table
(for-each (for ([p-decl (in-list precs)])
(lambda (p-decl) (define assoc (car p-decl))
(begin0 (for ([term-sym (in-list (cdr p-decl))])
(let ((assoc (car p-decl))) (hash-set! prec-table term-sym (make-prec counter assoc)))
(for-each (set! counter (add1 counter)))
(lambda (term-sym)
(hash-table-put! prec-table term-sym (make-prec counter assoc)))
(cdr p-decl)))
(set! counter (add1 counter))))
precs)
;; Build the terminal structures ;; Build the terminal structures
(map (for/list ([term-sym (in-list term-list)])
(lambda (term-sym)
(make-term term-sym (make-term term-sym
#f #f
(hash-table-get prec-table term-sym (lambda () #f)))) (hash-ref prec-table term-sym (λ () #f)))))
term-list)))
;; Retrieves the terminal symbols from a terminals-def (See terminal-syntax.rkt) ;; Retrieves the terminal symbols from a terminals-def (See terminal-syntax.rkt)
;; get-terms-from-def: identifier? -> (listof identifier?) ;; get-terms-from-def: identifier? -> (listof identifier?)
(define (get-terms-from-def term-syn) (define (get-terms-from-def term-syn)
(let ((t (syntax-local-value term-syn (lambda () #f)))) (define t (syntax-local-value term-syn #f))
(cond (cond
((terminals-def? t) (syntax->list (terminals-def-t t))) [(terminals-def? t) (syntax->list (terminals-def-t t))]
((e-terminals-def? t) (syntax->list (e-terminals-def-t t))) [(e-terminals-def? t) (syntax->list (e-terminals-def-t t))]
(else [else
(raise-syntax-error (raise-syntax-error
'parser-tokens 'parser-tokens
"undefined token group" "undefined token group"
term-syn))))) term-syn)]))
(define (get-term-list term-group-names) (define (get-term-list term-group-names)
(remove-duplicates (remove-duplicates
(cons (datum->syntax-object #f 'error) (cons (datum->syntax #f 'error)
(apply append (apply append (map get-terms-from-def term-group-names)))))
(map get-terms-from-def term-group-names)))))
(define (parse-input term-defs start ends prec-decls prods src-pos)
(let* ((start-syms (map syntax-e start))
(list-of-terms (map syntax-e (get-term-list term-defs)))
(end-terms (define (parse-input term-defs start ends prec-decls prods src-pos)
(map (define start-syms (map syntax-e start))
(lambda (end) (define list-of-terms (map syntax-e (get-term-list term-defs)))
(define end-terms
(for/list ([end (in-list ends)])
(unless (memq (syntax-e end) list-of-terms) (unless (memq (syntax-e end) list-of-terms)
(raise-syntax-error (raise-syntax-error
'parser-end-tokens 'parser-end-tokens
(format "End token ~a not defined as a token" (format "End token ~a not defined as a token"
(syntax-e end)) (syntax-e end))
end)) end))
(syntax-e end)) (syntax-e end)))
ends))
;; Get the list of terminals out of input-terms ;; Get the list of terminals out of input-terms
(define list-of-non-terms
(list-of-non-terms
(syntax-case prods () (syntax-case prods ()
(((non-term production ...) ...) [((NON-TERM PRODUCTION ...) ...)
(begin (begin
(for-each (for ([nts (in-list (syntax->list #'(NON-TERM ...)))]
(lambda (nts) #:when (memq (syntax->datum nts) list-of-terms))
(if (memq (syntax-object->datum nts) list-of-terms)
(raise-syntax-error (raise-syntax-error
'parser-non-terminals 'parser-non-terminals
(format "~a used as both token and non-terminal" (format "~a used as both token and non-terminal" (syntax->datum nts))
(syntax-object->datum nts)) nts))
nts))) (let ([dup (duplicate-list? (syntax->datum #'(NON-TERM ...)))])
(syntax->list (syntax (non-term ...)))) (when dup
(let ((dup (duplicate-list? (syntax-object->datum
(syntax (non-term ...))))))
(if dup
(raise-syntax-error (raise-syntax-error
'parser-non-terminals 'parser-non-terminals
(format "non-terminal ~a defined multiple times" (format "non-terminal ~a defined multiple times" dup)
dup)
prods))) prods)))
(syntax->datum #'(NON-TERM ...)))]
(syntax-object->datum (syntax (non-term ...))))) [_ (raise-syntax-error
(_
(raise-syntax-error
'parser-grammar 'parser-grammar
"Grammar must be of the form (grammar (non-terminal productions ...) ...)" "Grammar must be of the form (grammar (non-terminal productions ...) ...)"
prods)))) prods)]))
;; Check the precedence declarations for errors and turn them into data ;; Check the precedence declarations for errors and turn them into data
(precs (define precs
(syntax-case prec-decls () (syntax-case prec-decls ()
(((type term ...) ...) [((TYPE TERM ...) ...)
(let ((p-terms (let ([p-terms (syntax->datum #'(TERM ... ...))])
(syntax-object->datum (syntax (term ... ...)))))
(cond (cond
((duplicate-list? p-terms) => [(duplicate-list? p-terms) =>
(lambda (d) (λ (d)
(raise-syntax-error (raise-syntax-error
'parser-precedences 'parser-precedences
(format "duplicate precedence declaration for token ~a" (format "duplicate precedence declaration for token ~a" d)
d) prec-decls))]
prec-decls))) [else (for ([t (in-list (syntax->list #'(TERM ... ...)))]
(else #:when (not (memq (syntax->datum t) list-of-terms)))
(for-each
(lambda (a)
(for-each
(lambda (t)
(if (not (memq (syntax-object->datum t)
list-of-terms))
(raise-syntax-error (raise-syntax-error
'parser-precedences 'parser-precedences
(format (format "Precedence declared for non-token ~a" (syntax->datum t))
"Precedence declared for non-token ~a" t))
(syntax-object->datum t)) (for ([type (in-list (syntax->list #'(TYPE ...)))]
t))) #:unless (memq (syntax->datum type) `(left right nonassoc)))
(syntax->list a)))
(syntax->list (syntax ((term ...) ...))))
(for-each
(lambda (type)
(if (not (memq (syntax-object->datum type)
`(left right nonassoc)))
(raise-syntax-error (raise-syntax-error
'parser-precedences 'parser-precedences
"Associativity must be left, right or nonassoc" "Associativity must be left, right or nonassoc"
type))) type))
(syntax->list (syntax (type ...)))) (syntax->datum prec-decls)]))]
(syntax-object->datum prec-decls))))) [#f null]
(#f null) [_ (raise-syntax-error
(_
(raise-syntax-error
'parser-precedences 'parser-precedences
"Precedence declaration must be of the form (precs (assoc term ...) ...) where assoc is left, right or nonassoc" "Precedence declaration must be of the form (precs (assoc term ...) ...) where assoc is left, right or nonassoc"
prec-decls)))) prec-decls)]))
(terms (build-terms list-of-terms precs)) (define terms (build-terms list-of-terms precs))
(define non-terms (map (λ (non-term) (make-non-term non-term #f))
(non-terms (map (lambda (non-term) (make-non-term non-term #f))
list-of-non-terms)) list-of-non-terms))
(term-table (make-hash-table)) (define term-table (make-hasheq))
(non-term-table (make-hash-table))) (define non-term-table (make-hasheq))
(for-each (lambda (t) (for ([t (in-list terms)])
(hash-table-put! term-table (gram-sym-symbol t) t)) (hash-set! term-table (gram-sym-symbol t) t))
terms)
(for-each (lambda (nt) (for ([nt (in-list non-terms)])
(hash-table-put! non-term-table (gram-sym-symbol nt) nt)) (hash-set! non-term-table (gram-sym-symbol nt) nt))
non-terms)
(let* (
;; parse-prod: syntax-object -> gram-sym vector ;; parse-prod: syntax-object -> gram-sym vector
(parse-prod (define (parse-prod prod-so)
(lambda (prod-so)
(syntax-case prod-so () (syntax-case prod-so ()
((prod-rhs-sym ...) [(PROD-RHS-SYM ...)
(andmap identifier? (syntax->list prod-so)) (andmap identifier? (syntax->list prod-so))
(begin (begin
(for-each (lambda (t) (for ([t (in-list (syntax->list prod-so))]
(if (memq (syntax-object->datum t) end-terms) #:when (memq (syntax->datum t) end-terms))
(raise-syntax-error (raise-syntax-error
'parser-production-rhs 'parser-production-rhs
(format "~a is an end token and cannot be used in a production" (format "~a is an end token and cannot be used in a production" (syntax->datum t))
(syntax-object->datum t)) t))
t))) (for/vector ([s (in-list (syntax->list prod-so))])
(syntax->list prod-so)) (cond
(list->vector [(hash-ref term-table (syntax->datum s) #f)]
(map (lambda (s) [(hash-ref non-term-table (syntax->datum s) #f)]
(hash-table-get [else (raise-syntax-error
term-table
(syntax-object->datum s)
(lambda ()
(hash-table-get
non-term-table
(syntax-object->datum s)
(lambda ()
(raise-syntax-error
'parser-production-rhs 'parser-production-rhs
(format (format "~a is not declared as a terminal or non-terminal" (syntax->datum s))
"~a is not declared as a terminal or non-terminal" s)])))]
(syntax-object->datum s)) [_ (raise-syntax-error
s))))))
(syntax->list prod-so)))))
(_
(raise-syntax-error
'parser-production-rhs 'parser-production-rhs
"production right-hand-side must have form (symbol ...)" "production right-hand-side must have form (symbol ...)"
prod-so))))) prod-so)]))
;; parse-action: syntax-object * syntax-object -> syntax-object ;; parse-action: syntax-object * syntax-object -> syntax-object
(parse-action (define (parse-action rhs act-in)
(lambda (rhs act) (define-values (args biggest) (get-args 1 (syntax->list rhs) src-pos term-defs))
(let-values ([(args biggest) (get-args 1 (syntax->list rhs) src-pos term-defs)]) (define act
(let ([act
(if biggest (if biggest
(with-syntax ([$n-start-pos (datum->syntax-object (car biggest) '$n-start-pos)] (with-syntax ([(CAR-BIGGEST . CDR-BIGGEST) biggest]
[$n-end-pos (datum->syntax-object (cdr biggest) '$n-end-pos)]) [$N-START-POS (datum->syntax (car biggest) '$n-start-pos)]
#`(let ([$n-start-pos #,(car biggest)] [$N-END-POS (datum->syntax (cdr biggest) '$n-end-pos)]
[$n-end-pos #,(cdr biggest)]) [ACT-IN act-in])
#,act)) #'(let ([$N-START-POS CAR-BIGGEST]
act)]) [$N-END-POS CDR-BIGGEST])
(quasisyntax/loc act ACT-IN))
(lambda #,args act-in))
#,act)))))) (with-syntax ([ARGS args][ACT act])
(syntax/loc #'ACT (λ ARGS ACT))))
;; parse-prod+action: non-term * syntax-object -> production ;; parse-prod+action: non-term * syntax-object -> production
(parse-prod+action (define (parse-prod+action nt prod-so)
(lambda (nt prod-so)
(syntax-case prod-so () (syntax-case prod-so ()
((prod-rhs action) [(PROD-RHS ACTION)
(let ((p (parse-prod (syntax prod-rhs)))) (let ([p (parse-prod #'PROD-RHS)])
(make-prod (make-prod
nt nt
p p
#f #f
(let loop ((i (sub1 (vector-length p)))) (let loop ([i (sub1 (vector-length p))])
(if (>= i 0) (if (>= i 0)
(let ((gs (vector-ref p i))) (let ([gs (vector-ref p i)])
(if (term? gs) (if (term? gs)
(term-prec gs) (term-prec gs)
(loop (sub1 i)))) (loop (sub1 i))))
#f)) #f))
(parse-action (syntax prod-rhs) (syntax action))))) (parse-action #'PROD-RHS #'ACTION)))]
((prod-rhs (prec term) action) [(PROD-RHS (PREC TERM) ACTION)
(identifier? (syntax term)) (identifier? #'TERM)
(let ((p (parse-prod (syntax prod-rhs)))) (let ([p (parse-prod #'PROD-RHS)])
(make-prod (make-prod
nt nt
p p
#f #f
(term-prec (term-prec
(hash-table-get (cond
term-table [(hash-ref term-table (syntax->datum #'TERM) #f)]
(syntax-object->datum (syntax term)) [else (raise-syntax-error
(lambda ()
(raise-syntax-error
'parser-production-rhs 'parser-production-rhs
(format (format
"unrecognized terminal ~a in precedence declaration" "unrecognized terminal ~a in precedence declaration"
(syntax-object->datum (syntax term))) (syntax->datum #'TERM))
(syntax term))))) #'TERM)]))
(parse-action (syntax prod-rhs) (syntax action))))) (parse-action #'PROD-RHS #'ACTION)))]
(_ [_ (raise-syntax-error
(raise-syntax-error
'parser-production-rhs 'parser-production-rhs
"production must have form [(symbol ...) expression] or [(symbol ...) (prec symbol) expression]" "production must have form [(symbol ...) expression] or [(symbol ...) (prec symbol) expression]"
prod-so))))) prod-so)]))
;; parse-prod-for-nt: syntax-object -> production list ;; parse-prod-for-nt: syntax-object -> production list
(parse-prods-for-nt (define (parse-prods-for-nt prods-so)
(lambda (prods-so)
(syntax-case prods-so () (syntax-case prods-so ()
((nt productions ...) [(NT PRODUCTIONS ...)
(> (length (syntax->list (syntax (productions ...)))) 0) (positive? (length (syntax->list #'(PRODUCTIONS ...))))
(let ((nt (hash-table-get non-term-table (let ([nt (hash-ref non-term-table (syntax->datum #'NT))])
(syntax-object->datum (syntax nt))))) (map (λ (p) (parse-prod+action nt p)) (syntax->list #'(PRODUCTIONS ...))))]
(map (lambda (p) (parse-prod+action nt p)) [_ (raise-syntax-error
(syntax->list (syntax (productions ...))))))
(_
(raise-syntax-error
'parser-productions 'parser-productions
"A production for a non-terminal must be (non-term right-hand-side ...) with at least 1 right hand side" "A production for a non-terminal must be (non-term right-hand-side ...) with at least 1 right hand side"
prods-so)))))) prods-so)]))
(for-each (for ([sstx (in-list start)]
(lambda (sstx ssym) [ssym (in-list start-syms)]
(unless (memq ssym list-of-non-terms) #:unless (memq ssym list-of-non-terms))
(raise-syntax-error (raise-syntax-error
'parser-start 'parser-start
(format "Start symbol ~a not defined as a non-terminal" ssym) (format "Start symbol ~a not defined as a non-terminal" ssym)
sstx))) sstx))
start start-syms)
(let* ((starts (map (lambda (x) (make-non-term (gensym) #f)) start-syms)) (define starts (map (λ (x) (make-non-term (gensym) #f)) start-syms))
(end-non-terms (map (lambda (x) (make-non-term (gensym) #f)) start-syms)) (define end-non-terms (map (λ (x) (make-non-term (gensym) #f)) start-syms))
(parsed-prods (map parse-prods-for-nt (syntax->list prods))) (define parsed-prods (map parse-prods-for-nt (syntax->list prods)))
(start-prods (define start-prods (for/list ([start (in-list starts)]
(map (lambda (start end-non-term) [end-non-term (in-list end-non-terms)])
(list (make-prod start (vector end-non-term) #f #f (list (make-prod start (vector end-non-term) #f #f #'values))))
(syntax (lambda (x) x))))) (define new-prods
starts end-non-terms)) (append start-prods
(prods (for/list ([end-nt (in-list end-non-terms)]
`(,@start-prods [start-sym (in-list start-syms)])
,@(map (for/list ([end (in-list end-terms)])
(lambda (end-nt start-sym)
(map
(lambda (end)
(make-prod end-nt (make-prod end-nt
(vector (vector
(hash-table-get non-term-table start-sym) (hash-ref non-term-table start-sym)
(hash-table-get term-table end)) (hash-ref term-table end))
#f #f
#f #f
(syntax (lambda (x) x)))) #'values)))
end-terms)) parsed-prods))
end-non-terms start-syms)
,@parsed-prods)))
(make-object grammar% (make-object grammar%
prods new-prods
(map car start-prods) (map car start-prods)
terms terms
(append starts (append end-non-terms non-terms)) (append starts (append end-non-terms non-terms))
(map (lambda (term-name) (map (λ (term-name) (hash-ref term-table term-name)) end-terms)))
(hash-table-get term-table term-name))
end-terms)))))))

351
br-parser-tools-lib/br-parser-tools/private-yacc/lalr.rkt
Unescape Escape View File

@ -1,160 +1,150 @@
(module lalr mzscheme #lang racket/base
(require "lr0.rkt"
;; Compute LALR lookaheads from DeRemer and Pennello 1982
(require "lr0.rkt"
"grammar.rkt" "grammar.rkt"
mzlib/list racket/list
mzlib/class) racket/class)
(provide compute-LA) ;; Compute LALR lookaheads from DeRemer and Pennello 1982
;; compute-DR: LR0-automaton * grammar -> (trans-key -> term set) (provide compute-LA)
;; computes for each state, non-term transition pair, the terminals
;; which can transition out of the resulting state
;; output term set is represented in bit-vector form
(define (compute-DR a g)
(lambda (tk)
(let ((r (send a run-automaton (trans-key-st tk) (trans-key-gs tk))))
(term-list->bit-vector
(filter
(lambda (term)
(send a run-automaton r term))
(send g get-terms))))))
;; compute-reads:
;; LR0-automaton * grammar -> (trans-key -> trans-key list)
(define (compute-reads a g)
(let ((nullable-non-terms
(filter (lambda (nt) (send g nullable-non-term? nt))
(send g get-non-terms))))
(lambda (tk)
(let ((r (send a run-automaton (trans-key-st tk) (trans-key-gs tk))))
(map (lambda (x) (make-trans-key r x))
(filter (lambda (non-term) (send a run-automaton r non-term))
nullable-non-terms))))))
;; compute-read: LR0-automaton * grammar -> (trans-key -> term set) ;; compute-DR: LR0-automaton * grammar -> (trans-key -> term set)
;; output term set is represented in bit-vector form ;; computes for each state, non-term transition pair, the terminals
(define (compute-read a g) ;; which can transition out of the resulting state
(let* ((dr (compute-DR a g)) ;; output term set is represented in bit-vector form
(reads (compute-reads a g))) (define ((compute-DR a g) tk)
(define r (send a run-automaton (trans-key-st tk) (trans-key-gs tk)))
(term-list->bit-vector
(filter (λ (term) (send a run-automaton r term)) (send g get-terms))))
;; compute-reads:
;; LR0-automaton * grammar -> (trans-key -> trans-key list)
(define (compute-reads a g)
(define nullable-non-terms (filter (λ (nt) (send g nullable-non-term? nt)) (send g get-non-terms)))
(λ (tk)
(define r (send a run-automaton (trans-key-st tk) (trans-key-gs tk)))
(for/list ([non-term (in-list nullable-non-terms)]
#:when (send a run-automaton r non-term))
(make-trans-key r non-term))))
;; compute-read: LR0-automaton * grammar -> (trans-key -> term set)
;; output term set is represented in bit-vector form
(define (compute-read a g)
(define dr (compute-DR a g))
(define reads (compute-reads a g))
(digraph-tk->terml (send a get-mapped-non-term-keys) (digraph-tk->terml (send a get-mapped-non-term-keys)
reads reads
dr dr
(send a get-num-states)))) (send a get-num-states)))
;; returns the list of all k such that state k transitions to state start on the ;; returns the list of all k such that state k transitions to state start on the
;; transitions in rhs (in order) ;; transitions in rhs (in order)
(define (run-lr0-backward a rhs dot-pos start num-states) (define (run-lr0-backward a rhs dot-pos start num-states)
(let loop ((states (list start)) (let loop ([states (list start)]
(i (sub1 dot-pos))) [i (sub1 dot-pos)])
(cond (cond
((< i 0) states) [(< i 0) states]
(else (loop (send a run-automaton-back states (vector-ref rhs i)) [else (loop (send a run-automaton-back states (vector-ref rhs i))
(sub1 i)))))) (sub1 i))])))
;; prod->items-for-include: grammar * prod * non-term -> lr0-item list ;; prod->items-for-include: grammar * prod * non-term -> lr0-item list
;; returns the list of all (B -> beta . nt gamma) such that prod = (B -> beta nt gamma) ;; returns the list of all (B -> beta . nt gamma) such that prod = (B -> beta nt gamma)
;; and gamma =>* epsilon ;; and gamma =>* epsilon
(define (prod->items-for-include g prod nt) (define (prod->items-for-include g prod nt)
(let* ((rhs (prod-rhs prod)) (define rhs (prod-rhs prod))
(rhs-l (vector-length rhs))) (define rhs-l (vector-length rhs))
(append (if (and (> rhs-l 0) (eq? nt (vector-ref rhs (sub1 rhs-l)))) (append (if (and (> rhs-l 0) (eq? nt (vector-ref rhs (sub1 rhs-l))))
(list (make-item prod (sub1 rhs-l))) (list (make-item prod (sub1 rhs-l)))
null) null)
(let loop ((i (sub1 rhs-l))) (let loop ([i (sub1 rhs-l)])
(cond (cond
((and (> i 0) [(and (> i 0)
(non-term? (vector-ref rhs i)) (non-term? (vector-ref rhs i))
(send g nullable-non-term? (vector-ref rhs i))) (send g nullable-non-term? (vector-ref rhs i)))
(if (eq? nt (vector-ref rhs (sub1 i))) (if (eq? nt (vector-ref rhs (sub1 i)))
(cons (make-item prod (sub1 i)) (cons (make-item prod (sub1 i))
(loop (sub1 i))) (loop (sub1 i)))
(loop (sub1 i)))) (loop (sub1 i)))]
(else null)))))) [else null]))))
;; prod-list->items-for-include: grammar * prod list * non-term -> lr0-item list ;; prod-list->items-for-include: grammar * prod list * non-term -> lr0-item list
;; return the list of all (B -> beta . nt gamma) such that (B -> beta nt gamma) in prod-list ;; return the list of all (B -> beta . nt gamma) such that (B -> beta nt gamma) in prod-list
;; and gamma =>* epsilon ;; and gamma =>* epsilon
(define (prod-list->items-for-include g prod-list nt) (define (prod-list->items-for-include g prod-list nt)
(apply append (map (lambda (prod) (prod->items-for-include g prod nt)) prod-list))) (apply append (map (λ (prod) (prod->items-for-include g prod nt)) prod-list)))
;; comput-includes: lr0-automaton * grammar -> (trans-key -> trans-key list) ;; comput-includes: lr0-automaton * grammar -> (trans-key -> trans-key list)
(define (compute-includes a g) (define (compute-includes a g)
(let ((num-states (send a get-num-states)) (define num-states (send a get-num-states))
(items-for-input-nt (make-vector (send g get-num-non-terms) null))) (define items-for-input-nt (make-vector (send g get-num-non-terms) null))
(for-each (for ([input-nt (in-list (send g get-non-terms))])
(lambda (input-nt)
(vector-set! items-for-input-nt (non-term-index input-nt) (vector-set! items-for-input-nt (non-term-index input-nt)
(prod-list->items-for-include g (send g get-prods) input-nt))) (prod-list->items-for-include g (send g get-prods) input-nt)))
(send g get-non-terms)) (λ (tk)
(lambda (tk) (define goal-state (trans-key-st tk))
(let* ((goal-state (trans-key-st tk)) (define non-term (trans-key-gs tk))
(non-term (trans-key-gs tk)) (define items (vector-ref items-for-input-nt (non-term-index non-term)))
(items (vector-ref items-for-input-nt (non-term-index non-term))))
(trans-key-list-remove-dups (trans-key-list-remove-dups
(apply append (apply append
(map (lambda (item) (for/list ([item (in-list items)])
(let* ((prod (item-prod item)) (define prod (item-prod item))
(rhs (prod-rhs prod)) (define rhs (prod-rhs prod))
(lhs (prod-lhs prod))) (define lhs (prod-lhs prod))
(map (lambda (state) (map (λ (state) (make-trans-key state lhs))
(make-trans-key state lhs))
(run-lr0-backward a (run-lr0-backward a
rhs rhs
(item-dot-pos item) (item-dot-pos item)
goal-state goal-state
num-states)))) num-states)))))))
items)))))))
;; compute-lookback: lr0-automaton * grammar -> (kernel * proc -> trans-key list)
;; compute-lookback: lr0-automaton * grammar -> (kernel * proc -> trans-key list) (define (compute-lookback a g)
(define (compute-lookback a g) (define num-states (send a get-num-states))
(let ((num-states (send a get-num-states))) (λ (state prod)
(lambda (state prod) (map (λ (k) (make-trans-key k (prod-lhs prod)))
(map (lambda (k) (make-trans-key k (prod-lhs prod))) (run-lr0-backward a (prod-rhs prod) (vector-length (prod-rhs prod)) state num-states))))
(run-lr0-backward a (prod-rhs prod) (vector-length (prod-rhs prod)) state num-states)))))
;; compute-follow: LR0-automaton * grammar -> (trans-key -> term set)
;; compute-follow: LR0-automaton * grammar -> (trans-key -> term set) ;; output term set is represented in bit-vector form
;; output term set is represented in bit-vector form (define (compute-follow a g includes)
(define (compute-follow a g includes) (define read (compute-read a g))
(let ((read (compute-read a g)))
(digraph-tk->terml (send a get-mapped-non-term-keys) (digraph-tk->terml (send a get-mapped-non-term-keys)
includes includes
read read
(send a get-num-states)))) (send a get-num-states)))
;; compute-LA: LR0-automaton * grammar -> kernel * prod -> term set ;; compute-LA: LR0-automaton * grammar -> kernel * prod -> term set
;; output term set is represented in bit-vector form ;; output term set is represented in bit-vector form
(define (compute-LA a g) (define (compute-LA a g)
(let* ((includes (compute-includes a g)) (define includes (compute-includes a g))
(lookback (compute-lookback a g)) (define lookback (compute-lookback a g))
(follow (compute-follow a g includes))) (define follow (compute-follow a g includes))
(lambda (k p) (λ (k p)
(let* ((l (lookback k p)) (define l (lookback k p))
(f (map follow l))) (define f (map follow l))
(apply bitwise-ior (cons 0 f)))))) (apply bitwise-ior (cons 0 f))))
(define (print-DR dr a g)
(define (print-DR dr a g)
(print-input-st-sym dr "DR" a g print-output-terms)) (print-input-st-sym dr "DR" a g print-output-terms))
(define (print-Read Read a g) (define (print-Read Read a g)
(print-input-st-sym Read "Read" a g print-output-terms)) (print-input-st-sym Read "Read" a g print-output-terms))
(define (print-includes i a g) (define (print-includes i a g)
(print-input-st-sym i "includes" a g print-output-st-nt)) (print-input-st-sym i "includes" a g print-output-st-nt))
(define (print-lookback l a g) (define (print-lookback l a g)
(print-input-st-prod l "lookback" a g print-output-st-nt)) (print-input-st-prod l "lookback" a g print-output-st-nt))
(define (print-follow f a g) (define (print-follow f a g)
(print-input-st-sym f "follow" a g print-output-terms)) (print-input-st-sym f "follow" a g print-output-terms))
(define (print-LA l a g) (define (print-LA l a g)
(print-input-st-prod l "LA" a g print-output-terms)) (print-input-st-prod l "LA" a g print-output-terms))
(define (print-input-st-sym f name a g print-output) (define (print-input-st-sym f name a g print-output)
(printf "~a:\n" name) (printf "~a:\n" name)
(send a for-each-state (send a for-each-state
(lambda (state) (λ (state)
(for-each (for-each
(lambda (non-term) (λ (non-term)
(let ((res (f (make-trans-key state non-term)))) (let ([res (f (make-trans-key state non-term))])
(if (not (null? res)) (when (not (null? res))
(printf "~a(~a, ~a) = ~a\n" (printf "~a(~a, ~a) = ~a\n"
name name
state state
@ -163,16 +153,16 @@
(send g get-non-terms)))) (send g get-non-terms))))
(newline)) (newline))
(define (print-input-st-prod f name a g print-output) (define (print-input-st-prod f name a g print-output)
(printf "~a:\n" name) (printf "~a:\n" name)
(send a for-each-state (send a for-each-state
(lambda (state) (λ (state)
(for-each (for-each
(lambda (non-term) (λ (non-term)
(for-each (for-each
(lambda (prod) (λ (prod)
(let ((res (f state prod))) (let ([res (f state prod)])
(if (not (null? res)) (when (not (null? res))
(printf "~a(~a, ~a) = ~a\n" (printf "~a(~a, ~a) = ~a\n"
name name
(kernel-index state) (kernel-index state)
@ -181,97 +171,82 @@
(send g get-prods-for-non-term non-term))) (send g get-prods-for-non-term non-term)))
(send g get-non-terms))))) (send g get-non-terms)))))
(define (print-output-terms r) (define (print-output-terms r)
(map (map gram-sym-symbol r))
(lambda (p)
(gram-sym-symbol p))
r))
(define (print-output-st-nt r) (define (print-output-st-nt r)
(map (map (λ (p) (list (kernel-index (trans-key-st p)) (gram-sym-symbol (trans-key-gs p)))) r))
(lambda (p)
(list
(kernel-index (trans-key-st p))
(gram-sym-symbol (trans-key-gs p))))
r))
;; init-tk-map : int -> (vectorof hashtable?) ;; init-tk-map : int -> (vectorof hashtable?)
(define (init-tk-map n) (define (init-tk-map n)
(let ((v (make-vector n #f))) (define v (make-vector n #f))
(let loop ((i (sub1 (vector-length v)))) (let loop ([i (sub1 (vector-length v))])
(when (>= i 0) (when (>= i 0)
(vector-set! v i (make-hash-table)) (vector-set! v i (make-hasheq))
(loop (sub1 i)))) (loop (sub1 i))))
v)) v)
;; lookup-tk-map : (vectorof (symbol? int hashtable)) -> trans-key? -> int ;; lookup-tk-map : (vectorof (symbol? int hashtable)) -> trans-key? -> int
(define (lookup-tk-map map) (define ((lookup-tk-map map) tk)
(lambda (tk) (define st (trans-key-st tk))
(let ((st (trans-key-st tk)) (define gs (trans-key-gs tk))
(gs (trans-key-gs tk))) (hash-ref (vector-ref map (kernel-index st))
(hash-table-get (vector-ref map (kernel-index st))
(gram-sym-symbol gs) (gram-sym-symbol gs)
(lambda () 0))))) (λ () 0)))
;; add-tk-map : (vectorof (symbol? int hashtable)) -> trans-key int -> ;; add-tk-map : (vectorof (symbol? int hashtable)) -> trans-key int ->
(define (add-tk-map map) (define ((add-tk-map map) tk v)
(lambda (tk v) (define st (trans-key-st tk))
(let ((st (trans-key-st tk)) (define gs (trans-key-gs tk))
(gs (trans-key-gs tk))) (hash-set! (vector-ref map (kernel-index st))
(hash-table-put! (vector-ref map (kernel-index st))
(gram-sym-symbol gs) (gram-sym-symbol gs)
v)))) v))
;; digraph-tk->terml: ;; digraph-tk->terml:
;; (trans-key list) * (trans-key -> trans-key list) * (trans-key -> term list) * int * int * int ;; (trans-key list) * (trans-key -> trans-key list) * (trans-key -> term list) * int * int * int
;; -> (trans-key -> term list) ;; -> (trans-key -> term list)
;; DeRemer and Pennello 1982 ;; DeRemer and Pennello 1982
;; Computes (f x) = (f- x) union Union{(f y) | y in (edges x)} ;; Computes (f x) = (f- x) union Union{(f y) | y in (edges x)}
;; A specialization of digraph in the file graph.rkt ;; A specialization of digraph in the file graph.rkt
(define (digraph-tk->terml nodes edges f- num-states) (define (digraph-tk->terml nodes edges f- num-states)
(letrec [
;; Will map elements of trans-key to term sets represented as bit vectors ;; Will map elements of trans-key to term sets represented as bit vectors
(results (init-tk-map num-states)) (define results (init-tk-map num-states))
;; Maps elements of trans-keys to integers. ;; Maps elements of trans-keys to integers.
(N (init-tk-map num-states)) (define N (init-tk-map num-states))
(get-N (lookup-tk-map N)) (define get-N (lookup-tk-map N))
(set-N (add-tk-map N)) (define set-N (add-tk-map N))
(get-f (lookup-tk-map results)) (define get-f (lookup-tk-map results))
(set-f (add-tk-map results)) (define set-f (add-tk-map results))
(stack null) (define stack null)
(push (lambda (x) (define (push x) (set! stack (cons x stack)))
(set! stack (cons x stack)))) (define (pop) (begin0
(pop (lambda ()
(begin0
(car stack) (car stack)
(set! stack (cdr stack))))) (set! stack (cdr stack))))
(depth (lambda () (length stack))) (define (depth) (length stack))
;; traverse: 'a -> ;; traverse: 'a ->
(traverse (define (traverse x)
(lambda (x)
(push x) (push x)
(let ((d (depth))) (let ([d (depth)])
(set-N x d) (set-N x d)
(set-f x (f- x)) (set-f x (f- x))
(for-each (lambda (y) (for-each (λ (y)
(when (= 0 (get-N y)) (when (= 0 (get-N y))
(traverse y)) (traverse y))
(set-f x (bitwise-ior (get-f x) (get-f y))) (set-f x (bitwise-ior (get-f x) (get-f y)))
(set-N x (min (get-N x) (get-N y)))) (set-N x (min (get-N x) (get-N y))))
(edges x)) (edges x))
(when (= d (get-N x)) (when (= d (get-N x))
(let loop ((p (pop))) (let loop ([p (pop)])
(set-N p +inf.0) (set-N p +inf.0)
(set-f p (get-f x)) (set-f p (get-f x))
(unless (equal? x p) (unless (equal? x p)
(loop (pop))))))))] (loop (pop)))))))
(for-each (lambda (x)
(when (= 0 (get-N x)) (for ([x (in-list nodes)]
(traverse x))) #:when (zero? (get-N x)))
nodes) (traverse x))
get-f)) get-f)
)

386
br-parser-tools-lib/br-parser-tools/private-yacc/lr0.rkt
Unescape Escape View File

@ -1,84 +1,78 @@
(module lr0 mzscheme #lang racket/base
(require "grammar.rkt"
;; Handle the LR0 automaton
(require "grammar.rkt"
"graph.rkt" "graph.rkt"
mzlib/list racket/list
mzlib/class) racket/class)
;; Handle the LR0 automaton
(provide build-lr0-automaton lr0% (provide build-lr0-automaton lr0%
(struct trans-key (st gs)) trans-key-list-remove-dups (struct-out trans-key) trans-key-list-remove-dups
kernel-items kernel-index) kernel-items kernel-index)
;; kernel = (make-kernel (LR1-item list) index) ;; kernel = (make-kernel (LR1-item list) index)
;; the list must be kept sorted according to item<? so that equal? can ;; the list must be kept sorted according to item<? so that equal? can
;; be used to compare kernels ;; be used to compare kernels
;; Each kernel is assigned a unique index, 0 <= index < number of states ;; Each kernel is assigned a unique index, 0 <= index < number of states
;; trans-key = (make-trans-key kernel gram-sym) ;; trans-key = (make-trans-key kernel gram-sym)
(define-struct kernel (items index) (make-inspector)) (define-struct kernel (items index) #:inspector (make-inspector))
(define-struct trans-key (st gs) (make-inspector)) (define-struct trans-key (st gs) #:inspector (make-inspector))
(define (trans-key<? a b) (define (trans-key<? a b)
(let ((kia (kernel-index (trans-key-st a))) (define kia (kernel-index (trans-key-st a)))
(kib (kernel-index (trans-key-st b)))) (define kib (kernel-index (trans-key-st b)))
(or (< kia kib) (or (< kia kib)
(and (= kia kib) (and (= kia kib)
(< (non-term-index (trans-key-gs a)) (< (non-term-index (trans-key-gs a))
(non-term-index (trans-key-gs b))))))) (non-term-index (trans-key-gs b))))))
(define (trans-key-list-remove-dups tkl) (define (trans-key-list-remove-dups tkl)
(let loop ((sorted (sort tkl trans-key<?))) (let loop ([sorted (sort tkl trans-key<?)])
(cond (cond
((null? sorted) null) [(null? sorted) null]
((null? (cdr sorted)) sorted) [(null? (cdr sorted)) sorted]
(else [else
(if (and (= (non-term-index (trans-key-gs (car sorted))) (if (and (= (non-term-index (trans-key-gs (car sorted)))
(non-term-index (trans-key-gs (cadr sorted)))) (non-term-index (trans-key-gs (cadr sorted))))
(= (kernel-index (trans-key-st (car sorted))) (= (kernel-index (trans-key-st (car sorted)))
(kernel-index (trans-key-st (cadr sorted))))) (kernel-index (trans-key-st (cadr sorted)))))
(loop (cdr sorted)) (loop (cdr sorted))
(cons (car sorted) (loop (cdr sorted)))))))) (cons (car sorted) (loop (cdr sorted))))])))
;; build-transition-table : int (listof (cons/c trans-key X) -> ;; build-transition-table : int (listof (cons/c trans-key X) ->
;; (vectorof (symbol X hashtable)) ;; (vectorof (symbol X hashtable))
(define (build-transition-table num-states assoc) (define (build-transition-table num-states assoc)
(let ((transitions (make-vector num-states #f))) (define transitions (make-vector num-states #f))
(let loop ((i (sub1 (vector-length transitions)))) (let loop ([i (sub1 (vector-length transitions))])
(when (>= i 0) (when (>= i 0)
(vector-set! transitions i (make-hash-table)) (vector-set! transitions i (make-hasheq))
(loop (sub1 i)))) (loop (sub1 i))))
(for-each (for ([trans-key/kernel (in-list assoc)])
(lambda (trans-key/kernel) (define tk (car trans-key/kernel))
(let ((tk (car trans-key/kernel))) (hash-set! (vector-ref transitions (kernel-index (trans-key-st tk)))
(hash-table-put! (vector-ref transitions (kernel-index (trans-key-st tk)))
(gram-sym-symbol (trans-key-gs tk)) (gram-sym-symbol (trans-key-gs tk))
(cdr trans-key/kernel)))) (cdr trans-key/kernel)))
assoc) transitions)
transitions))
;; reverse-assoc : (listof (cons/c trans-key? kernel?)) ->
;; reverse-assoc : (listof (cons/c trans-key? kernel?)) -> ;; (listof (cons/c trans-key? (listof kernel?)))
;; (listof (cons/c trans-key? (listof kernel?))) (define (reverse-assoc assoc)
(define (reverse-assoc assoc) (define reverse-hash (make-hash))
(let ((reverse-hash (make-hash-table 'equal)) (define (hash-table-add! ht k v)
(hash-table-add! (hash-set! ht k (cons v (hash-ref ht k (λ () null)))))
(lambda (ht k v) (for ([trans-key/kernel (in-list assoc)])
(hash-table-put! ht k (cons v (hash-table-get ht k (lambda () null))))))) (define tk (car trans-key/kernel))
(for-each
(lambda (trans-key/kernel)
(let ((tk (car trans-key/kernel)))
(hash-table-add! reverse-hash (hash-table-add! reverse-hash
(make-trans-key (cdr trans-key/kernel) (make-trans-key (cdr trans-key/kernel)
(trans-key-gs tk)) (trans-key-gs tk))
(trans-key-st tk)))) (trans-key-st tk)))
assoc) (hash-map reverse-hash cons))
(hash-table-map reverse-hash cons)))
;; kernel-list-remove-duplicates ;; kernel-list-remove-duplicates
;; LR0-automaton = object of class lr0% ;; LR0-automaton = object of class lr0%
(define lr0% (define lr0%
(class object% (class object%
(super-instantiate ()) (super-instantiate ())
;; term-assoc : (listof (cons/c trans-key? kernel?)) ;; term-assoc : (listof (cons/c trans-key? kernel?))
@ -113,64 +107,56 @@
;; for-each-state : (state ->) -> ;; for-each-state : (state ->) ->
;; Iteration over the states in an automaton ;; Iteration over the states in an automaton
(define/public (for-each-state f) (define/public (for-each-state f)
(let ((num-states (vector-length states))) (define num-states (vector-length states))
(let loop ((i 0)) (let loop ([i 0])
(if (< i num-states) (when (< i num-states)
(begin
(f (vector-ref states i)) (f (vector-ref states i))
(loop (add1 i))))))) (loop (add1 i)))))
;; run-automaton: kernel? gram-sym? -> (union kernel #f) ;; run-automaton: kernel? gram-sym? -> (union kernel #f)
;; returns the state reached from state k on input s, or #f when k ;; returns the state reached from state k on input s, or #f when k
;; has no transition on s ;; has no transition on s
(define/public (run-automaton k s) (define/public (run-automaton k s)
(hash-table-get (vector-ref transitions (kernel-index k)) (hash-ref (vector-ref transitions (kernel-index k))
(gram-sym-symbol s) (gram-sym-symbol s)
(lambda () #f))) (λ () #f)))
;; run-automaton-back : (listof kernel?) gram-sym? -> (listof kernel) ;; run-automaton-back : (listof kernel?) gram-sym? -> (listof kernel)
;; returns the list of states that can reach k by transitioning on s. ;; returns the list of states that can reach k by transitioning on s.
(define/public (run-automaton-back k s) (define/public (run-automaton-back k s)
(apply append (for*/list ([k (in-list k)]
(map [val (in-list (hash-ref (vector-ref reverse-transitions (kernel-index k))
(lambda (k)
(hash-table-get (vector-ref reverse-transitions (kernel-index k))
(gram-sym-symbol s) (gram-sym-symbol s)
(lambda () null))) (λ () null)))])
k))))) val))))
(define (union comp<?) (define ((union comp<?) l1 l2)
(letrec ((union (let loop ([l1 l1] [l2 l2])
(lambda (l1 l2)
(cond (cond
((null? l1) l2) [(null? l1) l2]
((null? l2) l1) [(null? l2) l1]
(else (let ((c1 (car l1)) [else (define c1 (car l1))
(c2 (car l2))) (define c2 (car l2))
(cond (cond
((comp<? c1 c2) [(comp<? c1 c2) (cons c1 (loop (cdr l1) l2))]
(cons c1 (union (cdr l1) l2))) [(comp<? c2 c1) (cons c2 (loop l1 (cdr l2)))]
((comp<? c2 c1) [else (loop (cdr l1) l2)])])))
(cons c2 (union l1 (cdr l2))))
(else (union (cdr l1) l2)))))))))
union))
;; The kernels in the automaton are represented cannonically. ;; The kernels in the automaton are represented cannonically.
;; That is (equal? a b) <=> (eq? a b) ;; That is (equal? a b) <=> (eq? a b)
(define (kernel->string k) (define (kernel->string k)
(apply string-append (apply string-append
`("{" ,@(map (lambda (i) (string-append (item->string i) ", ")) `("{" ,@(map (λ (i) (string-append (item->string i) ", "))
(kernel-items k)) (kernel-items k))
"}"))) "}")))
;; build-LR0-automaton: grammar -> LR0-automaton ;; build-LR0-automaton: grammar -> LR0-automaton
;; Constructs the kernels of the sets of LR(0) items of g ;; Constructs the kernels of the sets of LR(0) items of g
(define (build-lr0-automaton grammar) (define (build-lr0-automaton grammar)
; (printf "LR(0) automaton:\n") ; (printf "LR(0) automaton:\n")
(letrec ( (define epsilons (make-hash))
(epsilons (make-hash-table 'equal)) (define grammar-symbols (append (send grammar get-non-terms)
(grammar-symbols (append (send grammar get-non-terms)
(send grammar get-terms))) (send grammar get-terms)))
;; first-non-term: non-term -> non-term list ;; first-non-term: non-term -> non-term list
;; given a non-terminal symbol C, return those non-terminal ;; given a non-terminal symbol C, return those non-terminal
@ -178,195 +164,151 @@
;; non-terminals n where -> means a rightmost derivation in many ;; non-terminals n where -> means a rightmost derivation in many
;; steps. Assumes that each non-term can be reduced to a string ;; steps. Assumes that each non-term can be reduced to a string
;; of terms. ;; of terms.
(first-non-term (define first-non-term
(digraph (send grammar get-non-terms) (digraph (send grammar get-non-terms)
(lambda (nt) (λ (nt)
(filter non-term? (filter non-term?
(map (lambda (prod) (map (λ (prod) (sym-at-dot (make-item prod 0)))
(sym-at-dot (make-item prod 0)))
(send grammar get-prods-for-non-term nt)))) (send grammar get-prods-for-non-term nt))))
(lambda (nt) (list nt)) (λ (nt) (list nt))
(union non-term<?) (union non-term<?)
(lambda () null))) (λ () null)))
;; closure: LR1-item list -> LR1-item list ;; closure: LR1-item list -> LR1-item list
;; Creates a set of items containing i s.t. if A -> n.Xm is in it, ;; Creates a set of items containing i s.t. if A -> n.Xm is in it,
;; X -> .o is in it too. ;; X -> .o is in it too.
(LR0-closure (define (LR0-closure i)
(lambda (i)
(cond (cond
((null? i) null) [(null? i) null]
(else [else
(let ((next-gsym (sym-at-dot (car i)))) (define next-gsym (sym-at-dot (car i)))
(cond (cond
((non-term? next-gsym) [(non-term? next-gsym)
(cons (car i) (cons (car i)
(append (append
(apply append (for*/list ([non-term (in-list (first-non-term next-gsym))]
(map (lambda (non-term) [x (in-list (send grammar
(map (lambda (x)
(make-item x 0))
(send grammar
get-prods-for-non-term get-prods-for-non-term
non-term))) non-term))])
(first-non-term next-gsym))) (make-item x 0))
(LR0-closure (cdr i))))) (LR0-closure (cdr i))))]
(else [else (cons (car i) (LR0-closure (cdr i)))])]))
(cons (car i) (LR0-closure (cdr i))))))))))
;; maps trans-keys to kernels ;; maps trans-keys to kernels
(automaton-term null) (define automaton-term null)
(automaton-non-term null) (define automaton-non-term null)
;; keeps the kernels we have seen, so we can have a unique ;; keeps the kernels we have seen, so we can have a unique
;; list for each kernel ;; list for each kernel
(kernels (make-hash-table 'equal)) (define kernels (make-hash))
(define counter 0)
(counter 0)
;; goto: LR1-item list -> LR1-item list list ;; goto: LR1-item list -> LR1-item list list
;; creates new kernels by moving the dot in each item in the ;; creates new kernels by moving the dot in each item in the
;; LR0-closure of kernel to the right, and grouping them by ;; LR0-closure of kernel to the right, and grouping them by
;; the term/non-term moved over. Returns the kernels not ;; the term/non-term moved over. Returns the kernels not
;; yet seen, and places the trans-keys into automaton ;; yet seen, and places the trans-keys into automaton
(goto (define (goto kernel)
(lambda (kernel)
(let (
;; maps a gram-syms to a list of items ;; maps a gram-syms to a list of items
(table (make-hash-table)) (define table (make-hasheq))
;; add-item!: ;; add-item!:
;; (symbol (listof item) hashtable) item? -> ;; (symbol (listof item) hashtable) item? ->
;; adds i into the table grouped with the grammar ;; adds i into the table grouped with the grammar
;; symbol following its dot ;; symbol following its dot
(add-item! (define (add-item! table i)
(lambda (table i) (define gs (sym-at-dot i))
(let ((gs (sym-at-dot i)))
(cond (cond
(gs [gs (define already (hash-ref table (gram-sym-symbol gs) (λ () null)))
(let ((already
(hash-table-get table
(gram-sym-symbol gs)
(lambda () null))))
(unless (member i already) (unless (member i already)
(hash-table-put! table (hash-set! table (gram-sym-symbol gs) (cons i already)))]
(gram-sym-symbol gs) ((zero? (vector-length (prod-rhs (item-prod i))))
(cons i already))))) (define current (hash-ref epsilons kernel (λ () null)))
((= 0 (vector-length (prod-rhs (item-prod i)))) (hash-set! epsilons kernel (cons i current)))))
(let ((current (hash-table-get epsilons
kernel
(lambda () null))))
(hash-table-put! epsilons
kernel
(cons i current)))))))))
;; Group the items of the LR0 closure of the kernel ;; Group the items of the LR0 closure of the kernel
;; by the character after the dot ;; by the character after the dot
(for-each (lambda (item) (for ([item (in-list (LR0-closure (kernel-items kernel)))])
(add-item! table item)) (add-item! table item))
(LR0-closure (kernel-items kernel)))
;; each group is a new kernel, with the dot advanced. ;; each group is a new kernel, with the dot advanced.
;; sorts the items in a kernel so kernels can be compared ;; sorts the items in a kernel so kernels can be compared
;; with equal? for using the table kernels to make sure ;; with equal? for using the table kernels to make sure
;; only one representitive of each kernel is created ;; only one representitive of each kernel is created
(define is
(let loop ([gsyms grammar-symbols])
(cond
[(null? gsyms) null]
[else
(define items (hash-ref table (gram-sym-symbol (car gsyms)) (λ () null)))
(cond
[(null? items) (loop (cdr gsyms))]
[else (cons (list (car gsyms) items)
(loop (cdr gsyms)))])])))
(filter (filter
(lambda (x) x) values
(map (for/list ([i (in-list is)])
(lambda (i) (define gs (car i))
(let* ((gs (car i)) (define items (cadr i))
(items (cadr i)) (define new #f)
(new #f) (define new-kernel (sort (filter values (map move-dot-right items)) item<?))
(new-kernel (sort (define unique-kernel (hash-ref kernels new-kernel
(filter (lambda (x) x) (λ ()
(map move-dot-right items)) (define k (make-kernel new-kernel counter))
item<?))
(unique-kernel (hash-table-get
kernels
new-kernel
(lambda ()
(let ((k (make-kernel
new-kernel
counter)))
(set! new #t) (set! new #t)
(set! counter (add1 counter)) (set! counter (add1 counter))
(hash-table-put! kernels (hash-set! kernels new-kernel k)
new-kernel k)))
k) (if (term? gs)
k)))))
(cond
((term? gs)
(set! automaton-term (cons (cons (make-trans-key kernel gs) (set! automaton-term (cons (cons (make-trans-key kernel gs)
unique-kernel) unique-kernel)
automaton-term))) automaton-term))
(else
(set! automaton-non-term (cons (cons (make-trans-key kernel gs) (set! automaton-non-term (cons (cons (make-trans-key kernel gs)
unique-kernel) unique-kernel)
automaton-non-term)))) automaton-non-term)))
#;(printf "~a -> ~a on ~a\n" #;(printf "~a -> ~a on ~a\n"
(kernel->string kernel) (kernel->string kernel)
(kernel->string unique-kernel) (kernel->string unique-kernel)
(gram-sym-symbol gs)) (gram-sym-symbol gs))
(if new (and new unique-kernel))))
unique-kernel
#f)))
(let loop ((gsyms grammar-symbols))
(cond
((null? gsyms) null)
(else
(let ((items (hash-table-get table
(gram-sym-symbol (car gsyms))
(lambda () null))))
(cond
((null? items) (loop (cdr gsyms)))
(else
(cons (list (car gsyms) items)
(loop (cdr gsyms))))))))))))))
(starts (define starts (map (λ (init-prod) (list (make-item init-prod 0)))
(map (lambda (init-prod) (list (make-item init-prod 0)))
(send grammar get-init-prods))) (send grammar get-init-prods)))
(startk (define startk (for/list ([start (in-list starts)])
(map (lambda (start) (define k (make-kernel start counter))
(let ((k (make-kernel start counter))) (hash-set! kernels start k)
(hash-table-put! kernels start k)
(set! counter (add1 counter)) (set! counter (add1 counter))
k)) k))
starts)) (define new-kernels (make-queue))
(new-kernels (make-queue))) (let loop ([old-kernels startk]
[seen-kernels null])
(let loop ((old-kernels startk)
(seen-kernels null))
(cond (cond
((and (empty-queue? new-kernels) (null? old-kernels)) [(and (empty-queue? new-kernels) (null? old-kernels))
(make-object lr0% (make-object lr0% automaton-term automaton-non-term
automaton-term (list->vector (reverse seen-kernels)) epsilons)]
automaton-non-term [(null? old-kernels) (loop (deq! new-kernels) seen-kernels)]
(list->vector (reverse seen-kernels)) [else
epsilons))
((null? old-kernels)
(loop (deq! new-kernels) seen-kernels))
(else
(enq! new-kernels (goto (car old-kernels))) (enq! new-kernels (goto (car old-kernels)))
(loop (cdr old-kernels) (cons (car old-kernels) seen-kernels))))))) (loop (cdr old-kernels) (cons (car old-kernels) seen-kernels))])))
(define-struct q (f l) (make-inspector)) (define-struct q (f l) #:inspector (make-inspector) #:mutable)
(define (empty-queue? q) (define (empty-queue? q) (null? (q-f q)))
(null? (q-f q))) (define (make-queue) (make-q null null))
(define (make-queue)
(make-q null null)) (define (enq! q i)
(define (enq! q i) (cond
(if (empty-queue? q) [(empty-queue? q)
(let ((i (mcons i null))) (let ([i (mcons i null)])
(set-q-l! q i) (set-q-l! q i)
(set-q-f! q i)) (set-q-f! q i))]
(begin [else
(set-mcdr! (q-l q) (mcons i null)) (set-mcdr! (q-l q) (mcons i null))
(set-q-l! q (mcdr (q-l q)))))) (set-q-l! q (mcdr (q-l q)))]))
(define (deq! q)
(define (deq! q)
(begin0 (begin0
(mcar (q-f q)) (mcar (q-f q))
(set-q-f! q (mcdr (q-f q))))) (set-q-f! q (mcdr (q-f q)))))
)

94
br-parser-tools-lib/br-parser-tools/private-yacc/parser-actions.rkt
Unescape Escape View File

@ -1,54 +1,54 @@
(module parser-actions mzscheme #lang racket/base
(require "grammar.rkt") (require "grammar.rkt")
(provide (all-defined-except make-reduce make-reduce*) (provide (except-out (all-defined-out) make-reduce make-reduce*)
(rename make-reduce* make-reduce)) (rename-out [make-reduce* make-reduce]))
;; An action is ;; An action is
;; - (make-shift int) ;; - (make-shift int)
;; - (make-reduce prod runtime-action) ;; - (make-reduce prod runtime-action)
;; - (make-accept) ;; - (make-accept)
;; - (make-goto int) ;; - (make-goto int)
;; - (no-action) ;; - (no-action)
;; A reduce contains a runtime-reduce so that sharing of the reduces can ;; A reduce contains a runtime-reduce so that sharing of the reduces can
;; be easily transferred to sharing of runtime-reduces. ;; be easily transferred to sharing of runtime-reduces.
(define-struct action () (make-inspector)) (define-struct action () #:inspector (make-inspector))
(define-struct (shift action) (state) (make-inspector)) (define-struct (shift action) (state) #:inspector (make-inspector))
(define-struct (reduce action) (prod runtime-reduce) (make-inspector)) (define-struct (reduce action) (prod runtime-reduce) #:inspector (make-inspector))
(define-struct (accept action) () (make-inspector)) (define-struct (accept action) () #:inspector (make-inspector))
(define-struct (goto action) (state) (make-inspector)) (define-struct (goto action) (state) #:inspector (make-inspector))
(define-struct (no-action action) () (make-inspector)) (define-struct (no-action action) () #:inspector (make-inspector))
(define (make-reduce* p) (define (make-reduce* p)
(make-reduce p (make-reduce p
(vector (prod-index p) (vector (prod-index p)
(gram-sym-symbol (prod-lhs p)) (gram-sym-symbol (prod-lhs p))
(vector-length (prod-rhs p))))) (vector-length (prod-rhs p)))))
;; A runtime-action is ;; A runtime-action is
;; non-negative-int (shift) ;; non-negative-int (shift)
;; (vector int symbol int) (reduce) ;; (vector int symbol int) (reduce)
;; 'accept (accept) ;; 'accept (accept)
;; negative-int (goto) ;; negative-int (goto)
;; #f (no-action) ;; #f (no-action)
(define (action->runtime-action a) (define (action->runtime-action a)
(cond (cond
((shift? a) (shift-state a)) [(shift? a) (shift-state a)]
((reduce? a) (reduce-runtime-reduce a)) [(reduce? a) (reduce-runtime-reduce a)]
((accept? a) 'accept) [(accept? a) 'accept]
((goto? a) (- (+ (goto-state a) 1))) [(goto? a) (- (+ (goto-state a) 1))]
((no-action? a) #f))) [(no-action? a) #f]))
(define (runtime-shift? x) (and (integer? x) (>= x 0))) (define (runtime-shift? x) (and (integer? x) (>= x 0)))
(define runtime-reduce? vector?) (define runtime-reduce? vector?)
(define (runtime-accept? x) (eq? x 'accept)) (define (runtime-accept? x) (eq? x 'accept))
(define (runtime-goto? x) (and (integer? x) (< x 0))) (define (runtime-goto? x) (and (integer? x) (< x 0)))
(define runtime-shift-state values) (define runtime-shift-state values)
(define (runtime-reduce-prod-num x) (vector-ref x 0)) (define (runtime-reduce-prod-num x) (vector-ref x 0))
(define (runtime-reduce-lhs x) (vector-ref x 1)) (define (runtime-reduce-lhs x) (vector-ref x 1))
(define (runtime-reduce-rhs-length x) (vector-ref x 2)) (define (runtime-reduce-rhs-length x) (vector-ref x 2))
(define (runtime-goto-state x) (- (+ x 1))) (define (runtime-goto-state x) (- (+ x 1)))
)

128
br-parser-tools-lib/br-parser-tools/private-yacc/parser-builder.rkt
Unescape Escape View File

@ -1,98 +1,89 @@
(module parser-builder mzscheme #lang racket/base
(require "input-file-parser.rkt"
(require "input-file-parser.rkt"
"grammar.rkt" "grammar.rkt"
"table.rkt" "table.rkt"
mzlib/class racket/class
racket/contract) racket/contract)
(require-for-template mzscheme) (require (for-template racket/base))
(provide/contract (provide/contract [build-parser (-> string? any/c any/c
(build-parser (-> string? any/c any/c
(listof identifier?) (listof identifier?)
(listof identifier?) (listof identifier?)
(listof identifier?) (listof identifier?)
(or/c syntax? #f) (or/c syntax? #f)
syntax? syntax?
(values any/c any/c any/c any/c)))) (values any/c any/c any/c any/c))])
;; fix-check-syntax : (listof identifier?) (listof identifier?) (listof identifier?) ;; fix-check-syntax : (listof identifier?) (listof identifier?) (listof identifier?)
;; (union syntax? false/c) syntax?) -> syntax? ;; (union syntax? false/c) syntax?) -> syntax?
(define (fix-check-syntax input-terms start ends assocs prods) (define (fix-check-syntax input-terms start ends assocs prods)
(let* ((term-binders (get-term-list input-terms)) (define term-binders (get-term-list input-terms))
(get-term-binder (define get-term-binder
(let ((t (make-hash-table))) (let ([t (make-hasheq)])
(for-each (for ([term (in-list term-binders)])
(lambda (term) (hash-set! t (syntax-e term) term))
(hash-table-put! t (syntax-e term) term)) (λ (x)
term-binders) (define r (hash-ref t (syntax-e x) (λ () #f)))
(lambda (x)
(let ((r (hash-table-get t (syntax-e x) (lambda () #f))))
(if r (if r
(syntax-local-introduce (datum->syntax-object r (syntax-e x) x x)) (syntax-local-introduce (datum->syntax r (syntax-e x) x x))
x))))) x))))
(rhs-list (define rhs-list (syntax-case prods ()
(syntax-case prods () [((_ RHS ...) ...) (syntax->list #'(RHS ... ...))]))
(((_ rhs ...) ...) (with-syntax ([(TMP ...) (map syntax-local-introduce term-binders)]
(syntax->list (syntax (rhs ... ...))))))) [(TERM-GROUP ...)
(with-syntax (((tmp ...) (map syntax-local-introduce term-binders)) (map (λ (tg)
((term-group ...)
(map (lambda (tg)
(syntax-property (syntax-property
(datum->syntax-object tg #f) (datum->syntax tg #f)
'disappeared-use 'disappeared-use
tg)) tg))
input-terms)) input-terms)]
((end ...) [(END ...) (map get-term-binder ends)]
(map get-term-binder ends)) [(START ...) (map get-term-binder start)]
((start ...) [(BIND ...) (syntax-case prods ()
(map get-term-binder start)) (((BIND _ ...) ...)
((bind ...) (syntax->list #'(BIND ...))))]
(syntax-case prods () [((BOUND ...) ...)
(((bind _ ...) ...) (map (λ (rhs)
(syntax->list (syntax (bind ...))))))
(((bound ...) ...)
(map
(lambda (rhs)
(syntax-case rhs () (syntax-case rhs ()
(((bound ...) (_ pbound) __) [((BOUND ...) (_ PBOUND) __)
(map get-term-binder (map get-term-binder
(cons (syntax pbound) (cons #'PBOUND (syntax->list #'(BOUND ...))))]
(syntax->list (syntax (bound ...)))))) [((BOUND ...) _)
(((bound ...) _)
(map get-term-binder (map get-term-binder
(syntax->list (syntax (bound ...))))))) (syntax->list #'(BOUND ...)))]))
rhs-list)) rhs-list)]
((prec ...) [(PREC ...)
(if assocs (if assocs
(map get-term-binder (map get-term-binder
(syntax-case assocs () (syntax-case assocs ()
(((__ term ...) ...) (((__ TERM ...) ...)
(syntax->list (syntax (term ... ...)))))) (syntax->list #'(TERM ... ...)))))
null))) null)])
#`(when #f #`(when #f
(let ((bind void) ... (tmp void) ...) (let ((BIND void) ... (TMP void) ...)
(void bound ... ... term-group ... start ... end ... prec ...)))))) (void BOUND ... ... TERM-GROUP ... START ... END ... PREC ...)))))
(require mzlib/list "parser-actions.rkt")
(define (build-parser filename src-pos suppress input-terms start end assocs prods) (require racket/list "parser-actions.rkt")
(let* ((grammar (parse-input input-terms start end assocs prods src-pos))
(table (build-table grammar filename suppress)) (define (build-parser filename src-pos suppress input-terms start end assocs prods)
(all-tokens (make-hash-table)) (define grammar (parse-input input-terms start end assocs prods src-pos))
(actions-code (define table (build-table grammar filename suppress))
`(vector ,@(map prod-action (send grammar get-prods))))) (define all-tokens (make-hasheq))
(for-each (lambda (term) (define actions-code `(vector ,@(map prod-action (send grammar get-prods))))
(hash-table-put! all-tokens (gram-sym-symbol term) #t))
(send grammar get-terms)) (for ([term (in-list (send grammar get-terms))])
(hash-set! all-tokens (gram-sym-symbol term) #t))
#;(let ((num-states (vector-length table)) #;(let ((num-states (vector-length table))
(num-gram-syms (+ (send grammar get-num-terms) (num-gram-syms (+ (send grammar get-num-terms)
(send grammar get-num-non-terms))) (send grammar get-num-non-terms)))
(num-ht-entries (apply + (map length (vector->list table)))) (num-ht-entries (apply + (map length (vector->list table))))
(num-reduces (num-reduces
(let ((ht (make-hash-table))) (let ((ht (make-hasheq)))
(for-each (for-each
(lambda (x) (λ (x)
(when (reduce? x) (when (reduce? x)
(hash-table-put! ht x #t))) (hash-set! ht x #t)))
(map cdr (apply append (vector->list table)))) (map cdr (apply append (vector->list table))))
(length (hash-table-map ht void))))) (length (hash-table-map ht void)))))
(printf "~a states, ~a grammar symbols, ~a hash-table entries, ~a reduces\n" (printf "~a states, ~a grammar symbols, ~a hash-table entries, ~a reduces\n"
@ -108,6 +99,5 @@
(values table (values table
all-tokens all-tokens
actions-code actions-code
(fix-check-syntax input-terms start end assocs prods)))) (fix-check-syntax input-terms start end assocs prods)))
)

312
br-parser-tools-lib/br-parser-tools/private-yacc/table.rkt
Unescape Escape View File

@ -1,132 +1,113 @@
#lang scheme/base #lang racket/base
(require "grammar.rkt"
;; Routine to build the LALR table
(require "grammar.rkt"
"lr0.rkt" "lr0.rkt"
"lalr.rkt" "lalr.rkt"
"parser-actions.rkt" "parser-actions.rkt"
racket/contract racket/contract
mzlib/list racket/list
mzlib/class) racket/class)
;; Routine to build the LALR table
(define (is-a-grammar%? x) (is-a? x grammar%))
(provide/contract (define (is-a-grammar%? x) (is-a? x grammar%))
(provide/contract
(build-table (-> is-a-grammar%? string? any/c (build-table (-> is-a-grammar%? string? any/c
(vectorof (listof (cons/c (or/c term? non-term?) action?)))))) (vectorof (listof (cons/c (or/c term? non-term?) action?))))))
;; A parse-table is (vectorof (listof (cons/c gram-sym? action))) ;; A parse-table is (vectorof (listof (cons/c gram-sym? action)))
;; A grouped-parse-table is (vectorof (listof (cons/c gram-sym? (listof action)))) ;; A grouped-parse-table is (vectorof (listof (cons/c gram-sym? (listof action))))
;; make-parse-table : int -> parse-table ;; make-parse-table : int -> parse-table
(define (make-parse-table num-states) (define (make-parse-table num-states)
(make-vector num-states null)) (make-vector num-states null))
;; table-add!: parse-table nat symbol action -> ;; table-add!: parse-table nat symbol action ->
(define (table-add! table state-index symbol val) (define (table-add! table state-index symbol val)
(vector-set! table state-index (cons (cons symbol val) (vector-set! table state-index (cons (cons symbol val)
(vector-ref table state-index)))) (vector-ref table state-index))))
;; group-table : parse-table -> grouped-parse-table ;; group-table : parse-table -> grouped-parse-table
(define (group-table table) (define (group-table table)
(list->vector (list->vector
(map (for/list ([state-entry (in-list (vector->list table))])
(lambda (state-entry) (define ht (make-hasheq))
(let ((ht (make-hash))) (for* ([gs/actions (in-list state-entry)]
(for-each [group (in-value (hash-ref ht (car gs/actions) (λ () null)))]
(lambda (gs/actions) #:unless (member (cdr gs/actions) group))
(let ((group (hash-ref ht (car gs/actions) (lambda () null)))) (hash-set! ht (car gs/actions) (cons (cdr gs/actions) group)))
(unless (member (cdr gs/actions) group) (hash-map ht cons))))
(hash-set! ht (car gs/actions) (cons (cdr gs/actions) group)))))
state-entry)
(hash-map ht cons)))
(vector->list table))))
;; table-map : (vectorof (listof (cons/c gram-sym? X))) (gram-sym? X -> Y) -> ;; table-map : (vectorof (listof (cons/c gram-sym? X))) (gram-sym? X -> Y) ->
;; (vectorof (listof (cons/c gram-sym? Y))) ;; (vectorof (listof (cons/c gram-sym? Y)))
(define (table-map f table) (define (table-map f table)
(list->vector (list->vector
(map (for/list ([state-entry (in-list (vector->list table))])
(lambda (state-entry) (for/list ([gs/X (in-list state-entry)])
(map (cons (car gs/X) (f (car gs/X) (cdr gs/X)))))))
(lambda (gs/X)
(cons (car gs/X) (f (car gs/X) (cdr gs/X))))
state-entry))
(vector->list table))))
(define (bit-vector-for-each f bv) (define (bit-vector-for-each f bv)
(letrec ((for-each (let loop ([bv bv] [number 0])
(lambda (bv number)
(cond (cond
((= 0 bv) (void)) [(zero? bv) (void)]
((= 1 (bitwise-and 1 bv)) [(= 1 (bitwise-and 1 bv))
(f number) (f number)
(for-each (arithmetic-shift bv -1) (add1 number))) (loop (arithmetic-shift bv -1) (add1 number))]
(else (for-each (arithmetic-shift bv -1) (add1 number))))))) [else (loop (arithmetic-shift bv -1) (add1 number))])))
(for-each bv 0)))
;; print-entry: symbol action output-port -> ;; print-entry: symbol action output-port ->
;; prints the action a for lookahead sym to the given port ;; prints the action a for lookahead sym to the given port
(define (print-entry sym a port) (define (print-entry sym a port)
(let ((s "\t~a\t\t\t\t\t~a\t~a\n")) (define s "\t~a\t\t\t\t\t~a\t~a\n")
(cond (cond
((shift? a) [(shift? a) (fprintf port s sym "shift" (shift-state a))]
(fprintf port s sym "shift" (shift-state a))) [(reduce? a) (fprintf port s sym "reduce" (prod-index (reduce-prod a)))]
((reduce? a) [(accept? a) (fprintf port s sym "accept" "")]
(fprintf port s sym "reduce" (prod-index (reduce-prod a)))) [(goto? a) (fprintf port s sym "goto" (goto-state a))]))
((accept? a)
(fprintf port s sym "accept" ""))
((goto? a)
(fprintf port s sym "goto" (goto-state a))))))
;; count: ('a -> bool) * 'a list -> num ;; count: ('a -> bool) * 'a list -> num
;; counts the number of elements in list that satisfy pred ;; counts the number of elements in list that satisfy pred
(define (count pred list) (define (count pred list)
(cond (cond
((null? list) 0) [(null? list) 0]
((pred (car list)) (+ 1 (count pred (cdr list)))) [(pred (car list)) (+ 1 (count pred (cdr list)))]
(else (count pred (cdr list))))) [else (count pred (cdr list))]))
;; display-parser: LR0-automaton grouped-parse-table (listof prod?) output-port -> ;; display-parser: LR0-automaton grouped-parse-table (listof prod?) output-port ->
;; Prints out the parser given by table. ;; Prints out the parser given by table.
(define (display-parser a grouped-table prods port) (define (display-parser a grouped-table prods port)
(let* ((SR-conflicts 0) (define SR-conflicts 0)
(RR-conflicts 0)) (define RR-conflicts 0)
(for-each (for ([prod (in-list prods)])
(lambda (prod)
(fprintf port (fprintf port
"~a\t~a\t=\t~a\n" "~a\t~a\t=\t~a\n"
(prod-index prod) (prod-index prod)
(gram-sym-symbol (prod-lhs prod)) (gram-sym-symbol (prod-lhs prod))
(map gram-sym-symbol (vector->list (prod-rhs prod))))) (map gram-sym-symbol (vector->list (prod-rhs prod)))))
prods)
(send a for-each-state (send a for-each-state
(lambda (state) (λ (state)
(fprintf port "State ~a\n" (kernel-index state)) (fprintf port "State ~a\n" (kernel-index state))
(for-each (lambda (item) (for ([item (in-list (kernel-items state))])
(fprintf port "\t~a\n" (item->string item))) (fprintf port "\t~a\n" (item->string item)))
(kernel-items state))
(newline port) (newline port)
(for-each (for ([gs/action (in-list (vector-ref grouped-table (kernel-index state)))])
(lambda (gs/action) (define sym (gram-sym-symbol (car gs/action)))
(let ((sym (gram-sym-symbol (car gs/action))) (define act (cdr gs/action))
(act (cdr gs/action)))
(cond (cond
((null? act) (void)) [(null? act) (void)]
((null? (cdr act)) [(null? (cdr act))
(print-entry sym (car act) port)) (print-entry sym (car act) port)]
(else [else
(fprintf port "begin conflict:\n") (fprintf port "begin conflict:\n")
(when (> (count reduce? act) 1) (when (> (count reduce? act) 1)
(set! RR-conflicts (add1 RR-conflicts))) (set! RR-conflicts (add1 RR-conflicts)))
(when (> (count shift? act) 0) (when (> (count shift? act) 0)
(set! SR-conflicts (add1 SR-conflicts))) (set! SR-conflicts (add1 SR-conflicts)))
(map (lambda (x) (print-entry sym x port)) act) (map (λ (x) (print-entry sym x port)) act)
(fprintf port "end conflict\n"))))) (fprintf port "end conflict\n")]))
(vector-ref grouped-table (kernel-index state)))
(newline port))) (newline port)))
(when (> SR-conflicts 0) (when (> SR-conflicts 0)
@ -136,47 +117,45 @@
(when (> RR-conflicts 0) (when (> RR-conflicts 0)
(fprintf port "~a reduce/reduce conflict~a\n" (fprintf port "~a reduce/reduce conflict~a\n"
RR-conflicts RR-conflicts
(if (= RR-conflicts 1) "" "s"))))) (if (= RR-conflicts 1) "" "s"))))
;; resolve-conflict : (listof action?) -> action? bool bool ;; resolve-conflict : (listof action?) -> action? bool bool
(define (resolve-conflict actions) (define (resolve-conflict actions)
(cond (cond
((null? actions) (values (make-no-action) #f #f)) [(null? actions) (values (make-no-action) #f #f)]
((null? (cdr actions)) [(null? (cdr actions)) (values (car actions) #f #f)]
(values (car actions) #f #f)) [else
(else (define SR-conflict? (> (count shift? actions) 0))
(let ((SR-conflict? (> (count shift? actions) 0)) (define RR-conflict? (> (count reduce? actions) 1))
(RR-conflict? (> (count reduce? actions) 1)))
(let loop ((current-guess #f) (let loop ((current-guess #f)
(rest actions)) (rest actions))
(cond (cond
((null? rest) (values current-guess SR-conflict? RR-conflict?)) [(null? rest) (values current-guess SR-conflict? RR-conflict?)]
((shift? (car rest)) (values (car rest) SR-conflict? RR-conflict?)) [(shift? (car rest)) (values (car rest) SR-conflict? RR-conflict?)]
((not current-guess) [(not current-guess) (loop (car rest) (cdr rest))]
(loop (car rest) (cdr rest))) [(and (reduce? (car rest))
((and (reduce? (car rest))
(< (prod-index (reduce-prod (car rest))) (< (prod-index (reduce-prod (car rest)))
(prod-index (reduce-prod current-guess)))) (prod-index (reduce-prod current-guess))))
(loop (car rest) (cdr rest))) (loop (car rest) (cdr rest))]
((accept? (car rest)) [(accept? (car rest))
(eprintf "accept/reduce or accept/shift conflicts. Check the grammar for useless cycles of productions\n") (eprintf "accept/reduce or accept/shift conflicts. Check the grammar for useless cycles of productions\n")
(loop current-guess (cdr rest))) (loop current-guess (cdr rest))]
(else (loop current-guess (cdr rest))))))))) [else (loop current-guess (cdr rest))]))]))
;; resolve-conflicts : grouped-parse-table bool -> parse-table ;; resolve-conflicts : grouped-parse-table bool -> parse-table
(define (resolve-conflicts grouped-table suppress) (define (resolve-conflicts grouped-table suppress)
(let* ((SR-conflicts 0) (define SR-conflicts 0)
(RR-conflicts 0) (define RR-conflicts 0)
(table (table-map (define table (table-map
(lambda (gs actions) (λ (gs actions)
(let-values (((action SR? RR?) (let-values ([(action SR? RR?)
(resolve-conflict actions))) (resolve-conflict actions)])
(when SR? (when SR?
(set! SR-conflicts (add1 SR-conflicts))) (set! SR-conflicts (add1 SR-conflicts)))
(when RR? (when RR?
(set! RR-conflicts (add1 RR-conflicts))) (set! RR-conflicts (add1 RR-conflicts)))
action)) action))
grouped-table))) grouped-table))
(unless suppress (unless suppress
(when (> SR-conflicts 0) (when (> SR-conflicts 0)
(eprintf "~a shift/reduce conflict~a\n" (eprintf "~a shift/reduce conflict~a\n"
@ -186,61 +165,60 @@
(eprintf "~a reduce/reduce conflict~a\n" (eprintf "~a reduce/reduce conflict~a\n"
RR-conflicts RR-conflicts
(if (= RR-conflicts 1) "" "s")))) (if (= RR-conflicts 1) "" "s"))))
table)) table)
;; resolve-sr-conflict : (listof action) (union int #f) -> (listof action) ;; resolve-sr-conflict : (listof action) (union int #f) -> (listof action)
;; Resolves a single shift-reduce conflict, if precedences are in place. ;; Resolves a single shift-reduce conflict, if precedences are in place.
(define (resolve-sr-conflict/prec actions shift-prec) (define (resolve-sr-conflict/prec actions shift-prec)
(let* ((shift (if (shift? (car actions)) (define shift (if (shift? (car actions))
(car actions) (car actions)
(cadr actions))) (cadr actions)))
(reduce (if (shift? (car actions)) (define reduce (if (shift? (car actions))
(cadr actions) (cadr actions)
(car actions))) (car actions)))
(reduce-prec (prod-prec (reduce-prod reduce)))) (define reduce-prec (prod-prec (reduce-prod reduce)))
(cond (cond
((and shift-prec reduce-prec) [(and shift-prec reduce-prec)
(cond (cond
((< (prec-num shift-prec) (prec-num reduce-prec)) [(< (prec-num shift-prec) (prec-num reduce-prec))
(list reduce)) (list reduce)]
((> (prec-num shift-prec) (prec-num reduce-prec)) [(> (prec-num shift-prec) (prec-num reduce-prec))
(list shift)) (list shift)]
((eq? 'left (prec-assoc shift-prec)) [(eq? 'left (prec-assoc shift-prec))
(list reduce)) (list reduce)]
((eq? 'right (prec-assoc shift-prec)) [(eq? 'right (prec-assoc shift-prec))
(list shift)) (list shift)]
(else null))) [else null])]
(else actions)))) [else actions]))
;; resolve-prec-conflicts : parse-table -> grouped-parse-table ;; resolve-prec-conflicts : parse-table -> grouped-parse-table
(define (resolve-prec-conflicts table) (define (resolve-prec-conflicts table)
(table-map (table-map
(lambda (gs actions) (λ (gs actions)
(cond (cond
((and (term? gs) [(and (term? gs)
(= 2 (length actions)) (= 2 (length actions))
(or (shift? (car actions)) (or (shift? (car actions))
(shift? (cadr actions)))) (shift? (cadr actions))))
(resolve-sr-conflict/prec actions (term-prec gs))) (resolve-sr-conflict/prec actions (term-prec gs))]
(else actions))) [else actions]))
(group-table table))) (group-table table)))
;; build-table: grammar string bool -> parse-table ;; build-table: grammar string bool -> parse-table
(define (build-table g file suppress) (define (build-table g file suppress)
(let* ((a (build-lr0-automaton g)) (define a (build-lr0-automaton g))
(term-vector (list->vector (send g get-terms))) (define term-vector (list->vector (send g get-terms)))
(end-terms (send g get-end-terms)) (define end-terms (send g get-end-terms))
(table (make-parse-table (send a get-num-states))) (define table (make-parse-table (send a get-num-states)))
(get-lookahead (compute-LA a g)) (define get-lookahead (compute-LA a g))
(reduce-cache (make-hash))) (define reduce-cache (make-hash))
(for ([trans-key/state (in-list (send a get-transitions))])
(define from-state-index (kernel-index (trans-key-st (car trans-key/state))))
(define gs (trans-key-gs (car trans-key/state)))
(define to-state (cdr trans-key/state))
(for-each
(lambda (trans-key/state)
(let ((from-state-index (kernel-index (trans-key-st (car trans-key/state))))
(gs (trans-key-gs (car trans-key/state)))
(to-state (cdr trans-key/state)))
(table-add! table from-state-index gs (table-add! table from-state-index gs
(cond (cond
((non-term? gs) ((non-term? gs)
@ -249,19 +227,19 @@
(make-accept)) (make-accept))
(else (else
(make-shift (make-shift
(kernel-index to-state))))))) (kernel-index to-state))))))
(send a get-transitions))
(send a for-each-state (send a for-each-state
(lambda (state) (λ (state)
(for-each (for ([item (in-list (append (hash-ref (send a get-epsilon-trans) state (λ () null))
(lambda (item) (filter (λ (item)
(let ((item-prod (item-prod item))) (not (move-dot-right item)))
(kernel-items state))))])
(let ([item-prod (item-prod item)])
(bit-vector-for-each (bit-vector-for-each
(lambda (term-index) (λ (term-index)
(unless (start-item? item) (unless (start-item? item)
(let ((r (hash-ref reduce-cache item-prod (let ((r (hash-ref reduce-cache item-prod
(lambda () (λ ()
(let ((r (make-reduce item-prod))) (let ((r (make-reduce item-prod)))
(hash-set! reduce-cache item-prod r) (hash-set! reduce-cache item-prod r)
r))))) r)))))
@ -269,22 +247,18 @@
(kernel-index state) (kernel-index state)
(vector-ref term-vector term-index) (vector-ref term-vector term-index)
r)))) r))))
(get-lookahead state item-prod)))) (get-lookahead state item-prod))))))
(append (hash-ref (send a get-epsilon-trans) state (lambda () null))
(filter (lambda (item)
(not (move-dot-right item)))
(kernel-items state))))))
(let ((grouped-table (resolve-prec-conflicts table))) (define grouped-table (resolve-prec-conflicts table))
(unless (string=? file "") (unless (string=? file "")
(with-handlers [(exn:fail:filesystem? (with-handlers [(exn:fail:filesystem?
(lambda (e) (λ (e)
(eprintf (eprintf
"Cannot write debug output to file \"~a\": ~a\n" "Cannot write debug output to file \"~a\": ~a\n"
file file
(exn-message e))))] (exn-message e))))]
(call-with-output-file file (call-with-output-file file
(lambda (port) (λ (port)
(display-parser a grouped-table (send g get-prods) port)) (display-parser a grouped-table (send g get-prods) port))
#:exists 'truncate))) #:exists 'truncate)))
(resolve-conflicts grouped-table suppress)))) (resolve-conflicts grouped-table suppress))

147
br-parser-tools-lib/br-parser-tools/private-yacc/yacc-helper.rkt
Unescape Escape View File

@ -1,118 +1,71 @@
(module yacc-helper mzscheme #lang racket/base
(require (prefix-in rl: racket/list)
(require mzlib/list
"../private-lex/token-syntax.rkt") "../private-lex/token-syntax.rkt")
;; General helper routines ;; General helper routines
(provide duplicate-list? remove-duplicates overlap? vector-andmap display-yacc)
(provide duplicate-list? remove-duplicates overlap? vector-andmap display-yacc)
(define (vector-andmap f v) (define (vector-andmap pred vec)
(let loop ((i 0)) (for/and ([item (in-vector vec)])
(cond (pred vec)))
((= i (vector-length v)) #t)
(else (if (f (vector-ref v i))
(loop (add1 i))
#f)))))
;; duplicate-list?: symbol list -> #f | symbol ;; duplicate-list?: symbol list -> #f | symbol
;; returns a symbol that exists twice in l, or false if no such symbol ;; returns a symbol that exists twice in l, or false if no such symbol
;; exists ;; exists
(define (duplicate-list? l) (define (duplicate-list? syms)
(letrec ((t (make-hash-table)) (rl:check-duplicates syms eq?))
(dl? (lambda (l)
(cond
((null? l) #f)
((hash-table-get t (car l) (lambda () #f)) =>
(lambda (x) x))
(else
(hash-table-put! t (car l) (car l))
(dl? (cdr l)))))))
(dl? l)))
;; remove-duplicates: syntax-object list -> syntax-object list ;; remove-duplicates: syntax-object list -> syntax-object list
;; removes the duplicates from the lists ;; removes the duplicates from the lists
(define (remove-duplicates sl) (define (remove-duplicates syms)
(let ((t (make-hash-table))) (rl:remove-duplicates syms equal? #:key syntax->datum))
(letrec ((x
(lambda (sl)
(cond
((null? sl) sl)
((hash-table-get t (syntax-object->datum (car sl)) (lambda () #f))
(x (cdr sl)))
(else
(hash-table-put! t (syntax-object->datum (car sl)) #t)
(cons (car sl) (x (cdr sl))))))))
(x sl))))
;; overlap?: symbol list * symbol list -> #f | symbol ;; overlap?: symbol list * symbol list -> #f | symbol
;; Returns an symbol in l1 intersect l2, or #f is no such symbol exists ;; Returns an symbol in l1 intersect l2, or #f is no such symbol exists
(define (overlap? l1 l2) (define (overlap? syms1 syms2)
(let/ec ret (for/first ([sym1 (in-list syms1)]
(let ((t (make-hash-table))) #:when (memq sym1 syms2))
(for-each (lambda (s1) sym1))
(hash-table-put! t s1 s1))
l1)
(for-each (lambda (s2)
(cond
((hash-table-get t s2 (lambda () #f)) =>
(lambda (o) (ret o)))))
l2)
#f)))
(define (display-yacc grammar tokens start precs port) (define (display-yacc grammar tokens start precs port)
(let-syntax ((p (syntax-rules () (let-syntax ([p (syntax-rules ()
((_ args ...) (fprintf port args ...))))) ((_ args ...) (fprintf port args ...)))])
(let* ((tokens (map syntax-local-value tokens)) (let* ([tokens (map syntax-local-value tokens)]
(eterms (filter e-terminals-def? tokens)) [eterms (filter e-terminals-def? tokens)]
(terms (filter terminals-def? tokens)) [terms (filter terminals-def? tokens)]
(term-table (make-hash-table)) [term-table (make-hasheq)]
(display-rhs [display-rhs
(lambda (rhs) (λ (rhs)
(for-each (lambda (sym) (p "~a " (hash-table-get term-table sym (lambda () sym)))) (for ([sym (in-list (car rhs))])
(car rhs)) (p "~a " (hash-ref term-table sym (λ () sym))))
(if (= 3 (length rhs)) (when (= 3 (length rhs))
(p "%prec ~a" (cadadr rhs))) (p "%prec ~a" (cadadr rhs)))
(p "\n")))) (p "\n"))])
(for-each (for* ([t (in-list eterms)]
(lambda (t) [t (in-list (syntax->datum (e-terminals-def-t t)))])
(for-each (hash-set! term-table t (format "'~a'" t)))
(lambda (t) (for* ([t (in-list terms)]
(hash-table-put! term-table t (format "'~a'" t))) [t (in-list (syntax->datum (terminals-def-t t)))])
(syntax-object->datum (e-terminals-def-t t))))
eterms)
(for-each
(lambda (t)
(for-each
(lambda (t)
(p "%token ~a\n" t) (p "%token ~a\n" t)
(hash-table-put! term-table t (format "~a" t))) (hash-set! term-table t (format "~a" t)))
(syntax-object->datum (terminals-def-t t)))) (when precs
terms) (for ([prec (in-list precs)])
(if precs
(for-each (lambda (prec)
(p "%~a " (car prec)) (p "%~a " (car prec))
(for-each (lambda (tok) (for ([tok (in-list (cdr prec))])
(p " ~a" (hash-table-get term-table tok))) (p " ~a" (hash-ref term-table tok)))
(cdr prec)) (p "\n")))
(p "\n"))
precs))
(p "%start ~a\n" start) (p "%start ~a\n" start)
(p "%%\n") (p "%%\n")
(for ([prod (in-list grammar)])
(for-each (lambda (prod) (define nt (car prod))
(let ((nt (car prod)))
(p "~a: " nt) (p "~a: " nt)
(display-rhs (cadr prod)) (display-rhs (cadr prod))
(for-each (lambda (rhs) (for ([rhs (in-list (cddr prod))])
(p "| ") (p "| ")
(display-rhs rhs)) (display-rhs rhs))
(cddr prod)) (p ";\n"))
(p ";\n")))
grammar)
(p "%%\n")))) (p "%%\n"))))
)

121
br-parser-tools-lib/br-parser-tools/yacc-to-scheme.rkt
Unescape Escape View File

@ -1,55 +1,53 @@
(module yacc-to-scheme mzscheme #lang racket/base
(require br-parser-tools/lex (require br-parser-tools/lex
(prefix : br-parser-tools/lex-sre) (prefix-in : br-parser-tools/lex-sre)
br-parser-tools/yacc br-parser-tools/yacc
syntax/readerr syntax/readerr
mzlib/list) racket/list)
(provide trans) (provide trans)
(define match-double-string (define match-double-string
(lexer (lexer
((:+ (:~ #\" #\\)) (append (string->list lexeme) [(:+ (:~ #\" #\\)) (append (string->list lexeme)
(match-double-string input-port))) (match-double-string input-port))]
((:: #\\ any-char) (cons (string-ref lexeme 1) (match-double-string input-port))) [(:: #\\ any-char) (cons (string-ref lexeme 1) (match-double-string input-port))]
(#\" null))) [#\" null]))
(define match-single-string (define match-single-string
(lexer (lexer
((:+ (:~ #\' #\\)) (append (string->list lexeme) [(:+ (:~ #\' #\\)) (append (string->list lexeme)
(match-single-string input-port))) (match-single-string input-port))]
((:: #\\ any-char) (cons (string-ref lexeme 1) (match-single-string input-port))) [(:: #\\ any-char) (cons (string-ref lexeme 1) (match-single-string input-port))]
(#\' null))) [#\' null]))
(define-lex-abbrevs (define-lex-abbrevs
(letter (:or (:/ "a" "z") (:/ "A" "Z"))) [letter (:or (:/ "a" "z") (:/ "A" "Z"))]
(digit (:/ "0" "9")) [digit (:/ "0" "9")]
(initial (:or letter (char-set "!$%&*/<=>?^_~@"))) [initial (:or letter (char-set "!$%&*/<=>?^_~@"))]
(subsequent (:or initial digit (char-set "+-.@"))) [subsequent (:or initial digit (char-set "+-.@"))]
(comment (:: "/*" (complement (:: any-string "*/" any-string)) "*/"))) [comment (:: "/*" (complement (:: any-string "*/" any-string)) "*/")])
(define-empty-tokens x (define-empty-tokens x (EOF PIPE |:| SEMI |%%| %prec))
(EOF PIPE |:| SEMI |%%| %prec)) (define-tokens y (SYM STRING))
(define-tokens y
(SYM STRING))
(define get-token-grammar (define get-token-grammar
(lexer-src-pos (lexer-src-pos
("%%" '|%%|) ["%%" '|%%|]
(":" (string->symbol lexeme)) [":" (string->symbol lexeme)]
("%prec" (string->symbol lexeme)) ["%prec" (string->symbol lexeme)]
(#\| 'PIPE) [#\| 'PIPE]
((:+ (:or #\newline #\tab " " comment (:: "{" (:* (:~ "}")) "}"))) [(:+ (:or #\newline #\tab " " comment (:: "{" (:* (:~ "}")) "}")))
(return-without-pos (get-token-grammar input-port))) (return-without-pos (get-token-grammar input-port))]
(#\; 'SEMI) [#\; 'SEMI]
(#\' (token-STRING (string->symbol (list->string (match-single-string input-port))))) [#\' (token-STRING (string->symbol (list->string (match-single-string input-port))))]
(#\" (token-STRING (string->symbol (list->string (match-double-string input-port))))) [#\" (token-STRING (string->symbol (list->string (match-double-string input-port))))]
((:: initial (:* subsequent)) (token-SYM (string->symbol lexeme))))) [(:: initial (:* subsequent)) (token-SYM (string->symbol lexeme))]))
(define (parse-grammar enter-term enter-empty-term enter-non-term) (define (parse-grammar enter-term enter-empty-term enter-non-term)
(parser (parser
(tokens x y) (tokens x y)
(src-pos) (src-pos)
(error (lambda (tok-ok tok-name tok-value start-pos end-pos) (error (λ (tok-ok tok-name tok-value start-pos end-pos)
(raise-read-error (raise-read-error
(format "Error Parsing YACC grammar at token: ~a with value: ~a" tok-name tok-value) (format "Error Parsing YACC grammar at token: ~a with value: ~a" tok-name tok-value)
(file-path) (file-path)
@ -94,42 +92,39 @@
(enter-empty-term $1) (enter-empty-term $1)
(cons $1 $2))))))) (cons $1 $2)))))))
(define (symbol<? a b) (define (symbol<? a b)
(string<? (symbol->string a) (symbol->string b))) (string<? (symbol->string a) (symbol->string b)))
(define (trans filename) (define (trans filename)
(let* ((i (open-input-file filename)) (define i (open-input-file filename))
(terms (make-hash-table)) (define terms (make-hasheq))
(eterms (make-hash-table)) (define eterms (make-hasheq))
(nterms (make-hash-table)) (define nterms (make-hasheq))
(enter-term (define (enter-term s)
(lambda (s) (when (not (hash-ref nterms s (λ () #f)))
(if (not (hash-table-get nterms s (lambda () #f))) (hash-set! terms s #t)))
(hash-table-put! terms s #t)))) (define (enter-empty-term s)
(enter-empty-term (when (not (hash-ref nterms s (λ () #f)))
(lambda (s) (hash-set! eterms s #t)))
(if (not (hash-table-get nterms s (lambda () #f))) (define (enter-non-term s)
(hash-table-put! eterms s #t)))) (hash-remove! terms s)
(enter-non-term (hash-remove! eterms s)
(lambda (s) (hash-set! nterms s #t))
(hash-table-remove! terms s)
(hash-table-remove! eterms s)
(hash-table-put! nterms s #t))))
(port-count-lines! i) (port-count-lines! i)
(file-path filename) (file-path filename)
(regexp-match "%%" i) (regexp-match "%%" i)
(begin0 (begin0
(let ((gram ((parse-grammar enter-term enter-empty-term enter-non-term) (let ([gram ((parse-grammar enter-term enter-empty-term enter-non-term)
(lambda () (λ ()
(let ((t (get-token-grammar i))) (let ((t (get-token-grammar i)))
t))))) t)))])
`(begin `(begin
(define-tokens t ,(sort (hash-table-map terms (lambda (k v) k)) symbol<?)) (define-tokens t ,(sort (hash-map terms (λ (k v) k)) symbol<?))
(define-empty-tokens et ,(sort (hash-table-map eterms (lambda (k v) k)) symbol<?)) (define-empty-tokens et ,(sort (hash-map eterms (λ (k v) k)) symbol<?))
(parser (parser
(start ___) (start ___)
(end ___) (end ___)
(error ___) (error ___)
(tokens t et) (tokens t et)
(grammar ,@gram)))) (grammar ,@gram))))
(close-input-port i))))) (close-input-port i)))

350
br-parser-tools-lib/br-parser-tools/yacc.rkt
Unescape Escape View File

@ -1,14 +1,13 @@
#lang scheme/base #lang racket/base
(require (for-syntax racket/base
(require (for-syntax scheme/base
"private-yacc/parser-builder.rkt" "private-yacc/parser-builder.rkt"
"private-yacc/grammar.rkt" "private-yacc/grammar.rkt"
"private-yacc/yacc-helper.rkt" "private-yacc/yacc-helper.rkt"
"private-yacc/parser-actions.rkt")) "private-yacc/parser-actions.rkt")
(require "private-lex/token.rkt" "private-lex/token.rkt"
"private-yacc/parser-actions.rkt" "private-yacc/parser-actions.rkt"
mzlib/etc racket/local
mzlib/pretty racket/pretty
syntax/readerr) syntax/readerr)
(provide parser) (provide parser)
@ -17,139 +16,93 @@
;; convert-parse-table : (vectorof (listof (cons/c gram-sym? action?))) -> ;; convert-parse-table : (vectorof (listof (cons/c gram-sym? action?))) ->
;; (vectorof (symbol runtime-action hashtable)) ;; (vectorof (symbol runtime-action hashtable))
(define-for-syntax (convert-parse-table table) (define-for-syntax (convert-parse-table table)
(list->vector (for/vector ([state-entry (in-vector table)])
(map (let ([ht (make-hasheq)])
(lambda (state-entry) (for ([gs/action (in-list state-entry)])
(let ((ht (make-hasheq)))
(for-each
(lambda (gs/action)
(hash-set! ht (hash-set! ht
(gram-sym-symbol (car gs/action)) (gram-sym-symbol (car gs/action))
(action->runtime-action (cdr gs/action)))) (action->runtime-action (cdr gs/action))))
state-entry) ht)))
ht))
(vector->list table))))
(define-syntax (parser stx) (define-syntax (parser stx)
(syntax-case stx () (syntax-case stx ()
((_ args ...) [(_ ARGS ...)
(let ((arg-list (syntax->list (syntax (args ...)))) (let ([arg-list (syntax->list #'(ARGS ...))]
(src-pos #f) [src-pos #f]
(debug #f) [debug #f]
(error #f) [error #f]
(tokens #f) [tokens #f]
(start #f) [start #f]
(end #f) [end #f]
(precs #f) [precs #f]
(suppress #f) [suppress #f]
(grammar #f) [grammar #f]
(yacc-output #f)) [yacc-output #f])
(for-each (for ([arg (in-list (syntax->list #'(ARGS ...)))])
(lambda (arg)
(syntax-case* arg (debug error tokens start end precs grammar (syntax-case* arg (debug error tokens start end precs grammar
suppress src-pos yacc-output) suppress src-pos yacc-output)
(lambda (a b) (λ (a b) (eq? (syntax-e a) (syntax-e b)))
(eq? (syntax-e a) (syntax-e b))) [(debug FILENAME)
((debug filename)
(cond (cond
((not (string? (syntax-e (syntax filename)))) [(not (string? (syntax-e #'FILENAME)))
(raise-syntax-error (raise-syntax-error #f "Debugging filename must be a string" stx #'FILENAME)]
#f [debug (raise-syntax-error #f "Multiple debug declarations" stx)]
"Debugging filename must be a string" [else (set! debug (syntax-e #'FILENAME))])]
stx [(suppress) (set! suppress #t)]
(syntax filename))) [(src-pos) (set! src-pos #t)]
(debug [(error EXPRESSION)
(raise-syntax-error #f "Multiple debug declarations" stx))
(else
(set! debug (syntax-e (syntax filename))))))
((suppress)
(set! suppress #t))
((src-pos)
(set! src-pos #t))
((error expression)
(if error (if error
(raise-syntax-error #f "Multiple error declarations" stx) (raise-syntax-error #f "Multiple error declarations" stx)
(set! error (syntax expression)))) (set! error #'EXPRESSION))]
((tokens def ...) [(tokens DEF ...)
(begin (begin
(when tokens (when tokens
(raise-syntax-error #f "Multiple tokens declarations" stx)) (raise-syntax-error #f "Multiple tokens declarations" stx))
(let ((defs (syntax->list (syntax (def ...))))) (let ((defs (syntax->list #'(DEF ...))))
(for-each (for ([d (in-list defs)]
(lambda (d) #:unless (identifier? d))
(unless (identifier? d) (raise-syntax-error #f "Token-group name must be an identifier" stx d))
(raise-syntax-error (set! tokens defs)))]
#f [(start symbol ...)
"Token-group name must be an identifier" (let ([symbols (syntax->list #'(symbol ...))])
stx (for ([sym (in-list symbols)]
d))) #:unless (identifier? sym))
defs) (raise-syntax-error #f "Start symbol must be a symbol" stx sym))
(set! tokens defs))))
((start symbol ...)
(let ((symbols (syntax->list (syntax (symbol ...)))))
(for-each
(lambda (sym)
(unless (identifier? sym)
(raise-syntax-error #f
"Start symbol must be a symbol"
stx
sym)))
symbols)
(when start (when start
(raise-syntax-error #f "Multiple start declarations" stx)) (raise-syntax-error #f "Multiple start declarations" stx))
(when (null? symbols) (when (null? symbols)
(raise-syntax-error #f (raise-syntax-error #f "Missing start symbol" stx arg))
"Missing start symbol" (set! start symbols))]
stx [(end SYMBOLS ...)
arg)) (let ((symbols (syntax->list #'(SYMBOLS ...))))
(set! start symbols))) (for ([sym (in-list symbols)]
((end symbols ...) #:unless (identifier? sym))
(let ((symbols (syntax->list (syntax (symbols ...))))) (raise-syntax-error #f "End token must be a symbol" stx sym))
(for-each (let ([d (duplicate-list? (map syntax-e symbols))])
(lambda (sym)
(unless (identifier? sym)
(raise-syntax-error #f
"End token must be a symbol"
stx
sym)))
symbols)
(let ((d (duplicate-list? (map syntax-e symbols))))
(when d (when d
(raise-syntax-error (raise-syntax-error #f (format "Duplicate end token definition for ~a" d) stx arg))
#f
(format "Duplicate end token definition for ~a" d)
stx
arg))
(when (null? symbols) (when (null? symbols)
(raise-syntax-error (raise-syntax-error #f "end declaration must contain at least 1 token" stx arg))
#f
"end declaration must contain at least 1 token"
stx
arg))
(when end (when end
(raise-syntax-error #f "Multiple end declarations" stx)) (raise-syntax-error #f "Multiple end declarations" stx))
(set! end symbols)))) (set! end symbols)))]
((precs decls ...) [(precs DECLS ...)
(if precs (if precs
(raise-syntax-error #f "Multiple precs declarations" stx) (raise-syntax-error #f "Multiple precs declarations" stx)
(set! precs (syntax/loc arg (decls ...))))) (set! precs (syntax/loc arg (DECLS ...))))]
((grammar prods ...) [(grammar PRODS ...)
(if grammar (if grammar
(raise-syntax-error #f "Multiple grammar declarations" stx) (raise-syntax-error #f "Multiple grammar declarations" stx)
(set! grammar (syntax/loc arg (prods ...))))) (set! grammar (syntax/loc arg (PRODS ...))))]
((yacc-output filename) [(yacc-output FILENAME)
(cond (cond
((not (string? (syntax-e (syntax filename)))) [(not (string? (syntax-e #'FILENAME)))
(raise-syntax-error #f (raise-syntax-error #f "Yacc-output filename must be a string" stx #'FILENAME)]
"Yacc-output filename must be a string" [yacc-output
stx (raise-syntax-error #f "Multiple yacc-output declarations" stx)]
(syntax filename))) [else
(yacc-output (set! yacc-output (syntax-e #'FILENAME))])]
(raise-syntax-error #f "Multiple yacc-output declarations" stx)) [_ (raise-syntax-error #f "argument must match (debug filename), (error expression), (tokens def ...), (start non-term), (end tokens ...), (precs decls ...), or (grammar prods ...)" stx arg)]))
(else
(set! yacc-output (syntax-e (syntax filename))))))
(_ (raise-syntax-error #f "argument must match (debug filename), (error expression), (tokens def ...), (start non-term), (end tokens ...), (precs decls ...), or (grammar prods ...)" stx arg))))
(syntax->list (syntax (args ...))))
(unless tokens (unless tokens
(raise-syntax-error #f "missing tokens declaration" stx)) (raise-syntax-error #f "missing tokens declaration" stx))
(unless error (unless error
@ -160,7 +113,7 @@
(raise-syntax-error #f "missing end declaration" stx)) (raise-syntax-error #f "missing end declaration" stx))
(unless start (unless start
(raise-syntax-error #f "missing start declaration" stx)) (raise-syntax-error #f "missing start declaration" stx))
(let-values (((table all-term-syms actions check-syntax-fix) (let-values ([(table all-term-syms actions check-syntax-fix)
(build-parser (if debug debug "") (build-parser (if debug debug "")
src-pos src-pos
suppress suppress
@ -168,66 +121,51 @@
start start
end end
precs precs
grammar))) grammar)])
(when (and yacc-output (not (string=? yacc-output ""))) (when (and yacc-output (not (string=? yacc-output "")))
(with-handlers [(exn:fail:filesystem? (with-handlers [(exn:fail:filesystem?
(lambda (e) (λ (e) (eprintf "Cannot write yacc-output to file \"~a\"\n" yacc-output)))]
(eprintf
"Cannot write yacc-output to file \"~a\"\n"
yacc-output)))]
(call-with-output-file yacc-output (call-with-output-file yacc-output
(lambda (port) (λ (port)
(display-yacc (syntax->datum grammar) (display-yacc (syntax->datum grammar)
tokens tokens
(map syntax->datum start) (map syntax->datum start)
(if precs (and precs (syntax->datum precs))
(syntax->datum precs)
#f)
port)) port))
#:exists 'truncate))) #:exists 'truncate)))
(with-syntax ((check-syntax-fix check-syntax-fix) (with-syntax ([check-syntax-fix check-syntax-fix]
(err error) [err error]
(ends end) [ends end]
(starts start) [starts start]
(debug debug) [debug debug]
(table (convert-parse-table table)) [table (convert-parse-table table)]
(all-term-syms all-term-syms) [all-term-syms all-term-syms]
(actions actions) [actions actions]
(src-pos src-pos)) [src-pos src-pos])
(syntax #'(begin
(begin
check-syntax-fix check-syntax-fix
(parser-body debug err (quote starts) (quote ends) table all-term-syms actions src-pos))))))) (parser-body debug err (quote starts) (quote ends) table all-term-syms actions src-pos)))))]
(_ [_ (raise-syntax-error #f "parser must have the form (parser args ...)" stx)]))
(raise-syntax-error #f
"parser must have the form (parser args ...)"
stx))))
(define (reduce-stack stack num ret-vals src-pos) (define (reduce-stack stack num ret-vals src-pos)
(cond (cond
((> num 0) [(positive? num)
(let* ((top-frame (car stack)) (define top-frame (car stack))
(ret-vals (let ([ret-vals (if src-pos
(if src-pos
(cons (stack-frame-value top-frame) (cons (stack-frame-value top-frame)
(cons (stack-frame-start-pos top-frame) (cons (stack-frame-start-pos top-frame)
(cons (stack-frame-end-pos top-frame) (cons (stack-frame-end-pos top-frame)
ret-vals))) ret-vals)))
(cons (stack-frame-value top-frame) ret-vals)))) (cons (stack-frame-value top-frame) ret-vals))])
(reduce-stack (cdr stack) (sub1 num) ret-vals src-pos))) (reduce-stack (cdr stack) (sub1 num) ret-vals src-pos))]
(else (values stack ret-vals)))) [else (values stack ret-vals)]))
;; extract-helper : (symbol or make-token) any any -> symbol any any any ;; extract-helper : (symbol or make-token) any any -> symbol any any any
(define (extract-helper tok v1 v2) (define (extract-helper tok v1 v2)
(cond (cond
((symbol? tok) [(symbol? tok) (values tok #f v1 v2)]
(values tok #f v1 v2)) [(token? tok) (values (real-token-name tok) (real-token-value tok) v1 v2)]
((token? tok) [else (raise-argument-error 'parser "(or/c symbol? token?)" 0 tok)]))
(values (real-token-name tok) (real-token-value tok) v1 v2))
(else (raise-argument-error 'parser
"(or/c symbol? token?)"
0
tok))))
;; well-formed-position-token?: any -> boolean ;; well-formed-position-token?: any -> boolean
;; Returns true if pt is a position token whose position-token-token ;; Returns true if pt is a position token whose position-token-token
@ -236,8 +174,7 @@
;; a tokenizer produces an erroneous position-token wrapped twice. ;; a tokenizer produces an erroneous position-token wrapped twice.
;; (as often happens when omitting return-without-pos). ;; (as often happens when omitting return-without-pos).
(define (well-formed-token-field? t) (define (well-formed-token-field? t)
(or (symbol? t) (or (symbol? t) (token? t)))
(token? t)))
(define (well-formed-position-token? pt) (define (well-formed-position-token? pt)
(and (position-token? pt) (and (position-token? pt)
@ -250,24 +187,18 @@
;; extract-src-pos : position-token -> symbol any any any ;; extract-src-pos : position-token -> symbol any any any
(define (extract-src-pos ip) (define (extract-src-pos ip)
(unless (well-formed-position-token? ip) (unless (well-formed-position-token? ip)
(raise-argument-error 'parser (raise-argument-error 'parser "well-formed-position-token?" 0 ip))
"well-formed-position-token?"
0
ip))
(extract-helper (position-token-token ip) (extract-helper (position-token-token ip)
(position-token-start-pos ip) (position-token-start-pos ip)
(position-token-end-pos ip))) (position-token-end-pos ip)))
(define (extract-srcloc ip) (define (extract-srcloc ip)
(unless (well-formed-srcloc-token? ip) (unless (well-formed-srcloc-token? ip)
(raise-argument-error 'parser (raise-argument-error 'parser "well-formed-srcloc-token?" 0 ip))
"well-formed-srcloc-token?" (define loc (srcloc-token-srcloc ip))
0
ip))
(let ([loc (srcloc-token-srcloc ip)])
(extract-helper (srcloc-token-token ip) (extract-helper (srcloc-token-token ip)
(position-token (srcloc-position loc) (srcloc-line loc) (srcloc-column loc)) (position-token (srcloc-position loc) (srcloc-line loc) (srcloc-column loc))
(position-token (+ (srcloc-position loc) (srcloc-span loc)) #f #f)))) (position-token (+ (srcloc-position loc) (srcloc-span loc)) #f #f)))
;; extract-no-src-pos : (symbol or make-token) -> symbol any any any ;; extract-no-src-pos : (symbol or make-token) -> symbol any any any
@ -295,24 +226,24 @@
(if (memq tok ends) (if (memq tok ends)
(raise-read-error "parser: Cannot continue after error" (raise-read-error "parser: Cannot continue after error"
#f #f #f #f #f) #f #f #f #f #f)
(let ((a (find-action stack tok val start-pos end-pos))) (let ([a (find-action stack tok val start-pos end-pos)])
(cond (cond
((runtime-shift? a) [(runtime-shift? a)
;; (printf "shift:~a\n" (runtime-shift-state a)) ;; (printf "shift:~a\n" (runtime-shift-state a))
(cons (make-stack-frame (runtime-shift-state a) (cons (make-stack-frame (runtime-shift-state a)
val val
start-pos start-pos
end-pos) end-pos)
stack)) stack)]
(else [else
;; (printf "discard input:~a\n" tok) ;; (printf "discard input:~a\n" tok)
(let-values (((tok val start-pos end-pos) (let-values ([(tok val start-pos end-pos)
(extract (get-token)))) (extract (get-token))])
(remove-input tok val start-pos end-pos)))))))) (remove-input tok val start-pos end-pos))])))))
(let remove-states () (let remove-states ()
(let ((a (find-action stack 'error #f start-pos end-pos))) (let ([a (find-action stack 'error #f start-pos end-pos)])
(cond (cond
((runtime-shift? a) [(runtime-shift? a)
;; (printf "shift:~a\n" (runtime-shift-state a)) ;; (printf "shift:~a\n" (runtime-shift-state a))
(set! stack (set! stack
(cons (cons
@ -321,61 +252,55 @@
start-pos start-pos
end-pos) end-pos)
stack)) stack))
(remove-input tok val start-pos end-pos)) (remove-input tok val start-pos end-pos)]
(else [else
;; (printf "discard state:~a\n" (car stack)) ;; (printf "discard state:~a\n" (car stack))
(cond (cond
((< (length stack) 2) [(< (length stack) 2)
(raise-read-error "parser: Cannot continue after error" (raise-read-error "parser: Cannot continue after error"
#f #f #f #f #f)) #f #f #f #f #f)]
(else [else
(set! stack (cdr stack)) (set! stack (cdr stack))
(remove-states))))))))) (remove-states)])])))))
(define (find-action stack tok val start-pos end-pos) (define (find-action stack tok val start-pos end-pos)
(unless (hash-ref all-term-syms (unless (hash-ref all-term-syms tok #f)
tok
#f)
(if src-pos (if src-pos
(err #f tok val start-pos end-pos) (err #f tok val start-pos end-pos)
(err #f tok val)) (err #f tok val))
(raise-read-error (format "parser: got token of unknown type ~a" tok) (raise-read-error (format "parser: got token of unknown type ~a" tok)
#f #f #f #f #f)) #f #f #f #f #f))
(hash-ref (vector-ref table (stack-frame-state (car stack))) (hash-ref (vector-ref table (stack-frame-state (car stack))) tok #f))
tok
#f))
(define (make-parser start-number) (define ((make-parser start-number) get-token)
(lambda (get-token)
(unless (and (procedure? get-token) (unless (and (procedure? get-token)
(procedure-arity-includes? get-token 0)) (procedure-arity-includes? get-token 0))
(error 'get-token "expected a nullary procedure, got ~e" get-token)) (error 'get-token "expected a nullary procedure, got ~e" get-token))
(let parsing-loop ((stack (make-empty-stack start-number)) (let parsing-loop ([stack (make-empty-stack start-number)]
(ip (get-token))) [ip (get-token)])
(let-values (((tok val start-pos end-pos) (let-values ([(tok val start-pos end-pos) (extract ip)])
(extract ip))) (let ([action (find-action stack tok val start-pos end-pos)])
(let ((action (find-action stack tok val start-pos end-pos)))
(cond (cond
((runtime-shift? action) [(runtime-shift? action)
;; (printf "shift:~a\n" (runtime-shift-state action)) ;; (printf "shift:~a\n" (runtime-shift-state action))
(parsing-loop (cons (make-stack-frame (runtime-shift-state action) (parsing-loop (cons (make-stack-frame (runtime-shift-state action)
val val
start-pos start-pos
end-pos) end-pos)
stack) stack)
(get-token))) (get-token))]
((runtime-reduce? action) [(runtime-reduce? action)
;; (printf "reduce:~a\n" (runtime-reduce-prod-num action)) ;; (printf "reduce:~a\n" (runtime-reduce-prod-num action))
(let-values (((new-stack args) (let-values ([(new-stack args)
(reduce-stack stack (reduce-stack stack
(runtime-reduce-rhs-length action) (runtime-reduce-rhs-length action)
null null
src-pos))) src-pos)])
(let ((goto (let ([goto
(runtime-goto-state (runtime-goto-state
(hash-ref (hash-ref
(vector-ref table (stack-frame-state (car new-stack))) (vector-ref table (stack-frame-state (car new-stack)))
(runtime-reduce-lhs action))))) (runtime-reduce-lhs action)))])
(parsing-loop (parsing-loop
(cons (cons
(if src-pos (if src-pos
@ -392,21 +317,18 @@
#f #f
#f)) #f))
new-stack) new-stack)
ip)))) ip)))]
((runtime-accept? action) [(runtime-accept? action)
;; (printf "accept\n") ;; (printf "accept\n")
(stack-frame-value (car stack))) (stack-frame-value (car stack))]
(else [else
(if src-pos (if src-pos
(err #t tok val start-pos end-pos) (err #t tok val start-pos end-pos)
(err #t tok val)) (err #t tok val))
(parsing-loop (fix-error stack tok val start-pos end-pos get-token) (parsing-loop (fix-error stack tok val start-pos end-pos get-token)
(get-token)))))))))) (get-token))]))))))
(cond
((null? (cdr starts)) (make-parser 0))
(else
(let loop ((l starts)
(i 0))
(cond (cond
((null? l) null) [(null? (cdr starts)) (make-parser 0)]
(else (cons (make-parser i) (loop (cdr l) (add1 i)))))))))) [else
(for/list ([(l i) (in-indexed starts)])
(make-parser i))])))

2
br-parser-tools-lib/info.rkt
Unescape Escape View File

@ -7,5 +7,3 @@
(define build-deps '("rackunit-lib")) (define build-deps '("rackunit-lib"))
(define pkg-desc "implementation (no documentation) part of \"br-parser-tools\"") (define pkg-desc "implementation (no documentation) part of \"br-parser-tools\"")
(define pkg-authors '(mflatt))

Write Preview
Loading…
Cancel
Save