beautiful-racket/beautiful-racket-ragg/br/ragg/codegen/codegen.rkt

#lang br

(require (for-template racket/base)
         racket/list
         racket/set
         racket/syntax
         syntax/srcloc
         br/ragg/rules/stx-types
         "flatten.rkt"
         syntax/id-table
         (prefix-in sat: "satisfaction.rkt")
         (prefix-in support: br/ragg/support)
         (prefix-in stxparse: syntax/parse))

(provide rules-codegen)


;; Generates the body of the module.
;; FIXME: abstract this so we can just call (rules ...) without
;; generating the whole module body.
(define (rules-codegen stx 
                       #:parser-provider-module [parser-provider-module 'parser-tools/yacc]
                       #:parser-provider-form [parser-provider-form 'parser])
  (syntax-case stx ()
    [(_  r ...)
     (begin
       ;; (listof stx)
       (define rules (syntax->list #'(r ...)))
       
       (when (empty? rules)
         (raise-syntax-error 'ragg
                             (format "The grammar does not appear to have any rules")
                             stx))
       
       (check-all-rules-defined! rules)
       (check-all-rules-no-duplicates! rules)
       (check-all-rules-satisfiable! rules)
       
       ;; We flatten the rules so we can use the yacc-style ruleset that parser-tools
       ;; supports.
       
       #|
       MB: `rules` still carries 'hide syntax property
       |#
       #;(report flattened-rules)


       #|
       MB: `flattened-rules` still carries 'hide syntax property
       |#
       (define flattened-rules (flatten-rules rules))
       #;(report flattened-rules)
       
       (define generated-rule-codes (map flat-rule->yacc-rule flattened-rules))

       #|
       MB: `generated-rule-codes` loses the 'hide syntax property
       |#
       #;(report generated-rule-codes)
       
       ;; The first rule, by default, is the start rule.
       (define rule-ids (for/list ([a-rule (in-list rules)])
                                  (rule-id a-rule)))
       (define start-id (first rule-ids))
       
       
       (define-values (implicit-tokens    ;; (listof identifier)
                       explicit-tokens)   ;; (listof identifier)
         (rules-collect-token-types rules))
       
       ;; (listof symbol)
       (define implicit-token-types
         (map string->symbol
              (set->list (list->set (map syntax-e implicit-tokens)))))
       
       ;; (listof symbol)
       (define explicit-token-types
         (set->list (list->set (map syntax-e explicit-tokens))))
       
       ;; (listof symbol)
       (define token-types
         (set->list (list->set (append (map (lambda (x) (string->symbol (syntax-e x)))
                                            implicit-tokens)
                                       (map syntax-e explicit-tokens)))))
       
       (with-syntax ([start-id start-id]
                     
                     [(token-type ...) token-types]
                     
                     [(token-type-constructor ...)
                      (map (lambda (x) (string->symbol (format "token-~a" x)))
                           token-types)]
                     
                     [(explicit-token-types ...) explicit-token-types]
                     [(implicit-token-types ...) implicit-token-types]
                     [(implicit-token-types-str ...) (map symbol->string implicit-token-types)]
                     [(implicit-token-type-constructor ...)
                      (map (lambda (x) (string->symbol (format "token-~a" x)))
                           implicit-token-types)]
                     [generated-grammar #`(grammar #,@generated-rule-codes)]
                     [parser-module parser-provider-module]
                     [parser-form parser-provider-form])
         (quasisyntax/loc stx
           (begin             
             (require parser-tools/lex
                      parser-module
                      br/ragg/codegen/runtime
                      br/ragg/support
                      br/ragg/private/internal-support
                      racket/set
                      (for-syntax syntax/parse racket/base))
             
             (provide parse
                      make-rule-parser
                      all-token-types
                      #;current-source
                      #;current-parser-error-handler
                      #;current-tokenizer-error-handler
                      #;[struct-out exn:fail:parsing]
                      )
             
             (define-tokens enumerated-tokens (token-type ...))
             
             ;; all-token-types lists all the tokens (except for EOF)
             (define all-token-types 
               (set-remove (set 'token-type ...) 'EOF))
             
             ;; For internal use by the permissive tokenizer only:
             (define all-tokens-hash/mutable
               (make-hash (list ;; Note: we also allow the eof object here, to make
                           ;; the permissive tokenizer even nicer to work with.
                           (cons eof token-EOF) 
                           (cons 'token-type token-type-constructor) ...)))
             
             
             #;(define default-lex/1
                 (lexer-src-pos [implicit-token-types-str
                                 (token 'implicit-token-types lexeme)]
                                ...
                                [(eof) (token eof)]))
             
             (define-syntax (make-rule-parser stx-2)
               (syntax-parse stx-2
                 [(_ start-rule:id)
                  (begin
                    ;; HACK HACK HACK
                    ;; The cfg-parser depends on the start-rule provided in (start ...) to have the same
                    ;; context as the rest of this body, so I need to hack this.  I don't like this, but
                    ;; I don't know what else to do.  Hence recolored-start-rule.
                    (unless (member (syntax-e #'start-rule)
                                    '#,(map syntax-e rule-ids))
                      (raise-syntax-error #f
                                          (format "Rule ~a is not defined in the grammar" (syntax-e #'start-rule))
                                          stx-2))
                    
                    (define recolored-start-rule (datum->syntax (syntax #,stx) (syntax-e #'start-rule)))
                    #`(let ([THE-GRAMMAR (parser-form (tokens enumerated-tokens)
                                                      (src-pos)
                                                      (start #,recolored-start-rule)
                                                      (end EOF)
                                                      (error THE-ERROR-HANDLER)
                                                      generated-grammar)])
                        (case-lambda [(tokenizer)
                                      (define next-token
                                        (make-permissive-tokenizer tokenizer all-tokens-hash/mutable))
                                      (THE-GRAMMAR next-token)]
                                     [(source tokenizer)
                                      (parameterize ([current-source source])
                                        (parse tokenizer))])))]))
             
             (define parse (make-rule-parser start-id))))))]))


;; Given a flattened rule, returns a syntax for the code that
;; preserves as much source location as possible.
;;
;; Each rule is defined to return a list with the following structure:
;;
;;     stx :== (name (U tokens rule-stx) ...)
;;
(define (flat-rule->yacc-rule a-flat-rule)
  #;(report a-flat-rule)
  (syntax-case a-flat-rule ()
    [(rule-type origin name clauses ...)
     (begin
       (define translated-clauses
         (map (lambda (clause) (translate-clause clause #'name #'origin))
              (syntax->list #'(clauses ...))))
       (with-syntax ([(translated-clause ...) translated-clauses])
         #`[name translated-clause ...]))]))



;; translates a single primitive rule clause.
;; A clause is a simple list of ids, lit, vals, and inferred-id elements.
;; The action taken depends on the pattern type.
(define (translate-clause a-clause rule-name/false origin)
  (define translated-patterns
    (let loop ([primitive-patterns (syntax->list a-clause)])
      (cond
        [(empty? primitive-patterns)
         '()]
        [else
         (cons (syntax-case (first primitive-patterns) (id lit token inferred-id)
                 [(id val)
                  #'val]
                 [(lit val)
                  (datum->syntax #f (string->symbol (syntax-e #'val)) #'val)]
                 [(token val)
                  #'val]
                 [(inferred-id val reason)
                  #'val])
               (loop (rest primitive-patterns)))])))
  
  (define translated-actions
    (for/list ([translated-pattern (in-list translated-patterns)]
               [primitive-pattern (syntax->list a-clause)]
               [pos (in-naturals 1)])
              (with-syntax ([$X 
                             (format-id translated-pattern "$~a" pos)]
                            [$X-start-pos 
                             (format-id translated-pattern "$~a-start-pos" pos)]
                            [$X-end-pos
                             (format-id translated-pattern "$~a-end-pos" pos)])
                (syntax-case primitive-pattern (id lit token inferred-id)
                  
                  ;; When a rule usage is inferred, the value of $X is a syntax object
                  ;; whose head is the name of the inferred rule . We strip that out,
                  ;; leaving the residue to be absorbed.
                  [(inferred-id val reason)
                   #'(syntax-case $X ()
                       [(inferred-rule-name . rest)
                        (syntax->list #'rest)])]
                  [(id val)
                   #'(list $X)]
                  ;; move the 'hide syntax property into the translated-action
                  ;; because syntax gets datum-ized
                  [(lit val)
                   #`(list (atomic-datum->syntax $X $X-start-pos $X-end-pos #,(syntax-property primitive-pattern 'hide)))]
                  [(token val)
                   #`(list (atomic-datum->syntax $X $X-start-pos $X-end-pos #,(syntax-property primitive-pattern 'hide)))]))))
  
  (define whole-rule-loc
    (if (empty? translated-patterns)
        #'(list (current-source) #f #f #f #f)
        (with-syntax ([$1-start-pos (datum->syntax (first translated-patterns) '$1-start-pos)]
                      [$n-end-pos (format-id (last translated-patterns) "$~a-end-pos" (length translated-patterns))])
          #`(positions->srcloc $1-start-pos $n-end-pos))))
  
  (with-syntax ([(translated-pattern ...) translated-patterns]
                [(translated-action ...) translated-actions])
    #`[(translated-pattern ...)
       (rule-components->syntax '#,rule-name/false translated-action ...
                                #:srcloc #,whole-rule-loc)]))



;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; collect-token-types: (listof rule-syntax) -> (values (listof identifier) (listof identifier))
;;
;; Given a rule, automatically derive the list of implicit and
;; explicit token types we need to generate.
;; 
;; Note: EOF is reserved, and will always be included in the list
;; of explicit token types, though the user is not allow to express it themselves.
(define (rules-collect-token-types rules)
  (define-values (implicit explicit)
    (for/fold ([implicit '()]
               [explicit (list (datum->syntax (first rules) 'EOF))])
              ([r (in-list rules)])
      (rule-collect-token-types r implicit explicit)))
  (values (reverse implicit) (reverse explicit)))

(define (rule-collect-token-types a-rule implicit explicit)
  (syntax-case a-rule (rule)
    [(rule id a-pattern)
     (pattern-collect-implicit-token-types #'a-pattern implicit explicit)]))

(define (pattern-collect-implicit-token-types a-pattern implicit explicit)
  (let loop ([a-pattern a-pattern]
             [implicit implicit]
             [explicit explicit])
    (syntax-case a-pattern (id lit token choice repeat maybe seq)
      [(id val)
       (values implicit explicit)]
      [(lit val)
       (values (cons #'val implicit) explicit)]
      [(token val)
       (begin
         (when (eq? (syntax-e #'val) 'EOF)
           (raise-syntax-error #f "Token EOF is reserved and can not be used in a grammar" #'val))
         (values implicit (cons #'val explicit)))]
      [(choice vals ...)
       (for/fold ([implicit implicit]
                  [explicit explicit])
                 ([v (in-list (syntax->list #'(vals ...)))])
         (loop v implicit explicit))]
      [(repeat min val)
       (loop #'val implicit explicit)]
      [(maybe val)
       (loop #'val implicit explicit)]
      [(seq vals ...)
       (for/fold ([implicit implicit]
                  [explicit explicit])
                 ([v (in-list (syntax->list #'(vals ...)))])
         (loop v implicit explicit))])))


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; rule-id: rule -> identifier-stx
;; Get the binding id of a rule.
(define (rule-id a-rule)
  (syntax-case a-rule (rule)
    [(rule id a-pattern)
     #'id]))

(define (rule-pattern a-rule)
  (syntax-case a-rule (rule)
    [(rule id a-pattern)
     #'a-pattern]))


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; check-all-rules-defined!: (listof rule-stx) -> void
(define (check-all-rules-defined! rules)
  (define table (make-free-id-table))
  ;; Pass one: collect all the defined rule names.
  (for ([a-rule (in-list rules)])
       (free-id-table-set! table (rule-id a-rule) #t))
  ;; Pass two: check each referenced id, and make sure it's been defined.
  (for ([a-rule (in-list rules)])
       (for ([referenced-id (in-list (rule-collect-used-ids a-rule))])
            (unless (free-id-table-ref table referenced-id (lambda () #f))
              (raise-syntax-error #f (format "Rule ~a has no definition" (syntax-e referenced-id))
                                  referenced-id)))))

;; check-all-rules-no-duplicates!: (listof rule-stx) -> void
(define (check-all-rules-no-duplicates! rules)
  (define table (make-free-id-table))
  ;; Pass one: collect all the defined rule names.
  (for ([a-rule (in-list rules)])
       (define maybe-other-rule-id (free-id-table-ref table (rule-id a-rule) (lambda () #f)))
       (when maybe-other-rule-id
         (raise-syntax-error #f (format "Rule ~a has a duplicate definition" (syntax-e (rule-id a-rule)))
                             (rule-id a-rule)
                             #f
                             (list (rule-id a-rule) maybe-other-rule-id)))
       (free-id-table-set! table (rule-id a-rule) (rule-id a-rule))))



;; rule-collect-used-ids: rule-stx -> (listof identifier)
;; Given a rule, extracts a list of identifiers
(define (rule-collect-used-ids a-rule)
  (syntax-case a-rule (rule)
    [(rule id a-pattern)
     (pattern-collect-used-ids #'a-pattern '())]))

;; pattern-collect-used-ids: pattern-stx (listof identifier) -> (listof identifier)
;; Returns a flat list of rule identifiers referenced in the pattern.
(define (pattern-collect-used-ids a-pattern acc)
  (let loop ([a-pattern a-pattern]
             [acc acc])
    (syntax-case a-pattern (id lit token choice repeat maybe seq)
      [(id val)
       (cons #'val acc)]
      [(lit val)
       acc]
      [(token val)
       acc]
      [(choice vals ...)
       (for/fold ([acc acc])
                 ([v (in-list (syntax->list #'(vals ...)))])
         (loop v acc))]
      [(repeat min val)
       (loop #'val acc)]
      [(maybe val)
       (loop #'val acc)]
      [(seq vals ...)
       (for/fold ([acc acc])
                 ([v (in-list (syntax->list #'(vals ...)))])
         (loop v acc))])))


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; check-all-rules-satisfiable: (listof rule-stx) -> void
;; Does a simple graph traversal / topological sort-like thing to make sure that, for
;; any rule, there's some finite sequence of tokens that
;; satisfies it.  If this is not the case, then something horrible
;; has happened, and we need to tell the user about it.
;;
;; NOTE: Assumes all referenced rules have definitions.
(define (check-all-rules-satisfiable! rules)
  (define toplevel-rule-table (make-free-id-table))
  (for ([a-rule (in-list rules)])
       (free-id-table-set! toplevel-rule-table 
                           (rule-id a-rule) 
                           (sat:make-and)))
  (define leaves '())
  
  (define (make-leaf)
    (define a-leaf (sat:make-and))
    (set! leaves (cons a-leaf leaves))
    a-leaf)
  
  (define (process-pattern a-pattern)
    (syntax-case a-pattern (id lit token choice repeat maybe seq)
      [(id val)
       (free-id-table-ref toplevel-rule-table #'val)]
      [(lit val)
       (make-leaf)]
      [(token val)
       (make-leaf)]
      [(choice vals ...)
       (begin
         (define an-or-node (sat:make-or))
         (for ([v (in-list (syntax->list #'(vals ...)))])
              (define a-child (process-pattern v))
              (sat:add-child! an-or-node a-child))
         an-or-node)]
      [(repeat min val)
       (syntax-case #'min ()
         [0
          (make-leaf)]
         [else
          (process-pattern #'val)])]
      [(maybe val)
       (make-leaf)]
      [(seq vals ...)
       (begin
         (define an-and-node (sat:make-and))
         (for ([v (in-list (syntax->list #'(vals ...)))])
              (define a-child (process-pattern v))
              (sat:add-child! an-and-node a-child))
         an-and-node)]))
  
  (for ([a-rule (in-list rules)])
       (define rule-node (free-id-table-ref toplevel-rule-table (rule-id a-rule)))
       (sat:add-child! rule-node (process-pattern (rule-pattern a-rule))))
  
  (for ([a-leaf leaves])
       (sat:visit! a-leaf))
  
  (for ([a-rule (in-list rules)])
       (define rule-node (free-id-table-ref toplevel-rule-table (rule-id a-rule)))
       (unless (sat:node-yes? rule-node)
         (raise-syntax-error #f (format "Rule ~a has no finite derivation" (syntax-e (rule-id a-rule)))
                             (rule-id a-rule)))))