diff --git a/beautiful-racket/br/demo/jsonic/tokenizer.rkt b/beautiful-racket/br/demo/jsonic/tokenizer.rkt index 13aa18c..fc99bab 100644 --- a/beautiful-racket/br/demo/jsonic/tokenizer.rkt +++ b/beautiful-racket/br/demo/jsonic/tokenizer.rkt @@ -1,5 +1,5 @@ #lang br/quicklang -(require brag/lexer-support) +(require brag/support) (define (tokenize port) (define (next-token) (define our-lexer diff --git a/brag/brag/brag.scrbl b/brag/brag/brag.scrbl index 342b34f..180cbb5 100755 --- a/brag/brag/brag.scrbl +++ b/brag/brag/brag.scrbl @@ -918,8 +918,6 @@ all-token-types - - @section{Support API} @defmodule[brag/support] @@ -928,6 +926,9 @@ The @racketmodname[brag/support] module provides functions to interact with @tt{brag} programs. The most useful is the @racket[token] function, which produces tokens to be parsed. +In addition to the exports shown below, the @racketmodname[brag/support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex]. + + @defproc[(token [type (or/c string? symbol?)] [val any/c #f] [#:line line (or/c positive-integer? #f) #f] @@ -973,11 +974,7 @@ The exception raised when parsing fails. property, so if this exception reaches DrRacket's default error handler, DrRacket should highlight the offending locations in the source.} -@section{Lexer support API} - -@defmodule[brag/lexer-support] -In addition to the exports shown below, the @racketmodname[brag/lexer-support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex]. @defproc[(apply-tokenizer [tokenizer procedure?] [source-string (or/c string? diff --git a/brag/brag/lexer-support.rkt b/brag/brag/lexer-support.rkt deleted file mode 100755 index d8a3bf5..0000000 --- a/brag/brag/lexer-support.rkt +++ /dev/null @@ -1,27 +0,0 @@ -#lang racket/base -(require "support.rkt" - parser-tools/lex - racket/string - (prefix-in : parser-tools/lex-sre) - (for-syntax racket/base)) -(provide (all-from-out "support.rkt") - (all-from-out parser-tools/lex) - (all-from-out parser-tools/lex-sre) - (all-defined-out)) - -(define (apply-tokenizer tokenize in) - (define input-port (if (string? in) - (open-input-string in) - in)) - (define token-producer (tokenize input-port)) - (for/list ([token (in-producer token-producer eof)]) - token)) - -(define (trim-delimiters left lexeme right) - (string-trim (string-trim lexeme left #:right? #f) right #:left? #f)) - -(define-lex-trans delimited-by - (λ(stx) - (syntax-case stx () - [(_ OPEN CLOSE) - #'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)]))) \ No newline at end of file diff --git a/brag/brag/support.rkt b/brag/brag/support.rkt index e63d868..e0f4c94 100755 --- a/brag/brag/support.rkt +++ b/brag/brag/support.rkt @@ -1,8 +1,17 @@ #lang racket/base - -(provide [struct-out token-struct] +(require parser-tools/lex + racket/string + (prefix-in : parser-tools/lex-sre) + (for-syntax racket/base)) +(provide (all-from-out parser-tools/lex) + (all-from-out parser-tools/lex-sre) + [struct-out token-struct] token - [struct-out exn:fail:parsing]) + [struct-out exn:fail:parsing] + apply-tokenizer + trim-delimiters + delimited-by) + (struct token-struct (type val offset line column span skip?) #:transparent) @@ -33,5 +42,22 @@ +(define (apply-tokenizer tokenize in) + (define input-port (if (string? in) + (open-input-string in) + in)) + (define token-producer (tokenize input-port)) + (for/list ([token (in-producer token-producer eof)]) + token)) + +(define (trim-delimiters left lexeme right) + (string-trim (string-trim lexeme left #:right? #f) right #:left? #f)) + +(define-lex-trans delimited-by + (λ(stx) + (syntax-case stx () + [(_ OPEN CLOSE) + #'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)]))) +