reorg

beautiful-racket/br/demo/jsonic/tokenizer.rkt

@@ -1,5 +1,5 @@
 #lang br/quicklang
-(require brag/lexer-support)
+(require brag/support)
 (define (tokenize port)
   (define (next-token)
     (define our-lexer

brag/brag/brag.scrbl

@@ -918,8 +918,6 @@ all-token-types
 
 
 
-
-
 @section{Support API}
 
 @defmodule[brag/support]
@@ -928,6 +926,9 @@ The @racketmodname[brag/support] module provides functions to interact with
 @tt{brag} programs. The most useful is the @racket[token] function, which
 produces tokens to be parsed.
 
+In addition to the exports shown below, the @racketmodname[brag/support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex].
+
+
 @defproc[(token [type (or/c string? symbol?)]
                 [val any/c #f]
                 [#:line line (or/c positive-integer? #f) #f]
@@ -973,11 +974,7 @@ The exception raised when parsing fails.
 property, so if this exception reaches DrRacket's default error handler,
 DrRacket should highlight the offending locations in the source.}
 
-@section{Lexer support API}
-
-@defmodule[brag/lexer-support]
 
-In addition to the exports shown below, the @racketmodname[brag/lexer-support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex].
 
 @defproc[(apply-tokenizer [tokenizer procedure?] 
                 [source-string (or/c string?

brag/brag/lexer-support.rkt

@@ -1,27 +0,0 @@
-#lang racket/base
-(require "support.rkt"
-         parser-tools/lex
-         racket/string
-         (prefix-in : parser-tools/lex-sre)
-         (for-syntax racket/base))
-(provide (all-from-out "support.rkt")
-         (all-from-out parser-tools/lex)
-         (all-from-out parser-tools/lex-sre)
-         (all-defined-out))
-
-(define (apply-tokenizer tokenize in)
-    (define input-port (if (string? in)
-                           (open-input-string in)
-                           in))
-    (define token-producer (tokenize input-port))
-    (for/list ([token (in-producer token-producer eof)])
-              token))
-
-(define (trim-delimiters left lexeme right)
-  (string-trim (string-trim lexeme left #:right? #f) right #:left? #f))
-
-(define-lex-trans delimited-by
-  (λ(stx)
-    (syntax-case stx ()
-      [(_ OPEN CLOSE)
-       #'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)])))

brag/brag/support.rkt

@@ -1,8 +1,17 @@
 #lang racket/base
-
-(provide [struct-out token-struct]
+(require parser-tools/lex
+         racket/string
+         (prefix-in : parser-tools/lex-sre)
+         (for-syntax racket/base))
+(provide (all-from-out parser-tools/lex)
+         (all-from-out parser-tools/lex-sre)
+         [struct-out token-struct]
          token
-         [struct-out exn:fail:parsing])
+         [struct-out exn:fail:parsing]
+         apply-tokenizer
+         trim-delimiters
+         delimited-by)
+
 
 (struct token-struct (type val offset line column span skip?) 
         #:transparent)
@@ -33,5 +42,22 @@
 
 
 
+(define (apply-tokenizer tokenize in)
+    (define input-port (if (string? in)
+                           (open-input-string in)
+                           in))
+    (define token-producer (tokenize input-port))
+    (for/list ([token (in-producer token-producer eof)])
+              token))
+
+(define (trim-delimiters left lexeme right)
+  (string-trim (string-trim lexeme left #:right? #f) right #:left? #f))
+
+(define-lex-trans delimited-by
+  (λ(stx)
+    (syntax-case stx ()
+      [(_ OPEN CLOSE)
+       #'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)])))
+