dev-srcloc
Matthew Butterick 8 years ago
parent 65b34b8409
commit 5f72ee4215

@ -1,5 +1,5 @@
#lang br/quicklang #lang br/quicklang
(require brag/lexer-support) (require brag/support)
(define (tokenize port) (define (tokenize port)
(define (next-token) (define (next-token)
(define our-lexer (define our-lexer

@ -918,8 +918,6 @@ all-token-types
@section{Support API} @section{Support API}
@defmodule[brag/support] @defmodule[brag/support]
@ -928,6 +926,9 @@ The @racketmodname[brag/support] module provides functions to interact with
@tt{brag} programs. The most useful is the @racket[token] function, which @tt{brag} programs. The most useful is the @racket[token] function, which
produces tokens to be parsed. produces tokens to be parsed.
In addition to the exports shown below, the @racketmodname[brag/support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex].
@defproc[(token [type (or/c string? symbol?)] @defproc[(token [type (or/c string? symbol?)]
[val any/c #f] [val any/c #f]
[#:line line (or/c positive-integer? #f) #f] [#:line line (or/c positive-integer? #f) #f]
@ -973,11 +974,7 @@ The exception raised when parsing fails.
property, so if this exception reaches DrRacket's default error handler, property, so if this exception reaches DrRacket's default error handler,
DrRacket should highlight the offending locations in the source.} DrRacket should highlight the offending locations in the source.}
@section{Lexer support API}
@defmodule[brag/lexer-support]
In addition to the exports shown below, the @racketmodname[brag/lexer-support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex].
@defproc[(apply-tokenizer [tokenizer procedure?] @defproc[(apply-tokenizer [tokenizer procedure?]
[source-string (or/c string? [source-string (or/c string?

@ -1,27 +0,0 @@
#lang racket/base
(require "support.rkt"
parser-tools/lex
racket/string
(prefix-in : parser-tools/lex-sre)
(for-syntax racket/base))
(provide (all-from-out "support.rkt")
(all-from-out parser-tools/lex)
(all-from-out parser-tools/lex-sre)
(all-defined-out))
(define (apply-tokenizer tokenize in)
(define input-port (if (string? in)
(open-input-string in)
in))
(define token-producer (tokenize input-port))
(for/list ([token (in-producer token-producer eof)])
token))
(define (trim-delimiters left lexeme right)
(string-trim (string-trim lexeme left #:right? #f) right #:left? #f))
(define-lex-trans delimited-by
(λ(stx)
(syntax-case stx ()
[(_ OPEN CLOSE)
#'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)])))

@ -1,8 +1,17 @@
#lang racket/base #lang racket/base
(require parser-tools/lex
(provide [struct-out token-struct] racket/string
(prefix-in : parser-tools/lex-sre)
(for-syntax racket/base))
(provide (all-from-out parser-tools/lex)
(all-from-out parser-tools/lex-sre)
[struct-out token-struct]
token token
[struct-out exn:fail:parsing]) [struct-out exn:fail:parsing]
apply-tokenizer
trim-delimiters
delimited-by)
(struct token-struct (type val offset line column span skip?) (struct token-struct (type val offset line column span skip?)
#:transparent) #:transparent)
@ -33,5 +42,22 @@
(define (apply-tokenizer tokenize in)
(define input-port (if (string? in)
(open-input-string in)
in))
(define token-producer (tokenize input-port))
(for/list ([token (in-producer token-producer eof)])
token))
(define (trim-delimiters left lexeme right)
(string-trim (string-trim lexeme left #:right? #f) right #:left? #f))
(define-lex-trans delimited-by
(λ(stx)
(syntax-case stx ()
[(_ OPEN CLOSE)
#'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)])))

Loading…
Cancel
Save