dev-srcloc
Matthew Butterick 8 years ago
parent 65b34b8409
commit 5f72ee4215

@ -1,5 +1,5 @@
#lang br/quicklang
(require brag/lexer-support)
(require brag/support)
(define (tokenize port)
(define (next-token)
(define our-lexer

@ -918,8 +918,6 @@ all-token-types
@section{Support API}
@defmodule[brag/support]
@ -928,6 +926,9 @@ The @racketmodname[brag/support] module provides functions to interact with
@tt{brag} programs. The most useful is the @racket[token] function, which
produces tokens to be parsed.
In addition to the exports shown below, the @racketmodname[brag/support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex].
@defproc[(token [type (or/c string? symbol?)]
[val any/c #f]
[#:line line (or/c positive-integer? #f) #f]
@ -973,11 +974,7 @@ The exception raised when parsing fails.
property, so if this exception reaches DrRacket's default error handler,
DrRacket should highlight the offending locations in the source.}
@section{Lexer support API}
@defmodule[brag/lexer-support]
In addition to the exports shown below, the @racketmodname[brag/lexer-support] module also provides everything from @racketmodname[brag/support], and everything from @racketmodname[parser-tools/lex].
@defproc[(apply-tokenizer [tokenizer procedure?]
[source-string (or/c string?

@ -1,27 +0,0 @@
#lang racket/base
(require "support.rkt"
parser-tools/lex
racket/string
(prefix-in : parser-tools/lex-sre)
(for-syntax racket/base))
(provide (all-from-out "support.rkt")
(all-from-out parser-tools/lex)
(all-from-out parser-tools/lex-sre)
(all-defined-out))
(define (apply-tokenizer tokenize in)
(define input-port (if (string? in)
(open-input-string in)
in))
(define token-producer (tokenize input-port))
(for/list ([token (in-producer token-producer eof)])
token))
(define (trim-delimiters left lexeme right)
(string-trim (string-trim lexeme left #:right? #f) right #:left? #f))
(define-lex-trans delimited-by
(λ(stx)
(syntax-case stx ()
[(_ OPEN CLOSE)
#'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)])))

@ -1,8 +1,17 @@
#lang racket/base
(provide [struct-out token-struct]
(require parser-tools/lex
racket/string
(prefix-in : parser-tools/lex-sre)
(for-syntax racket/base))
(provide (all-from-out parser-tools/lex)
(all-from-out parser-tools/lex-sre)
[struct-out token-struct]
token
[struct-out exn:fail:parsing])
[struct-out exn:fail:parsing]
apply-tokenizer
trim-delimiters
delimited-by)
(struct token-struct (type val offset line column span skip?)
#:transparent)
@ -33,5 +42,22 @@
(define (apply-tokenizer tokenize in)
(define input-port (if (string? in)
(open-input-string in)
in))
(define token-producer (tokenize input-port))
(for/list ([token (in-producer token-producer eof)])
token))
(define (trim-delimiters left lexeme right)
(string-trim (string-trim lexeme left #:right? #f) right #:left? #f))
(define-lex-trans delimited-by
(λ(stx)
(syntax-case stx ()
[(_ OPEN CLOSE)
#'(:seq OPEN (complement (:seq any-string CLOSE any-string)) CLOSE)])))

Loading…
Cancel
Save