return-without-srcloc

beautiful-racket-demo/basic-demo/lexer.rkt

@@ -3,7 +3,7 @@
 
 (define basic-lexer
   (lexer-srcloc
-   [(eof) eof]
+   [(eof) (return-without-srcloc eof)]
    [whitespace (token lexeme #:skip? #t)]
    [(from/to "rem" "\n") (token 'REM lexeme)]
    [(:or "print" "goto" "end" "+" ":") lexeme]
@@ -14,8 +14,11 @@
    [(from/to "\"" "\"")
     (token 'STRING (trim-ends "\"" lexeme "\""))]))
 
-(provide
+(provide basic-lexer)
- (contract-out
+
-  [basic-lexer
+
-   (input-port? . -> .
+(define (apply-lexer lexer str)
-                (or/c eof-object? string? srcloc-token?))]))
+(for/list ([t (in-port lexer (open-input-string str))])
+  t))
+
+(apply-lexer basic-lexer "10 rem")

beautiful-racket-demo/basic-demo/tokenizer.rkt

@@ -7,8 +7,4 @@
   (define (next-token) (basic-lexer ip))
   next-token)
 
-(provide
+(provide make-tokenizer)
- (contract-out
-  [make-tokenizer
-   ((input-port?) (path?) . ->* .
-                  (-> (or/c eof-object? string? srcloc-token?)))]))

br-parser-tools/br-parser-tools-doc/parser-tools/br-parser-tools.scrbl

@@ -167,14 +167,14 @@ are a few examples, using @racket[:] prefixed SRE syntax:
        @item{@racket[input-port] --- the input-port being
             processed (this is useful for matching input with multiple
             lexers).}
-       @item{@racket[(return-without-pos x)] is a function (continuation) that
+       @item{@racket[(return-without-pos x)] and @racket[(return-without-srcloc x)] are functions (continuations) that
-	immediately returns the value of @racket[x] from the lexer.  This useful
+	immediately return the value of @racket[x] from the lexer.  This useful
-	in a src-pos lexer to prevent the lexer from adding source
+	in a src-pos or src-loc lexer to prevent the lexer from adding source
 	information.  For example:
 
 	@racketblock[
 	(define get-token
-	  (lexer-src-pos
+	  (lexer-srcloc
 	  ...
 	  ((comment) (get-token input-port))
 	  ...))
@@ -182,7 +182,7 @@ are a few examples, using @racket[:] prefixed SRE syntax:
 
 	would wrap the source location information for the comment around
 	the value of the recursive call.  Using
-	@racket[((comment) (return-without-pos (get-token input-port)))] 
+	@racket[((comment) (return-without-srcloc (get-token input-port)))] 
 	will cause the value of the recursive call to be returned without
 	wrapping position around it.}
      ]
@@ -262,6 +262,7 @@ _action-result lexeme-srcloc)] instead of simply
 @defidform[lexeme-srcloc]
 @defidform[input-port]
 @defidform[return-without-pos]
+@defidform[return-without-srcloc]
 )]{
 
 Use of these names outside of a @racket[lexer] action is a syntax

br-parser-tools/br-parser-tools-lib/br-parser-tools/lex.rkt

@@ -46,7 +46,7 @@
                      [(eq? src-loc-style 'lexer-srcloc)
                       #`(let/ec ret
                           (syntax-parameterize 
-                              ([return-without-pos (make-rename-transformer #'ret)])
+                              ([return-without-srcloc (make-rename-transformer #'ret)])
                             (make-srcloc-token #,action lexeme-srcloc)))]
                      [else action])))
       (syntax/loc action
@@ -405,6 +405,6 @@
          ...
          (provide id ...))]))
   
-  (provide-lex-keyword start-pos end-pos lexeme lexeme-srcloc input-port return-without-pos)
+  (provide-lex-keyword start-pos end-pos lexeme lexeme-srcloc input-port return-without-pos return-without-srcloc)
 
   )