handle position-tokens correctly in `apply-tokenizer`

8 years ago · 2be7726c88
parent 747e59ebc5
commit 2be7726c88
1 changed files with 5 additions and 1 deletions
--- a/brag/brag/support.rkt
+++ b/brag/brag/support.rkt
@ -47,7 +47,11 @@
                           (open-input-string in)
                           in))
    (define token-producer (tokenize input-port))
-    (for/list ([token (in-producer token-producer eof)])
+    (for/list ([token (in-producer token-producer (λ(token)
+                                                    ;; position-tokens are produced by lexer-src-pos
+                                                    (eq? eof (if (position-token? token)
+                                                                 (position-token-token token)
+                                                                 token))))])
              token))

 (define (trim-ends left lexeme right)