From 2be7726c8851fdd8b5c606ebc11fbb78af56ac8d Mon Sep 17 00:00:00 2001 From: Matthew Butterick Date: Thu, 8 Dec 2016 09:44:19 -0800 Subject: [PATCH] handle position-tokens correctly in `apply-tokenizer` --- brag/brag/support.rkt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/brag/brag/support.rkt b/brag/brag/support.rkt index 5cf4ee7..23dcb4f 100755 --- a/brag/brag/support.rkt +++ b/brag/brag/support.rkt @@ -47,7 +47,11 @@ (open-input-string in) in)) (define token-producer (tokenize input-port)) - (for/list ([token (in-producer token-producer eof)]) + (for/list ([token (in-producer token-producer (λ(token) + ;; position-tokens are produced by lexer-src-pos + (eq? eof (if (position-token? token) + (position-token-token token) + token))))]) token)) (define (trim-ends left lexeme right)