beautiful-racket/beautiful-racket-demo/basic-demo/tokenizer.rkt

#lang br
(require brag/support)
(provide (all-defined-out))

(define basic-lexer
  (lexer-srcloc
   [(eof) eof]
   [whitespace (token lexeme #:skip? #t)]
   [(from/to "rem" "\n") (token 'REM (string-downcase lexeme))]
   [(:or "print" "goto" "end") (token (string-downcase lexeme)
                                      (string-downcase lexeme))]
   [(:+ numeric) (token 'NUMBER (string->number lexeme))]
   [(from/to "\"" "\"") (token 'STRING (trim-ends  "\"" lexeme "\""))]))

(define (tokenize ip)
  (port-count-lines! ip)
  (λ () (basic-lexer ip)))