From 2849f0ef14e97ed6ad5f8a8abec25270d1927c18 Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Thu, 13 Feb 2014 17:27:58 -0800
Subject: [PATCH 1/8] start xexpr file

---
 hyphenate/xexpr.rkt | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 hyphenate/xexpr.rkt

diff --git a/hyphenate/xexpr.rkt b/hyphenate/xexpr.rkt
new file mode 100644
index 00000000..a90e5efc
--- /dev/null
+++ b/hyphenate/xexpr.rkt
@@ -0,0 +1,39 @@
+#lang racket/base
+(require "main.rkt")
+(require xml)
+
+
+
+#|
+The following grammar describes expressions that create X-expressions:
+
+  xexpr	 	=	 	string
+ 	 	|	 	(list symbol (list (list symbol string) ...) xexpr ...)
+ 	 	|	 	(cons symbol (list xexpr ...))
+ 	 	|	 	symbol
+ 	 	|	 	valid-char?
+ 	 	|	 	cdata
+ 	 	|	 	misc
+
+|#
+
+
+;; recursively hyphenate strings within xexpr
+;; todo: add exclusion #:only [only-proc (λ(x) x)]
+(define (hyphenate-xexpr x ) 
+  (define exclusions '(style script)) ; omit these from ever being hyphenated
+  
+  (cond
+    ; todo: the only-proc semantics are illogical.
+    ; main issue: keep it out of tags like <style> that parse as textual elements, but are not.
+    ; So two choices, opt-out or opt-in.
+    ; Problem with opt-out: is set of outlier tags like <style> well-defined?
+    ; Won't it make hyphenation naturally overinclusive?
+    ; Problem with opt-in: conceals a lot of tags that naturally live inside other tags
+    ; only reaches text at the "root level" of the tag.
+    [(tagged-xexpr? x) (if (and (only-proc x) (not ((car x) . in? . exclusions)))
+                           (map-xexpr-elements hyphenate x)
+                           (map-xexpr-elements (λ(x) (if (tagged-xexpr? x) (hyphenate x) x)) x))] ; only process subxexprs
+    
+    [(string? x) (hyphenate-string x)]
+    [else x]))

From d2f754714fcd52eaa66e78119b6f02c194462f5a Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Fri, 14 Feb 2014 15:19:35 -0800
Subject: [PATCH 2/8] rename file

---
 hyphenate/{xexpr.rkt => hx.rkt} | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)
 rename hyphenate/{xexpr.rkt => hx.rkt} (82%)

diff --git a/hyphenate/xexpr.rkt b/hyphenate/hx.rkt
similarity index 82%
rename from hyphenate/xexpr.rkt
rename to hyphenate/hx.rkt
index a90e5efc..e0fc36fa 100644
--- a/hyphenate/xexpr.rkt
+++ b/hyphenate/hx.rkt
@@ -1,7 +1,7 @@
 #lang racket/base
 (require "main.rkt")
 (require xml)
-
+(require tagged-xexpr)
 
 
 #|
@@ -20,10 +20,23 @@ The following grammar describes expressions that create X-expressions:
 
 ;; recursively hyphenate strings within xexpr
 ;; todo: add exclusion #:only [only-proc (λ(x) x)]
-(define (hyphenate-xexpr x ) 
-  (define exclusions '(style script)) ; omit these from ever being hyphenated
+(define (hx x)
+  (cond 
+    [(string? x) (hyphenate x)]
+    [(tagged-xexpr? x) x]
+    ;;
+    ;;
+    [else x] ;; catches symbols, valid-chars, and cdata
+    )
   
-  (cond
+  )
+
+
+#|
+  (define exclusions '(style script)) ; omit these from ever being hyphenated
+
+
+(cond
     ; todo: the only-proc semantics are illogical.
     ; main issue: keep it out of tags like <style> that parse as textual elements, but are not.
     ; So two choices, opt-out or opt-in.
@@ -36,4 +49,8 @@ The following grammar describes expressions that create X-expressions:
                            (map-xexpr-elements (λ(x) (if (tagged-xexpr? x) (hyphenate x) x)) x))] ; only process subxexprs
     
     [(string? x) (hyphenate-string x)]
-    [else x]))
+    [else x])
+|#
+
+;; how to make cdata:
+;;  (make-cdata #f #f "<![CDATA[foobar]]>")
\ No newline at end of file

From bc483a027ed300ffd18f66ec8487841e32c7c57d Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 17 Feb 2014 20:49:02 -0800
Subject: [PATCH 3/8] improvements

---
 hyphenate/hx.rkt    | 56 ---------------------------------------------
 hyphenate/main.rkt  | 41 ++++++++++++++++++++++-----------
 hyphenate/tests.rkt |  1 +
 3 files changed, 29 insertions(+), 69 deletions(-)
 delete mode 100644 hyphenate/hx.rkt

diff --git a/hyphenate/hx.rkt b/hyphenate/hx.rkt
deleted file mode 100644
index e0fc36fa..00000000
--- a/hyphenate/hx.rkt
+++ /dev/null
@@ -1,56 +0,0 @@
-#lang racket/base
-(require "main.rkt")
-(require xml)
-(require tagged-xexpr)
-
-
-#|
-The following grammar describes expressions that create X-expressions:
-
-  xexpr	 	=	 	string
- 	 	|	 	(list symbol (list (list symbol string) ...) xexpr ...)
- 	 	|	 	(cons symbol (list xexpr ...))
- 	 	|	 	symbol
- 	 	|	 	valid-char?
- 	 	|	 	cdata
- 	 	|	 	misc
-
-|#
-
-
-;; recursively hyphenate strings within xexpr
-;; todo: add exclusion #:only [only-proc (λ(x) x)]
-(define (hx x)
-  (cond 
-    [(string? x) (hyphenate x)]
-    [(tagged-xexpr? x) x]
-    ;;
-    ;;
-    [else x] ;; catches symbols, valid-chars, and cdata
-    )
-  
-  )
-
-
-#|
-  (define exclusions '(style script)) ; omit these from ever being hyphenated
-
-
-(cond
-    ; todo: the only-proc semantics are illogical.
-    ; main issue: keep it out of tags like <style> that parse as textual elements, but are not.
-    ; So two choices, opt-out or opt-in.
-    ; Problem with opt-out: is set of outlier tags like <style> well-defined?
-    ; Won't it make hyphenation naturally overinclusive?
-    ; Problem with opt-in: conceals a lot of tags that naturally live inside other tags
-    ; only reaches text at the "root level" of the tag.
-    [(tagged-xexpr? x) (if (and (only-proc x) (not ((car x) . in? . exclusions)))
-                           (map-xexpr-elements hyphenate x)
-                           (map-xexpr-elements (λ(x) (if (tagged-xexpr? x) (hyphenate x) x)) x))] ; only process subxexprs
-    
-    [(string? x) (hyphenate-string x)]
-    [else x])
-|#
-
-;; how to make cdata:
-;;  (make-cdata #f #f "<![CDATA[foobar]]>")
\ No newline at end of file
diff --git a/hyphenate/main.rkt b/hyphenate/main.rkt
index 6fac8aef..26426968 100644
--- a/hyphenate/main.rkt
+++ b/hyphenate/main.rkt
@@ -1,6 +1,6 @@
 #lang racket/base
 (require racket/string racket/list racket/contract racket/vector)
-(require "patterns.rkt" "exceptions.rkt")
+(require "patterns.rkt" "exceptions.rkt" tagged-xexpr xml)
 
 (module+ test (require rackunit))
 
@@ -16,13 +16,13 @@
 
 (provide (contract-out 
           [hyphenate 
-           ((string?) ((or/c char? string?) #:exceptions (listof exception-word?) #:min-length (or/c integer? #f)) . ->* . string?)])
+           ((xexpr?) ((or/c char? string?) #:exceptions (listof exception-word?) #:min-length (or/c integer? #f)) . ->* . string?)])
          (contract-out 
           [hyphenatef
-           ((string? procedure?) ((or/c char? string?) #:exceptions (listof exception-word?) #:min-length (or/c integer? #f)) . ->* . string?)])
+           ((xexpr? procedure?) ((or/c char? string?) #:exceptions (listof exception-word?) #:min-length (or/c integer? #f)) . ->* . string?)])
          (contract-out 
           [unhyphenate 
-           ((string?) ((or/c char? string?)) . ->* . string?)]))
+           ((xexpr?) ((or/c char? string?)) . ->* . string?)]))
 
 ;; global data, define now but set! them later (because they're potentially big & slow)
 (define exceptions #f)
@@ -159,7 +159,7 @@
 ;; Hyphenate using a filter procedure.
 ;; Theoretically possible to do this externally,
 ;; but it would just mean doing the regexp-replace twice.
-(define (hyphenatef text proc [joiner default-joiner] #:exceptions [extra-exceptions '()]  #:min-length [min-length default-min-length])
+(define (hyphenatef x proc [joiner default-joiner] #:exceptions [extra-exceptions '()]  #:min-length [min-length default-min-length])
   
   
   ;; set up module data
@@ -170,15 +170,30 @@
   (define joiner-string (joiner->string joiner))
   (define word-pattern #px"\\w+") ;; more restrictive than exception-word
   ;; todo?: connect this regexp pattern to the one used in word? predicate
-  (regexp-replace* word-pattern text (λ(word) (if (proc word) (string-join (word->hyphenation-points word min-length) joiner-string) word))))
+  (define (insert-hyphens text)
+    (regexp-replace* word-pattern text (λ(word) (if (proc word) (string-join (word->hyphenation-points word min-length) joiner-string) word))))
+  
+  (let &hyphenate ([x x])
+    (cond
+      [(string? x) (insert-hyphens x)]
+      [(tagged-xexpr? x) (map-elements &hyphenate x)]
+      [else x])))
 
 
 ;; Default hyphenate function.
-(define (hyphenate text [joiner default-joiner]  #:exceptions [extra-exceptions '()] #:min-length [min-length default-min-length])
-  (hyphenatef text (λ(x) #t) joiner #:exceptions extra-exceptions #:min-length min-length))
-
-(define (unhyphenate text [joiner default-joiner])
-  (string-replace text (joiner->string joiner) ""))
-
-
+(define (hyphenate x [joiner default-joiner]  #:exceptions [extra-exceptions '()] #:min-length [min-length default-min-length])
+  (hyphenatef x (λ(x) #t) joiner #:exceptions extra-exceptions #:min-length min-length))
 
+(define (unhyphenate x [joiner default-joiner])
+  (define (remove-hyphens text)
+    (string-replace text (joiner->string joiner) ""))
+  
+  (let &unhyphenate ([x x])
+    (cond
+      [(string? x) (remove-hyphens x)]
+      [(tagged-xexpr? x) (map-elements &unhyphenate x)]
+      [else x])))
+  
+  
+  
+  
\ No newline at end of file
diff --git a/hyphenate/tests.rkt b/hyphenate/tests.rkt
index 143ac860..e2110799 100644
--- a/hyphenate/tests.rkt
+++ b/hyphenate/tests.rkt
@@ -15,6 +15,7 @@
 (check-equal? (unhyphenate "polyfoomorfoophism" "foo") "polymorphism")
 (check-equal? (hyphenate "polymorphism" #\* #:exceptions '("polymo-rphism")) "polymo*rphism")
 (check-equal? (hyphenate "circular polymorphism squandering") "cir\u00ADcu\u00ADlar poly\u00ADmor\u00ADphism squan\u00ADder\u00ADing")
+(check-equal? (hyphenate '(p "circular polymorphism" amp (em "squandering"))) '(p "cir\u00ADcu\u00ADlar poly\u00ADmor\u00ADphism" amp (em "squan\u00ADder\u00ADing")))
 (check-equal? (hyphenate "present project") "present project") ; exception words
 ;; test these last so exceptions have been set up already
 (check-equal? (word->hyphenation-points "polymorphism") '("poly" "mor" "phism"))

From 32a4b428cfbd96e39afa14f386b9f7dce35191d6 Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 17 Feb 2014 22:13:01 -0800
Subject: [PATCH 4/8] updates

---
 hyphenate/main.rkt                    | 40 +++++++++++++++++----------
 hyphenate/scribblings/hyphenate.scrbl | 27 ++++++++++--------
 2 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/hyphenate/main.rkt b/hyphenate/main.rkt
index 26426968..5842b20e 100644
--- a/hyphenate/main.rkt
+++ b/hyphenate/main.rkt
@@ -1,4 +1,5 @@
 #lang racket/base
+(require (for-syntax racket/base))
 (require racket/string racket/list racket/contract racket/vector)
 (require "patterns.rkt" "exceptions.rkt" tagged-xexpr xml)
 
@@ -14,15 +15,15 @@
 ;;; (also in the public domain)
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(provide (contract-out 
-          [hyphenate 
-           ((xexpr?) ((or/c char? string?) #:exceptions (listof exception-word?) #:min-length (or/c integer? #f)) . ->* . string?)])
-         (contract-out 
-          [hyphenatef
-           ((xexpr? procedure?) ((or/c char? string?) #:exceptions (listof exception-word?) #:min-length (or/c integer? #f)) . ->* . string?)])
-         (contract-out 
-          [unhyphenate 
-           ((xexpr?) ((or/c char? string?)) . ->* . string?)]))
+(define-syntax (define+provide/contract stx)
+  (syntax-case stx ()
+    [(_ (proc arg ... . rest-arg) contract body ...)
+     #'(define+provide/contract proc contract
+         (λ(arg ... . rest-arg) body ...))]
+    [(_ name contract body ...)
+     #'(begin
+         (provide (contract-out [name contract]))
+         (define name body ...))]))
 
 ;; global data, define now but set! them later (because they're potentially big & slow)
 (define exceptions #f)
@@ -157,10 +158,12 @@
   (if (char? joiner) (format "~a" joiner) joiner))
 
 ;; Hyphenate using a filter procedure.
-;; Theoretically possible to do this externally,
-;; but it would just mean doing the regexp-replace twice.
-(define (hyphenatef x proc [joiner default-joiner] #:exceptions [extra-exceptions '()]  #:min-length [min-length default-min-length])
-  
+(define+provide/contract (hyphenatef x proc [joiner default-joiner] 
+                                     #:exceptions [extra-exceptions '()]  
+                                     #:min-length [min-length default-min-length])
+  ((xexpr? procedure?) ((or/c char? string?) 
+                        #:exceptions (listof exception-word?) 
+                        #:min-length (or/c integer? #f)) . ->* . string?)
   
   ;; set up module data
   ;; todo?: change set! to parameterize
@@ -181,10 +184,17 @@
 
 
 ;; Default hyphenate function.
-(define (hyphenate x [joiner default-joiner]  #:exceptions [extra-exceptions '()] #:min-length [min-length default-min-length])
+(define+provide/contract (hyphenate x [joiner default-joiner]  
+                                    #:exceptions [extra-exceptions '()] 
+                                    #:min-length [min-length default-min-length])
+  ((xexpr/c) ((or/c char? string?) 
+             #:exceptions (listof exception-word?) 
+             #:min-length (or/c integer? #f)) . ->* . string?)
   (hyphenatef x (λ(x) #t) joiner #:exceptions extra-exceptions #:min-length min-length))
 
-(define (unhyphenate x [joiner default-joiner])
+(define+provide/contract (unhyphenate x [joiner default-joiner])
+  ((xexpr/c) ((or/c char? string?)) . ->* . string?)
+  
   (define (remove-hyphens text)
     (string-replace text (joiner->string joiner) ""))
   
diff --git a/hyphenate/scribblings/hyphenate.scrbl b/hyphenate/scribblings/hyphenate.scrbl
index cf80a433..f414f79f 100644
--- a/hyphenate/scribblings/hyphenate.scrbl
+++ b/hyphenate/scribblings/hyphenate.scrbl
@@ -14,12 +14,12 @@ A simple hyphenation engine that uses the Knuth–Liang hyphenation algorithm or
 
 I originally put together this module to handle hyphenation for my web-based book @link["http://practicaltypography.com"]{Butterick's Practical Typography} (which I made with Racket & Scribble). Though support for CSS-based hyphenation in web browsers is @link["http://caniuse.com/#search=hyphen"]{still iffy}, soft hyphens work reliably well. But putting them into the text manually is a drag. Thus a module was born.
 
-@section{Installation & updates}
+@section{Installation}
 
 At the command line:
 @verbatim{raco pkg install hyphenate}
 
-After that, you can update the package from the command line:
+After that, you can update the package like so:
 @verbatim{raco pkg update hyphenate}
 
 
@@ -27,15 +27,14 @@ After that, you can update the package from the command line:
 
 @defmodule[hyphenate]
 
-
 @defproc[
 (hyphenate 
-[text string?] 
+[xexpr xexpr/c] 
 [joiner (or/c char? string?) (integer->char #x00AD)]
 [#:exceptions exceptions (listof string?) empty]
 [#:min-length length (or/c integer? false?) 5])
 string?]
-Hyphenate @racket[_text] by calculating hyphenation points and inserting @racket[_joiner] at those points. By default, @racket[_joiner] is the soft hyphen (Unicode 00AD = decimal 173). Words shorter than @racket[#:min-length] @racket[_length] will not be hyphenated. To hyphenate words of any length, use @racket[#:min-length] @racket[#f].
+Hyphenate @racket[_xexpr] by calculating hyphenation points and inserting @racket[_joiner] at those points. By default, @racket[_joiner] is the soft hyphen (Unicode 00AD = decimal 173). Words shorter than @racket[#:min-length] @racket[_length] will not be hyphenated. To hyphenate words of any length, use @racket[#:min-length] @racket[#f].
 
 @margin-note{The REPL displays a soft hyphen as \u00AD. But in ordinary use, you'll only see a soft hyphen when it appears at the end of a line or page as part of a hyphenated word. Otherwise it's not displayed. In most of the examples here, I use a standard hyphen for clarity.}
 
@@ -80,18 +79,24 @@ For this reason, certain words can't be hyphenated algorithmically, because the
    
 This is the right result. If you used @italic{adder} to mean the machine, it would be hyphenated @italic{add-er}; if you meant the snake, it would be @italic{ad-der}. Better to avoid hyphenation than to hyphenate incorrectly.
 
+You can send HTML-style X-expressions through @racket[hyphenate]. It will recursively hyphenate the text strings, while leaving the tags and attributes alone.
+
+@examples[#:eval my-eval
+     (hyphenate '(p "polymorphically" (em "formatted" (strong "snowmen")))) 
+   ]
 
-Don't send raw HTML through @racket[hyphenate]. It can't distinguish HTML tags and attributes from textual content, so it will hyphenate everything, which will goof up your file.
+Don't send raw HTML through @racket[hyphenate]. It can't distinguish HTML tags and attributes from textual content, so everything will be hyphenated, thus goofing up your file. But you can easily convert HTML to an X-expression, hyphenate it, and then convert back.
 
 @examples[#:eval my-eval
-     (hyphenate "<body style=\"background: yellow\">Hello world</body>") 
+    (define html "<body style=\"background: yellow\">Hello snowman</body>")
+    (hyphenate html)
+    (xexpr->string (hyphenate (string->xexpr html))) 
    ]
    
-Instead, send your textual content through @racket[hyphenate] @italic{before} you put it into your HTML template. Or convert your HTML to an X-expression and process it selectively (e.g., with @racket[match]).
 
 @defproc[
 (hyphenatef
-[text string?] 
+[xexpr xexpr/c] 
 [pred procedure?]
 [joiner (or/c char? string?) (integer->char \#x00AD)]
 [#:exceptions exceptions (listof string?) empty]
@@ -121,10 +126,10 @@ It's possible to do fancier kinds of hyphenation restrictions that take account
 
 @defproc[
 (unhyphenate
-[text string?] 
+[xexpr xexpr/c] 
 [joiner (or/c char? string?) @(integer->char #x00AD)])
 string?]
-Remove @racket[_joiner] from @racket[_text] using @racket[string-replace].
+Remove @racket[_joiner] from @racket[_xexpr].
 
 A side effect of using @racket[hyphenate] is that soft hyphens (or whatever the @racket[_joiner] is) will be embedded in the output text. If you need to support copying of text, for instance in a GUI application, you'll probably want to strip out the hyphenation before the copied text is moved to the clipboard.
 

From 6665e89866a4c363f889c9ddce65027eb995bb73 Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 17 Feb 2014 23:07:26 -0800
Subject: [PATCH 5/8] update

---
 hyphenate/scribblings/hyphenate.scrbl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hyphenate/scribblings/hyphenate.scrbl b/hyphenate/scribblings/hyphenate.scrbl
index f414f79f..304b2460 100644
--- a/hyphenate/scribblings/hyphenate.scrbl
+++ b/hyphenate/scribblings/hyphenate.scrbl
@@ -3,7 +3,7 @@
 @(require scribble/eval (for-label racket "../main.rkt"))
 
 @(define my-eval (make-base-eval))
-@(my-eval `(require hyphenate))
+@(my-eval `(require hyphenate xml))
 
 
 @title{Hyphenate}

From c3ce347cba75e7e0ae9e43f10dc8332dc3734716 Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 17 Feb 2014 23:10:42 -0800
Subject: [PATCH 6/8] updates

---
 hyphenate/main.rkt                    | 9 ++++++---
 hyphenate/scribblings/hyphenate.scrbl | 6 +++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/hyphenate/main.rkt b/hyphenate/main.rkt
index 5842b20e..8e83cbb4 100644
--- a/hyphenate/main.rkt
+++ b/hyphenate/main.rkt
@@ -163,7 +163,7 @@
                                      #:min-length [min-length default-min-length])
   ((xexpr? procedure?) ((or/c char? string?) 
                         #:exceptions (listof exception-word?) 
-                        #:min-length (or/c integer? #f)) . ->* . string?)
+                        #:min-length (or/c integer? #f)) . ->* . xexpr/c)
   
   ;; set up module data
   ;; todo?: change set! to parameterize
@@ -183,17 +183,20 @@
       [else x])))
 
 
+
 ;; Default hyphenate function.
 (define+provide/contract (hyphenate x [joiner default-joiner]  
                                     #:exceptions [extra-exceptions '()] 
                                     #:min-length [min-length default-min-length])
   ((xexpr/c) ((or/c char? string?) 
              #:exceptions (listof exception-word?) 
-             #:min-length (or/c integer? #f)) . ->* . string?)
+             #:min-length (or/c integer? #f)) . ->* . xexpr/c)
   (hyphenatef x (λ(x) #t) joiner #:exceptions extra-exceptions #:min-length min-length))
 
+
+
 (define+provide/contract (unhyphenate x [joiner default-joiner])
-  ((xexpr/c) ((or/c char? string?)) . ->* . string?)
+  ((xexpr/c) ((or/c char? string?)) . ->* . xexpr/c)
   
   (define (remove-hyphens text)
     (string-replace text (joiner->string joiner) ""))
diff --git a/hyphenate/scribblings/hyphenate.scrbl b/hyphenate/scribblings/hyphenate.scrbl
index 304b2460..f5ce76e3 100644
--- a/hyphenate/scribblings/hyphenate.scrbl
+++ b/hyphenate/scribblings/hyphenate.scrbl
@@ -33,7 +33,7 @@ After that, you can update the package like so:
 [joiner (or/c char? string?) (integer->char #x00AD)]
 [#:exceptions exceptions (listof string?) empty]
 [#:min-length length (or/c integer? false?) 5])
-string?]
+xexpr/c]
 Hyphenate @racket[_xexpr] by calculating hyphenation points and inserting @racket[_joiner] at those points. By default, @racket[_joiner] is the soft hyphen (Unicode 00AD = decimal 173). Words shorter than @racket[#:min-length] @racket[_length] will not be hyphenated. To hyphenate words of any length, use @racket[#:min-length] @racket[#f].
 
 @margin-note{The REPL displays a soft hyphen as \u00AD. But in ordinary use, you'll only see a soft hyphen when it appears at the end of a line or page as part of a hyphenated word. Otherwise it's not displayed. In most of the examples here, I use a standard hyphen for clarity.}
@@ -101,7 +101,7 @@ Don't send raw HTML through @racket[hyphenate]. It can't distinguish HTML tags a
 [joiner (or/c char? string?) (integer->char \#x00AD)]
 [#:exceptions exceptions (listof string?) empty]
 [#:min-length length (or/c integer? false?) 5])
-string?]
+xexpr/c]
 Like @racket[hyphenate], but only words matching @racket[_pred] are hyphenated. Convenient if you want to prevent hyphenation of certain sets of words, like proper names:
 
 @examples[#:eval my-eval
@@ -128,7 +128,7 @@ It's possible to do fancier kinds of hyphenation restrictions that take account
 (unhyphenate
 [xexpr xexpr/c] 
 [joiner (or/c char? string?) @(integer->char #x00AD)])
-string?]
+xexpr/c]
 Remove @racket[_joiner] from @racket[_xexpr].
 
 A side effect of using @racket[hyphenate] is that soft hyphens (or whatever the @racket[_joiner] is) will be embedded in the output text. If you need to support copying of text, for instance in a GUI application, you'll probably want to strip out the hyphenation before the copied text is moved to the clipboard.

From 92741a90c7e8cc0988635b23e8c2fc43ae7e80bd Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Mon, 17 Feb 2014 23:11:37 -0800
Subject: [PATCH 7/8] update

---
 hyphenate/scribblings/hyphenate.scrbl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hyphenate/scribblings/hyphenate.scrbl b/hyphenate/scribblings/hyphenate.scrbl
index f5ce76e3..221e00a3 100644
--- a/hyphenate/scribblings/hyphenate.scrbl
+++ b/hyphenate/scribblings/hyphenate.scrbl
@@ -1,6 +1,6 @@
 #lang scribble/manual
 
-@(require scribble/eval (for-label racket "../main.rkt"))
+@(require scribble/eval (for-label racket "../main.rkt" xml))
 
 @(define my-eval (make-base-eval))
 @(my-eval `(require hyphenate xml))

From 8f77d5e4e96e295c95687996a7d1c39b2896e28c Mon Sep 17 00:00:00 2001
From: Matthew Butterick <mb@mbtype.com>
Date: Wed, 19 Feb 2014 13:08:51 -0800
Subject: [PATCH 8/8] add xexpr support

---
 hyphenate/info.rkt                    |  2 +-
 hyphenate/main.rkt                    | 52 +++++++++++----------------
 hyphenate/scribblings/hyphenate.scrbl | 32 ++++++++++-------
 hyphenate/tests.rkt                   |  7 +++-
 4 files changed, 47 insertions(+), 46 deletions(-)

diff --git a/hyphenate/info.rkt b/hyphenate/info.rkt
index 669b1011..329e1fdc 100644
--- a/hyphenate/info.rkt
+++ b/hyphenate/info.rkt
@@ -1,4 +1,4 @@
 #lang info
 (define collection "hyphenate")
-
 (define scribblings '(("scribblings/hyphenate.scrbl" ())))
+(define deps '("txexpr"))
\ No newline at end of file
diff --git a/hyphenate/main.rkt b/hyphenate/main.rkt
index 8e83cbb4..b4d0932d 100644
--- a/hyphenate/main.rkt
+++ b/hyphenate/main.rkt
@@ -1,9 +1,7 @@
 #lang racket/base
 (require (for-syntax racket/base))
 (require racket/string racket/list racket/contract racket/vector)
-(require "patterns.rkt" "exceptions.rkt" tagged-xexpr xml)
-
-(module+ test (require rackunit))
+(require "patterns.rkt" "exceptions.rkt" txexpr xml)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; Hyphenate module
@@ -25,10 +23,10 @@
          (provide (contract-out [name contract]))
          (define name body ...))]))
 
-;; global data, define now but set! them later (because they're potentially big & slow)
+;; module data, define now but set! them later (because they're potentially big & slow)
 (define exceptions #f)
 (define pattern-tree #f)
-;; global default values
+;; module default values
 (define default-min-length 5)
 (define default-joiner (integer->char #x00AD))
 
@@ -43,18 +41,10 @@
   
   (make-hash (map (λ(x) (cons (make-key x) (make-value x))) exn-strings)))
 
-
-
 ;; An exception-word is a string of word characters or hyphens.
 (define (exception-word? x)
   (if (regexp-match #px"^[\\w-]+$" x) #t #f))
 
-(module+ test
-  (check-true (exception-word? "Foobar"))
-  (check-true (exception-word? "foobar"))
-  (check-false (exception-word? "foobar!"))
-  (check-true (exception-word? "foo-bar"))
-  (check-false (exception-word? "foo bar")))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;; Helper functions
@@ -157,6 +147,15 @@
 (define (joiner->string joiner)
   (if (char? joiner) (format "~a" joiner) joiner))
 
+;; helper macro that applies proc to all strings found in xexpr input
+(define-syntax (apply-xexpr-strings stx)
+  (syntax-case stx ()
+    [(_ proc val) #'(let loop ([x val])
+                      (cond
+                        [(string? x) (proc x)]
+                        [(txexpr? x) (map-elements loop x)]
+                        [else x]))]))
+
 ;; Hyphenate using a filter procedure.
 (define+provide/contract (hyphenatef x proc [joiner default-joiner] 
                                      #:exceptions [extra-exceptions '()]  
@@ -176,37 +175,26 @@
   (define (insert-hyphens text)
     (regexp-replace* word-pattern text (λ(word) (if (proc word) (string-join (word->hyphenation-points word min-length) joiner-string) word))))
   
-  (let &hyphenate ([x x])
-    (cond
-      [(string? x) (insert-hyphens x)]
-      [(tagged-xexpr? x) (map-elements &hyphenate x)]
-      [else x])))
-
+  (apply-xexpr-strings insert-hyphens x))
 
 
-;; Default hyphenate function.
+;; Default hyphenate is a special case of hyphenatef.
 (define+provide/contract (hyphenate x [joiner default-joiner]  
                                     #:exceptions [extra-exceptions '()] 
                                     #:min-length [min-length default-min-length])
   ((xexpr/c) ((or/c char? string?) 
-             #:exceptions (listof exception-word?) 
-             #:min-length (or/c integer? #f)) . ->* . xexpr/c)
+              #:exceptions (listof exception-word?) 
+              #:min-length (or/c integer? #f)) . ->* . xexpr/c)
   (hyphenatef x (λ(x) #t) joiner #:exceptions extra-exceptions #:min-length min-length))
 
 
-
+;; Remove hyphens.
 (define+provide/contract (unhyphenate x [joiner default-joiner])
   ((xexpr/c) ((or/c char? string?)) . ->* . xexpr/c)
   
   (define (remove-hyphens text)
     (string-replace text (joiner->string joiner) ""))
   
-  (let &unhyphenate ([x x])
-    (cond
-      [(string? x) (remove-hyphens x)]
-      [(tagged-xexpr? x) (map-elements &unhyphenate x)]
-      [else x])))
-  
-  
-  
-  
\ No newline at end of file
+  (apply-xexpr-strings remove-hyphens x))  
+
+
diff --git a/hyphenate/scribblings/hyphenate.scrbl b/hyphenate/scribblings/hyphenate.scrbl
index 221e00a3..e98748f2 100644
--- a/hyphenate/scribblings/hyphenate.scrbl
+++ b/hyphenate/scribblings/hyphenate.scrbl
@@ -36,7 +36,7 @@ After that, you can update the package like so:
 xexpr/c]
 Hyphenate @racket[_xexpr] by calculating hyphenation points and inserting @racket[_joiner] at those points. By default, @racket[_joiner] is the soft hyphen (Unicode 00AD = decimal 173). Words shorter than @racket[#:min-length] @racket[_length] will not be hyphenated. To hyphenate words of any length, use @racket[#:min-length] @racket[#f].
 
-@margin-note{The REPL displays a soft hyphen as \u00AD. But in ordinary use, you'll only see a soft hyphen when it appears at the end of a line or page as part of a hyphenated word. Otherwise it's not displayed. In most of the examples here, I use a standard hyphen for clarity.}
+@margin-note{The REPL displays a soft hyphen as @code{\u00AD}. But in ordinary use, you'll only see a soft hyphen when it appears at the end of a line or page as part of a hyphenated word. Otherwise it's not displayed. In most of the examples here, I use a standard hyphen for clarity (by adding @code{#\-} as an argument).}
 
 @examples[#:eval my-eval
      (hyphenate "ergo polymorphic")
@@ -49,7 +49,7 @@ Because the hyphenation is based on an algorithm rather than a dictionary, it ma
 
 @examples[#:eval my-eval
     (hyphenate "scraunched strengths" #\-)
-     (hyphenate "Racketcon" #\-)
+     (hyphenate "RacketCon" #\-)
      (hyphenate "supercalifragilisticexpialidocious" #\-)
    ]
 
@@ -79,18 +79,20 @@ For this reason, certain words can't be hyphenated algorithmically, because the
    
 This is the right result. If you used @italic{adder} to mean the machine, it would be hyphenated @italic{add-er}; if you meant the snake, it would be @italic{ad-der}. Better to avoid hyphenation than to hyphenate incorrectly.
 
-You can send HTML-style X-expressions through @racket[hyphenate]. It will recursively hyphenate the text strings, while leaving the tags and attributes alone.
+You can send HTML-style X-expressions through @racket[hyphenate]. It will recursively hyphenate the text strings, while leaving the tags and attributes alone, as well as non-hyphenatable material (like character entities and CDATA).
 
 @examples[#:eval my-eval
-     (hyphenate '(p "polymorphically" (em "formatted" (strong "snowmen")))) 
+     (hyphenate '(p "strangely" (em "formatted" (strong "snowmen"))) #\-)
+     (hyphenate '(headline [[class "headline"]] "headline") #\-)
+      (hyphenate '(div "The (span epsilon) entity:" epsilon) #\-) 
    ]
 
-Don't send raw HTML through @racket[hyphenate]. It can't distinguish HTML tags and attributes from textual content, so everything will be hyphenated, thus goofing up your file. But you can easily convert HTML to an X-expression, hyphenate it, and then convert back.
+Don't send raw HTML or XML through @racket[hyphenate]. It can't distinguish tags and attributes from textual content, so everything will be hyphenated, thus goofing up your file. But you can easily convert HTML or XML to an X-expression, hyphenate it, and then convert back.
 
 @examples[#:eval my-eval
-    (define html "<body style=\"background: yellow\">Hello snowman</body>")
-    (hyphenate html)
-    (xexpr->string (hyphenate (string->xexpr html))) 
+    (define html "<body style=\"background: yellow\">Hello</body>")
+    (hyphenate html #\-)
+    (xexpr->string (hyphenate (string->xexpr html) #\-)) 
    ]
    
 
@@ -116,11 +118,12 @@ Sometimes you need @racket[hyphenatef] to prevent unintended consequences. For i
 @examples[#:eval my-eval
 (hyphenate "Hufflepuff golfing final on Tuesday" #\-)
 (define (no-ligs? word)
-  (not (ormap (λ(lig) (regexp-match lig word)) '("ff" "fi" "fl" "ffi" "ffl"))))
+  (not (ormap (λ(lig) (regexp-match lig word)) 
+  '("ff" "fi" "fl" "ffi" "ffl"))))
 (hyphenatef "Hufflepuff golfing final on Tuesday" no-ligs? #\-) 
 ]
  
-@margin-note{``Wouldn't it be better to exclude certain pairs of letters rather than whole words?'' Yes. But for now, not supported.}
+@margin-note{``Wouldn't it be better to exclude certain pairs of letters rather than whole words?'' Yes. But for now, that's not supported.}
 
 It's possible to do fancier kinds of hyphenation restrictions that take account of context, like not hyphenating the last word of a paragraph. But @racket[hyphenatef] only operates on words. So you'll have to write some fancier code. Separate out the words eligible for hyphenation, and then send them through good old @racket[hyphenate].
 
@@ -129,7 +132,12 @@ It's possible to do fancier kinds of hyphenation restrictions that take account
 [xexpr xexpr/c] 
 [joiner (or/c char? string?) @(integer->char #x00AD)])
 xexpr/c]
-Remove @racket[_joiner] from @racket[_xexpr].
+Remove @racket[_joiner] from @racket[_xexpr]. Like @racket[hyphenate], it works on nested X-expressions.
+
+@examples[#:eval my-eval
+     (hyphenate '(p "strangely" (em "formatted" (strong "snowmen"))) #\-)
+     (unhyphenate '(p "strange-ly" (em "for-mat-ted" (strong "snow-men"))) #\-)
+]
 
 A side effect of using @racket[hyphenate] is that soft hyphens (or whatever the @racket[_joiner] is) will be embedded in the output text. If you need to support copying of text, for instance in a GUI application, you'll probably want to strip out the hyphenation before the copied text is moved to the clipboard.
 
@@ -145,7 +153,7 @@ Use this function cautiously — if @racket[_joiner] appeared in the original in
      (unhyphenate (hyphenate "ribbon-cutting ceremony" #\-) #\-)
     ]
     
-It's also possible that soft hyphens could appear in your input string. Certain word processors allow users to @link["http://practicaltypography.com/optional-hyphens.html"]{insert soft hyphens} in their text.
+Keep in mind that soft hyphens could appear in your input string. Certain word processors allow users to @link["http://practicaltypography.com/optional-hyphens.html"]{insert soft hyphens} in their text.
 
 @examples[#:eval my-eval
      (hyphenate "True\u00ADType typefaces")
diff --git a/hyphenate/tests.rkt b/hyphenate/tests.rkt
index e2110799..59a55e89 100644
--- a/hyphenate/tests.rkt
+++ b/hyphenate/tests.rkt
@@ -3,7 +3,7 @@
 
 (require rackunit)
 
-(require/expose "main.rkt" (word->hyphenation-points))
+(require/expose "main.rkt" (word->hyphenation-points exception-word?))
 
 (check-equal? (hyphenate "polymorphism") "poly\u00ADmor\u00ADphism")
 (check-equal? (hyphenate "polymorphism" #:min-length 100) "polymorphism")
@@ -21,3 +21,8 @@
 (check-equal? (word->hyphenation-points "polymorphism") '("poly" "mor" "phism"))
 (check-equal? (word->hyphenation-points "present") '("present")) ; exception word
 
+(check-true (exception-word? "Foobar"))
+(check-true (exception-word? "foobar"))
+(check-false (exception-word? "foobar!"))
+(check-true (exception-word? "foo-bar"))
+(check-false (exception-word? "foo bar"))
\ No newline at end of file