first set of files

hyphenate/info.rkt

@@ -0,0 +1,2 @@
+#lang info
+(define collection "hyphenate")

hyphenate/main.rkt

@@ -0,0 +1,156 @@
+#lang racket/base
+(require racket/string racket/list racket/contract racket/vector)
+(require "data.rkt" "readability.rkt")
+
+(module+ test (require rackunit))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;; Hyphenate.rkt
+;;; Racket port of Ned Batchelder's hyphenate.py
+;;; http://nedbatchelder.com/code/modules/hyphenate.html
+;;; (in the public domain)
+;;; which in turn was an implementation
+;;; of the Liang hyphenation algorithm in TeX
+;;; (also in the public domain)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(provide hyphenate hyphenatef)
+
+;; global data, define now but set! them later (because they're potentially big & slow)
+(define exceptions #f)
+(define pattern-tree #f)
+
+;; Convert the hyphenated pattern into a point array for use later.
+(define/contract (list->exceptions exn-strings) 
+  ((listof string?) . -> . hash?)
+  (define (make-key x)
+    (string-replace x "-" ""))
+  
+  (define (make-value x)
+    (list->vector (cons 0 (map (λ(x) (if (equal? x "-") 1 0)) (regexp-split #px"[a-z]" x)))))
+  
+  (make-hash (map (λ(x) (cons (make-key x) (make-value x))) exn-strings)))
+
+
+
+;; A word is a string without whitespace.
+(define/contract (word? x)
+  (any/c . -> . boolean?)
+  (->boolean (regexp-match #px"^\\S+$" x)))
+
+(module+ test
+  (check-true (word? "Foobar"))
+  (check-true (word? "foobar"))
+  (check-true (word? "foo-bar"))
+  (check-false (word? "foo bar")))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;; Helper functions
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert a pattern like 'a1bc3d4' into a string of chars 'abcd'
+;; and a list of points [ 1, 0, 3, 4 ].
+(define (make-pattern-tree pattern-data)
+  (define tree (make-hash))
+  
+  ;; Insert the pattern into the tree.  Each character finds a dict
+  ;; another level down in the tree, and leaf nodes have the list of
+  ;; points.
+  (define (insert-pattern pat)
+    (let* ([chars (regexp-replace* #px"[0-9]" pat "")]
+           ;; regexp returns list of strings
+           [points (map (λ(x) (if (> (len x) 0) (string->number x) 0)) (regexp-split #px"[.a-z]" pat))]
+           [tree tree])
+      (for ([char chars])
+        (when (not (hash-has-key? tree char))
+          (hash-set! tree char (make-hash)))
+        (set! tree (hash-ref tree char)))
+      (hash-set! tree empty points)))
+  (map insert-pattern pattern-data)
+  tree)
+
+
+
+(define/contract (make-points word) 
+  (word? . -> . vector?)
+  
+  (define/contract (make-zeroes points)
+    (vector? . -> . vector?)
+    ; controls hyphenation zone from edges of word
+    ; todo: parameterize this setting
+    ; todo: does this count end-of-word punctuation? it shouldn't.
+    (map (λ(i) (vector-set! points i 0)) (list 1 2 (- (len points) 2) (- (len points) 3)))
+    points)
+  
+  (let* ([word (string-downcase word)]
+         [points 
+          (if (hash-has-key? exceptions word)
+              (hash-ref exceptions word)
+              (let* ([work (string-append "." (->string word) ".")]
+                     [points (make-vector (add1 (len work)) 0)]) 
+                (for ([i (len work)])
+                  (let ([tree pattern-tree])
+                    (for ([char (substring work i (len work))]
+                          #:break (not (hash-has-key? tree char)))
+                      (set! tree (hash-ref tree char))
+                      (when (hash-has-key? tree empty)
+                        (let ([point (hash-ref tree empty)])
+                          (for ([j (len point)])
+                            (vector-set! points (+ i j) (max (vector-ref points (+ i j)) (list-ref point j)))))))))
+                points))])
+    
+    ; make-zeroes controls minimum hyphenation distance from edge.
+    ; todo: dropping first 2 elements is needed for mysterious reasons to be documented later
+    (vector-drop (make-zeroes points) 2)))
+
+
+
+;; Find hyphenatable pieces of a word. This is not quite synonymous with syllables.
+(define/contract (word->pieces word #:min-length [min-length 4])
+  ((word?) (#:min-length integer?) . ->* . (listof string?))  
+  
+  (define (make-pieces word)
+    (define word-dissected (flatten (for/list ([char word] 
+                                               [point (make-points word)])
+                                      (if (even? point)
+                                          char ; even point denotes character
+                                          (cons char 'syllable))))) ; odd point denotes char + syllable
+    (map list->string (splitf-at* word-dissected symbol?)))
+  
+  (if (<= (len word) min-length)
+      (list word)  
+      (make-pieces word)))
+
+(define default-joiner (integer->char #x00AD))
+
+;; Hyphenate using a filter procedure.
+;; Theoretically possible to do this externally,
+;; but it would just mean doing the regexp-replace twice.
+(define/contract (hyphenatef text proc [joiner default-joiner] #:exceptions [extra-exceptions '()])
+  ((string? procedure?) (char? #:exceptions (listof word?)) . ->* . string?)
+  
+  ;; set up module data
+  (set! exceptions (list->exceptions (append default-exceptions (map ->string extra-exceptions))))
+  (when (not pattern-tree) (set! pattern-tree (make-pattern-tree default-patterns)))
+  
+  (regexp-replace* #px"\\w+" text (λ(word) (if (proc word) (string-join (word->pieces word) (->string joiner)) word))))
+
+
+;; Default hyphenate function.
+(define/contract (hyphenate text [joiner default-joiner]  #:exceptions [extra-exceptions '()])
+  ((string?) (char? #:exceptions (listof word?)) . ->* . string?) 
+  (hyphenatef text (λ(x) #t) joiner #:exceptions extra-exceptions))
+
+
+
+(module+ test
+  (check-equal? (hyphenate "polymorphism") "poly\u00ADmor\u00ADphism")
+  (check-equal? (hyphenatef "polymorphism" (λ(x) #f)) "polymorphism")
+  (check-equal? (hyphenate "polymorphism" #\-) "poly-mor-phism")
+  (check-equal? (hyphenate "polymorphism" #\* #:exceptions '("polymo-rphism")) "polymo*rphism")
+  
+  (check-equal? (hyphenate "circular polymorphism squandering") "cir\u00ADcu\u00ADlar poly\u00ADmor\u00ADphism squan\u00ADder\u00ADing")
+  (check-equal? (hyphenate "present project") "present project") ; exception words
+  ;; test these last so exceptions have been set up already
+  (check-equal? (word->pieces "polymorphism") '("poly" "mor" "phism"))
+  (check-equal? (word->pieces "present") '("present"))) ; exception word

hyphenate/readability.rkt

@@ -0,0 +1,153 @@
+#lang racket/base
+(require racket/contract net/url xml)
+(require (only-in racket/list empty? range splitf-at dropf dropf-right))
+(require (only-in racket/format ~a))
+(require (only-in racket/string string-join))
+(require (only-in racket/vector vector-member))
+(require (only-in racket/set set set->list set?))
+(module+ test (require rackunit))
+
+(provide (all-defined-out))
+
+
+
+;; general way of coercing to string
+(define/contract (->string x)
+  (any/c . -> . string?)
+  (cond 
+    [(string? x) x]
+    [(empty? x) ""]
+    [(symbol? x) (symbol->string x)]
+    [(number? x) (number->string x)]
+    [(path? x) (path->string x)]
+    [(char? x) (~a x)]
+    [(xexpr? x) (xexpr->string x)] ; put this last so other xexprish things don't get caught
+    [else (error (format "Can't make ~a into string" x))]))
+
+(module+ test
+  (check-equal? (->string "foo") "foo")
+  (check-equal? (->string '()) "")
+  (check-equal? (->string 'foo) "foo")
+  (check-equal? (->string 123) "123")
+  (define file-name-as-text "foo.txt")
+  (check-equal? (->string (string->path file-name-as-text)) file-name-as-text)
+  (check-equal? (->string #\¶) "¶")
+  (check-equal? (->string '(foo "bar")) "<foo>bar</foo>"))
+
+
+
+
+
+
+
+;; general way of coercing to a list
+(define/contract (->list x)
+  (any/c . -> . list?)
+  (cond 
+    [(list? x) x]
+    [(vector? x) (vector->list x)]
+    [(set? x) (set->list x)]
+    [else (list x)])) 
+
+(module+ test
+  (check-equal? (->list '(1 2 3)) '(1 2 3))
+  (check-equal? (->list (list->vector '(1 2 3))) '(1 2 3))
+  (check-equal? (->list (set 1 2 3)) '(3 2 1))
+  (check-equal? (->list "foo") (list "foo")))
+
+;; general way of coercing to vector
+(define (->vector x)
+  (any/c . -> . vector?)
+  ; todo: on bad input, it will pop a list error rather than vector error
+  (cond
+    [(vector? x) x]
+    [else (list->vector (->list x))]))
+
+
+
+;; general way of coercing to boolean
+(define/contract (->boolean x)
+  (any/c . -> . boolean?)
+  ;; in Racket, everything but #f is true
+  (if x #t #f))
+
+(module+ test
+  (check-true (->boolean #t))
+  (check-false (->boolean #f))
+  (check-true (->boolean "#f")) 
+  (check-true (->boolean "foo"))
+  (check-true (->boolean '()))
+  (check-true (->boolean '(1 2 3))))
+
+
+(define/contract (has-length? x)
+  (any/c . -> . boolean?)
+  (ormap (λ(proc) (proc x)) (list list? string? symbol? vector? hash? set?)))
+
+;; general way of asking for length
+(define/contract (len x)
+  (has-length? . -> . integer?)
+  (cond
+    [(list? x) (length x)]
+    [(string? x) (string-length x)]
+    [(symbol? x) (len (->string x))]
+    [(vector? x) (len (->list x))]
+    [(hash? x) (len (hash-keys x))]
+    [(set? x) (len (->list x))]
+    [else #f]))
+
+(module+ test
+  (check-equal? (len '(1 2 3)) 3)
+  (check-not-equal? (len '(1 2)) 3) ; len 2
+  (check-equal? (len "foo") 3)
+  (check-not-equal? (len "fo") 3) ; len 2
+  (check-equal? (len 'foo) 3)
+  (check-not-equal? (len 'fo) 3) ; len 2
+  (check-equal? (len (list->vector '(1 2 3))) 3)
+  (check-not-equal? (len (list->vector '(1 2))) 3) ; len 2
+  (check-equal? (len (set 1 2 3)) 3)
+  (check-not-equal? (len (set 1 2)) 3) ; len 2
+  (check-equal? (len (make-hash '((a . 1) (b . 2) (c . 3)))) 3)
+  (check-not-equal? (len (make-hash '((a . 1) (b . 2)))) 3)) ; len 2
+
+
+
+
+
+
+;; trim from beginning & end of list
+(define (trim items test-proc)
+  (list? procedure? . -> . list?)
+  (dropf-right (dropf items test-proc) test-proc))
+
+(module+ test
+  ;  (check-equal? (trim (list "\n" " " 1 2 3 "\n") whitespace?) '(1 2 3))
+  (check-equal? (trim (list 1 3 2 4 5 6 8 9 13) odd?) '(2 4 5 6 8)))
+
+
+
+;; split list into list of sublists using test-proc
+(define/contract (splitf-at* xs split-test)
+  ;; todo: better error message when split-test is not a predicate
+  (list? predicate/c . -> . (listof list?))
+  (define (&splitf-at* xs [acc '()]) ; use acc for tail recursion
+    (if (empty? xs) 
+        ;; reverse because accumulation is happening backward 
+        ;; (because I'm using cons to push latest match onto front of list)
+        (reverse acc)
+        (let-values ([(item rest) 
+                      ;; drop matching elements from front
+                      ;; then split on nonmatching 
+                      ;; = nonmatching item + other elements (which will start with matching)
+                      (splitf-at (dropf xs split-test) (compose1 not split-test))])
+          ;; recurse, and store new item in accumulator
+          (&splitf-at* rest (cons item acc)))))
+  
+  ;; trim off elements matching split-test
+  (&splitf-at* (trim xs split-test)))
+
+(module+ test
+  ; (check-equal? (splitf-at* '("foo" " " "bar" "\n" "\n" "ino") whitespace?) '(("foo")("bar")("ino")))
+  (check-equal? (splitf-at* '(1 2 3 4 5 6) even?) '((1)(3)(5))))
+
+