reuse places in `preheat-cache` for speed

pollen/private/preheat-cache.rkt

@@ -1,8 +1,9 @@
-#lang racket/base
+#lang debug racket/base
 (require racket/file
          racket/path
          racket/place
          racket/list
+         racket/match
          sugar/list
          "file-utils.rkt"
          "cache-utils.rkt"
@@ -17,34 +18,25 @@
   (and (file-exists? cache-db-file)
        (hash-has-key? (file->value cache-db-file) (paths->key path))))
 
-;; compile a path inside a place (= parallel processing)
-(define (path-into-place starting-dir path)
-  (message (format "caching: ~a" (find-relative-path starting-dir path)))
-  (define p
-    (place ch
-           (define path (place-channel-get ch))
-           (define-values (_ path-name __) (split-path path))
-           (message (format "compiling: ~a" path))
-           ;; use #false to signal compile error. Otherwise allow errors to pass.
-           (define result
-             (with-handlers ([exn:fail? (λ (e) (message (format "compile failed: ~a" path-name)) #false)])
-               (path->hash path)))
-           (place-channel-put ch result)))
-  (place-channel-put p path)
-  p)
-
 (define (preheat-cache starting-dir)
   (unless (and (path-string? starting-dir) (directory-exists? starting-dir))
     (raise-argument-error 'preheat-cache "directory" starting-dir))
   (define max-places (processor-count)) ; number of parallel processes to spawn at a time
+  (define worker-places (for/list ([i (in-range max-places)])
+                          (place ch
+                                 (let loop ()
+                                   (define result (with-handlers ([exn:fail? (λ (e) #false)])
+                                                        (path->hash (place-channel-get ch))))
+                                   (place-channel-put ch result)
+                                   (loop)))))
   (define paths-that-should-be-cached
     (for/list ([path (in-directory starting-dir)]
                #:when (for/or ([proc (in-list (list preproc-source?
                                                     markup-source?
                                                     markdown-source?
                                                     pagetree-source?))])
-                              (proc path)))
+                        (proc path)))
-              path))
+      path))
 
   ;; if a file is already in the cache, no need to hit it again.
   ;; this allows partially completed preheat jobs to resume.
@@ -54,9 +46,12 @@
   ;; that way, if there's a failure, the progress is preserved.
   ;; but the slowest file in a group will prevent further progress.
   (for ([path-group (in-list (slice-at uncached-paths max-places))])
-       (define path-places (map (λ (pg) (path-into-place starting-dir pg)) path-group))
+    (for ([path (in-list path-group)]
-       (for ([path (in-list path-group)]
+          [wp (in-list worker-places)])
-             [ppl (in-list path-places)])
+      (message (format "caching: ~a" (find-relative-path starting-dir path)))
-            (define result (place-channel-get ppl))
+      (place-channel-put wp path))
-            (when result ; #false is used to signal compile error
+    (for ([path (in-list path-group)]
-              (cache-ref! (paths->key path) (λ () result))))))
+          [wp (in-list worker-places)])
+      (match (place-channel-get wp)
+        [#false (message (format "compile failed: ~a" path))]
+        [result (cache-ref! (paths->key path) (λ () result))]))))

pollen/private/ts.rktd

@@ -1 +1 @@
-1555990565
+1555997664