dev-stylish
Matthew Butterick 6 years ago
parent 3e0f6c2398
commit 7f73c03da3

@ -1,55 +1,62 @@
#lang racket/base #lang racket/base
(require "file-utils.rkt" racket/file "cache-utils.rkt" "debug.rkt" racket/path racket/place sugar/list) (require racket/file
racket/path
racket/place
racket/list
sugar/list
"file-utils.rkt"
"cache-utils.rkt"
"debug.rkt")
(provide preheat-cache) (provide preheat-cache)
(define (preheat-cache starting-dir) (define (path-cached? path)
(unless (and (path-string? starting-dir) (directory-exists? starting-dir)) ;; #true = already cached; #false = not cached
(error 'preheat-cache (format "~a is not a directory" starting-dir)))
(define max-places (processor-count)) ; number of parallel processes to spawn at a time
(define paths-that-should-be-cached (for/list ([path (in-directory starting-dir)]
#:when (for/or ([proc (in-list (list preproc-source?
markup-source?
markdown-source?
pagetree-source?))])
(proc path)))
path))
;; if a file is already in the cache, no need to hit it again.
;; this allows partially completed preheat jobs to resume.
(define uncached-paths (filter
(λ (path)
;; #t = not cached; #f = already cached
;; seems like it would be slow to load cache.rktd but it's not. ;; seems like it would be slow to load cache.rktd but it's not.
(define-values (_ private-cache-dir) (make-cache-dirs path)) (define-values (_ private-cache-dir) (make-cache-dirs path))
(define cache-db-file (build-path private-cache-dir "cache.rktd")) (define cache-db-file (build-path private-cache-dir "cache.rktd"))
(cond (or (file-exists? cache-db-file)
[(not (file-exists? cache-db-file)) #t] (hash-has-key? (file->value cache-db-file) (paths->key path))))
[else (define cache-db (file->value cache-db-file))
(not (hash-has-key? cache-db (paths->key path)))])) paths-that-should-be-cached))
;; compile a path inside a place (= parallel processing) ;; compile a path inside a place (= parallel processing)
(define (path-into-place path) (define (path-into-place starting-dir path)
(message (format "caching: ~a" (find-relative-path starting-dir path))) (message (format "caching: ~a" (find-relative-path starting-dir path)))
(define p (place ch (define p
(place ch
(define path (place-channel-get ch)) (define path (place-channel-get ch))
(define-values (path-dir path-name _) (split-path path)) (define-values (_ path-name __) (split-path path))
(message (format "compiling: ~a" path)) (message (format "compiling: ~a" path))
;; use #f to signal compile error. Otherwise allow errors to pass. ;; use #false to signal compile error. Otherwise allow errors to pass.
(define result (with-handlers ([exn:fail? (λ _ (message (format "compile failed: ~a" path-name)) #f)]) (define result
(with-handlers ([exn:fail? (λ (e) (message (format "compile failed: ~a" path-name)) #false)])
(path->hash path))) (path->hash path)))
(place-channel-put ch result))) (place-channel-put ch result)))
(place-channel-put p path) (place-channel-put p path)
p) p)
(define (preheat-cache starting-dir)
(unless (and (path-string? starting-dir) (directory-exists? starting-dir))
(raise-argument-error 'preheat-cache "directory" starting-dir))
(define max-places (processor-count)) ; number of parallel processes to spawn at a time
(define paths-that-should-be-cached
(for/list ([path (in-directory starting-dir)]
#:when (for/or ([proc (in-list (list preproc-source?
markup-source?
markdown-source?
pagetree-source?))])
(proc path)))
path))
;; if a file is already in the cache, no need to hit it again.
;; this allows partially completed preheat jobs to resume.
(define uncached-paths (filter-not path-cached? paths-that-should-be-cached))
;; compile the paths in groups, so they can be incrementally saved. ;; compile the paths in groups, so they can be incrementally saved.
;; that way, if there's a failure, the progress is preserved. ;; that way, if there's a failure, the progress is preserved.
;; but the slowest file in a group will prevent further progress. ;; but the slowest file in a group will prevent further progress.
(for ([path-group (in-list (slice-at uncached-paths max-places))]) (for ([path-group (in-list (slice-at uncached-paths max-places))])
(define path-places (map path-into-place path-group)) (define path-places (map (λ (pg) (path-into-place starting-dir pg)) path-group))
(for ([path (in-list path-group)] (for ([path (in-list path-group)]
[ppl (in-list path-places)]) [ppl (in-list path-places)])
(define result (place-channel-get ppl)) (define result (place-channel-get ppl))
(when result ; #f is used to signal compile error (when result ; #false is used to signal compile error
(cache-ref! (paths->key path) (λ _ result)))))) (cache-ref! (paths->key path) (λ () result))))))

@ -1 +1 @@
1540858397 1540858400

Loading…
Cancel
Save