|
|
|
@ -48,16 +48,7 @@
|
|
|
|
|
|
|
|
|
|
(define (list-of-pathish? x) (and (list? x) (andmap pathish? x)))
|
|
|
|
|
|
|
|
|
|
(define+provide/contract (render-batch #:parallel [wants-parallel-render #false] . paths-in)
|
|
|
|
|
((#:parallel any/c) #:rest list-of-pathish? . ->* . void?)
|
|
|
|
|
;; Why not just (for-each render ...)?
|
|
|
|
|
;; Because certain files will pass through multiple times (e.g., templates)
|
|
|
|
|
;; And with render, they would be rendered repeatedly.
|
|
|
|
|
;; Using reset-modification-dates is sort of like session control.
|
|
|
|
|
(reset-mod-date-hash!)
|
|
|
|
|
(cond
|
|
|
|
|
[wants-parallel-render
|
|
|
|
|
|
|
|
|
|
(define (parallel-render paths-in job-count-arg)
|
|
|
|
|
(define source-paths
|
|
|
|
|
(let loop ([paths paths-in] [acc null])
|
|
|
|
|
(match (and (pair? paths) (->complete-path (car paths)))
|
|
|
|
@ -69,10 +60,10 @@
|
|
|
|
|
(define job-count
|
|
|
|
|
(min
|
|
|
|
|
(length source-paths)
|
|
|
|
|
(match wants-parallel-render
|
|
|
|
|
(match job-count-arg
|
|
|
|
|
[#true (processor-count)]
|
|
|
|
|
[(? exact-positive-integer? count) count]
|
|
|
|
|
[_ (raise-argument-error 'render-batch "exact positive integer" wants-parallel-render)])))
|
|
|
|
|
[_ (raise-argument-error 'render-batch "exact positive integer" job-count-arg)])))
|
|
|
|
|
|
|
|
|
|
;; initialize the workers
|
|
|
|
|
(define worker-evts
|
|
|
|
@ -96,7 +87,6 @@
|
|
|
|
|
|
|
|
|
|
;; `locks` and `blocks` are (listof (cons/c evt? path?))
|
|
|
|
|
(define starting-source-paths (sort source-paths string<? #:key path->string))
|
|
|
|
|
(define crashed-jobs
|
|
|
|
|
(let loop ([source-paths starting-source-paths]
|
|
|
|
|
[locks-in null]
|
|
|
|
|
[blocks-in null]
|
|
|
|
@ -117,6 +107,16 @@
|
|
|
|
|
(values (cons block locks) blocks)])))
|
|
|
|
|
(cond
|
|
|
|
|
[(eq? completed-job-count (length starting-source-paths))
|
|
|
|
|
;; second bite at the apple for crashed jobs.
|
|
|
|
|
;; 1) many crashes that arise in parallel rendering are
|
|
|
|
|
;; a result of concurrency issues (e.g. shared files not being readable at the right moment).
|
|
|
|
|
;; That is, they do not appear under serial rendering.
|
|
|
|
|
;; 2) even if a crash is legit (that is, there is a real flaw in the source)
|
|
|
|
|
;; and should be raised, we don't want to do it inside a parallel-rendering `place`
|
|
|
|
|
;; because then the place will never return, and the whole parallel job will never finish.
|
|
|
|
|
;; so we take the list of crashed jobs and try rendering them again serially.
|
|
|
|
|
;; if it was a concurrency-related error, it will disappear.
|
|
|
|
|
;; if it was a legit error, the render will stop and print a trace.
|
|
|
|
|
;; crashed jobs are completed jobs that weren't finished
|
|
|
|
|
(for/list ([(path finished?) (in-dict completed-jobs)]
|
|
|
|
|
#:unless finished?)
|
|
|
|
@ -129,7 +129,7 @@
|
|
|
|
|
[(cons path rest)
|
|
|
|
|
(place-channel-put wp (cons path poly-target))
|
|
|
|
|
(loop rest locks blocks completed-jobs completed-job-count)])]
|
|
|
|
|
[(list wpidx wp (and (or 'finished-job 'crashed-job) tag) path ms)
|
|
|
|
|
[(list wpidx wp (and (or 'finished-job 'crashed-job) tag) path ms)
|
|
|
|
|
(match tag
|
|
|
|
|
['finished-job
|
|
|
|
|
(message
|
|
|
|
@ -151,18 +151,18 @@
|
|
|
|
|
(add1 completed-job-count))]
|
|
|
|
|
[(list wpidx wp 'wants-lock path)
|
|
|
|
|
(loop source-paths locks (append blocks (list (cons wp path))) completed-jobs completed-job-count)])])))
|
|
|
|
|
;; second bite at the apple for crashed jobs.
|
|
|
|
|
;; 1) many crashes that arise in parallel rendering are
|
|
|
|
|
;; a result of concurrency issues (e.g. shared files not being readable at the right moment).
|
|
|
|
|
;; That is, they do not appear under serial rendering.
|
|
|
|
|
;; 2) even if a crash is legit (that is, there is a real flaw in the source)
|
|
|
|
|
;; and should be raised, we don't want to do it inside a parallel-rendering `place`
|
|
|
|
|
;; because then the place will never return, and the whole parallel job will never finish.
|
|
|
|
|
;; so we take the list of crashed jobs and try rendering them again serially.
|
|
|
|
|
;; if it was a concurrency-related error, it will disappear.
|
|
|
|
|
;; if it was a legit error, the render will stop and print a trace.
|
|
|
|
|
(for-each render-from-source-or-output-path crashed-jobs)]
|
|
|
|
|
[else (for-each render-from-source-or-output-path paths-in)]))
|
|
|
|
|
|
|
|
|
|
(define+provide/contract (render-batch #:parallel [wants-parallel-render #false] . paths-in)
|
|
|
|
|
((#:parallel any/c) #:rest list-of-pathish? . ->* . void?)
|
|
|
|
|
;; Why not just (for-each render ...)?
|
|
|
|
|
;; Because certain files will pass through multiple times (e.g., templates)
|
|
|
|
|
;; And with render, they would be rendered repeatedly.
|
|
|
|
|
;; Using reset-modification-dates is sort of like session control.
|
|
|
|
|
(reset-mod-date-hash!)
|
|
|
|
|
(for-each render-from-source-or-output-path (if wants-parallel-render
|
|
|
|
|
;; returns crashed jobs for serial rendering
|
|
|
|
|
(parallel-render paths-in wants-parallel-render)
|
|
|
|
|
paths-in)))
|
|
|
|
|
|
|
|
|
|
(define (pagetree->paths pagetree-or-path)
|
|
|
|
|
(define pagetree (if (pagetree? pagetree-or-path)
|
|
|
|
|