docs in progress

render.rkt

@@ -41,9 +41,9 @@
 
 (define/contract+provide (render-batch . xs)
   (() #:rest (listof pathish?) . ->* . void?)
-  ;; Why not pass #:force #t through with render?
+  ;; Why not just (map render ...)?
   ;; Because certain files will pass through multiple times (e.g., templates)
-  ;; And with #:force, they would be rendered repeatedly.
+  ;; And with render, they would be rendered repeatedly.
   ;; Using reset-modification-dates is sort of like session control.
   (reset-modification-dates) 
   (for-each render-to-file-if-needed xs))
@@ -70,8 +70,9 @@
 
 
 (define (project-requires-changed?)
-  (define rerequire-results (map file-needed-rerequire? (get-project-require-files)))
+  (define project-require-files (get-project-require-files))
-  (define requires-changed? (ormap (λ(x) x) rerequire-results))
+  (define rerequire-results (and project-require-files (map file-needed-rerequire? project-require-files)))
+  (define requires-changed? (and rerequire-results (ormap (λ(x) x) rerequire-results)))
   (when requires-changed?
     (begin
       (message "render: project requires have changed, resetting cache & file-modification table")
@@ -123,21 +124,16 @@
   (file->bytes source-path))
 
 
-(define (render-through-scribble-eval expr-to-eval)
-  (parameterize ([current-namespace (make-base-namespace)]
-                 [current-output-port (current-error-port)])
-    (eval expr-to-eval (current-namespace))))
-
-
 (define/contract (render-scribble-source source-path)
   (complete-path? . -> . bytes?)
   (match-define-values (source-dir _ _) (split-path source-path))
+  (file-needed-rerequire? source-path) ; called for its reqrequire side effect only, so dynamic-require below isn't cached
   (time (parameterize ([current-directory (->complete-path source-dir)])
-          (render-through-scribble-eval `(begin 
+          ;; BTW this next action has side effects: scribble will copy in its core files if they don't exist.
-                                           (require scribble/render)
+          ((dynamic-require 'scribble/render 'render) (list (dynamic-require source-path world:main-pollen-export)) (list source-path))))
-                                           (require (file ,(->string source-path)))
+  (define result (file->bytes (->output-path source-path)))
-                                           (render (list doc) '(,source-path))))))
+  (delete-file (->output-path source-path)) ; because render promises the data, not the side effect
-  (file->bytes (->output-path source-path)))
+  result)
 
 
 (define/contract (render-preproc-source source-path)
@@ -152,8 +148,6 @@
   (match-define-values (source-dir _ _) (split-path source-path))
   (define template-path (or maybe-template-path (get-template-for source-path)))
   (render-for-dev-server template-path) ; because template might have its own preprocessor source
-  
-  
   (define expr-to-eval 
     `(begin 
        (require (for-syntax racket/base))
@@ -174,7 +168,7 @@
   (and (markup-source? path)))
 
 
-(define/contract (get-template-for source-path)
+(define/contract+provide (get-template-for source-path)
   (complete-path? . -> . (or/c #f complete-path?))
   (match-define-values (source-dir _ _) (split-path source-path))
   (and (templated-source? source-path) ; doesn't make sense if it's not a templated source format

scribblings/old-pollen.scrbl

@@ -0,0 +1,169 @@
+#lang scribble/manual
+ 
+@title{Pollen}
+
+@author{Matthew Butterick}
+
+Pollen is a publishing system that lets authors create beautiful and functional web-based books. Pollen brings together tools for writing, designing, programming, and testing. I used Pollen to make my book @link["http://practicaltypography.com"]{Butterick's Practical Typography}. You can use it to make your next web-based book — it's easy, powerful, and fun.
+
+Skip to: Why Pollen?
+Skip to: Why Racket?
+Skip to: Quick tutorial
+
+@section{Setting up Pollen}
+
+Download Racket & install.
+
+@code{raco pkg install pollen}
+
+Open terminal and cd to your project directory.
+
+Run @code{racket}.
+
+At the prompt, type @code{(require pollen/setup)}.
+
+Then type @code{(setup)}. 
+
+Quit Racket with ctrl+D.
+
+At the terminal prompt, @code{./polcom start}.
+
+
+
+@section{Quick start}
+
+
+
+@subsection{You had me at Hello World}
+
+Launch DrRacket. Open a new document.
+
+Change the top line to #lang pollen. This will be the first line of any source file where you want to invoke the Pollen language.
+
+Type @code{Hello World} underneath.
+
+Run the file by clicking the Run button, or typing cmd-R.
+
+The result window will show @code{Hello World}.
+
+Congratulations, you've made your first Pollen document.
+
+If you like, change the text and run the file again. The new text will appear in the result window.
+
+Pollen is a programming language that operates in text mode by default. Meaning, all plain text in the source file is considered valid input, and gets passed through intact.
+
+@subsection{The plot thickens}
+
+Start a new Pollen document. Remember to change the top line.
+
+Underneath, type @code{Hello ◊(+ 1 2) Worlds}. The character before the left parenthesis is called a lozenge. Type it by [doing such and such].
+
+Ask yourself: what are you likely to get when you run the file?
+
+OK, now run the file.
+
+The result will be @code{Hello 3 Worlds}. Hopefully, that's what you expected. 
+
+Feel free to change the numbers inside the parenthesized expression and run the file again. The printed sum will change. You can also change the @code{+} sign to a @code{*} sign and make really big numbers. If you want to see your first stupid Pollen trick, type @code{Hello ◊(/ 38 57) of a World} and watch what happens.
+
+Erase everything but the top line.
+
+Type this: @code{
+◊(define name "Roxy")
+
+Hello ◊name}. 
+
+What do you suppose you'll get this time?
+
+Run the file. You'll see @code{Hello Roxy}.
+
+The lozenge character (◊) tells Pollen to interpret what follows as code rather than plain text. This character is therefore the gateway to all the programming functions available in Pollen. In the first case, it denoted a math expression. In the second case, it denoted the definition of a variable, and then the variable itself.
+
+
+@subsection{Making an HTML page with Pollen}
+
+By default, Pollen operates in preprocessor mode. That means it evaluates all the expressions in your document, renders each as text, and then outputs the whole document as a text file. 
+
+In this tutorial, you're going to make an HTML file. But you can use Pollen as a preprocessor for any kind of text file.
+
+@margin-note{That means Pollen can act as a preprocessor for CSS, JavaScript, XML — and even source files for other programming languages.}
+
+
+@subsection{Making an HTML page with Pollen in decoder mode}
+
+@subsection{Using the Pollen dashboard}
+
+
+
+
+@section{Why I made Pollen}
+
+The nerds have already raced ahead to the quick tutorial. That's okay. Because software isn't just data structures and functions. It's ideas, and choices, and policies. It's design. 
+
+I created Pollen to overcome certain tool limitations that surfaced repeatedly in my work. If you agree with my characterization of those problems, then you'll probably like the solution that Pollen offers. 
+
+If not, you probably won't.
+
+@subsection{The web-development problem}
+
+
+I made my first web page in 1994, shortly after the web was invented. I opened my text editor (at the time, @link["http://www.barebones.com/products/bbedit/"]{BBEdit}) and pecked out @code{<html><body>Hello world</body></html>}, then loaded it in @link["http://en.wikipedia.org/wiki/Mosaic_(web_browser)"]{Mosaic}. So did a million others.
+
+If you weren't around then, you didn't miss much. Everything about the web was horrible: the web browsers, the computers running the browsers, the dial-up connections feeding the browsers, and of course HTML itself. At that point, the desktop-software experience was already slick and refined. By comparison, using the web felt like banging rocks together.
+
+That's no longer true. The web is now 20 years old. During that time, most parts of the web have improved dramatically — the connections are faster, the browsers are more sophisticated, the screens have more pixels. 
+
+But one part has not: the way we make web pages. Over the years, tools promising to simplify HTML development have come and mostly gone — from PageMill to Dreamweaver. Meanwhile, true web jocks have remained loyal to the original HTML power tool: the humble text editor.
+
+In one way, this makes sense. Web pages are mostly made of text — HTML, CSS, JavaScript, and so on — and thus the simplest way to mainpulate them is with a text editor. While HTML and CSS are not programming languages, they lend themselves to semantic and logical structure that's most easily expressed by editing them as text. Text-based editing also makes debugging and performance improvements easier.
+
+But text-based editing is also limited. Though the underlying description of a web page is notionally human-readable, it's largely optimized to be readable by other software (namely, web browsers). HTML markup in particular is verbose and easily mistyped. And isn't it fatally dull to manage all the boilerplate, like surrounding every paragraph with @code{<p>...</p>}? Yes, it is.
+
+For these reasons, much of web development should lend itself to automation. But in practice, tools that enable this automation have been slow to arrive, and most come hobbled with unacceptable deficiencies.
+
+@subsubsection{Why not a content management system, like WordPress?}
+
+I used WordPress to make the original version of @link["http://typographyforlawyers.com/"]{Typography for Lawyers} (the precursor to Butterick's Practical Typography). Even WordPress founder Matt Mullenweg @link["http://ma.tt/2010/04/typography-for-lawyers/"]{thought} it was “a cool use of WordPress for a mini-book.” Thanks, Matt. At the time, WordPress was the best tool for the job.
+
+But at the risk of having my @link["http://gravatar.com"]{Gravatar} revoked, I'll tell you I became disenchanted with WordPress because:
+
+It's a resource hog.
+
+Performance is questionable.
+
+There's always a new security problem.
+
+No source control.
+
+PHP.
+
+@subsubsection{Why not a CSS preprocessor, like Sass or LESS?}
+
+A CSS preprocessor automates the generation of CSS data. These preprocessors do save time & effort, so using one is better than not using one. My objection is that they ask you to incur much of the overhead of learning a programming language but without delivering the benefits. Because unlike a general-purpose programming language, Sass and LESS can only manipulate CSS. Better to learn a programming language that can manipulate anything.
+
+@subsubsection{Why not a static blog generator, like Jekyll or Pelican?}
+
+
+
+@subsubsection{Why not a dynamic templating system, like Bottle?}
+
+
+
+
+
+
+@subsection{The book-publishing problem}
+
+
+@section{Why I used Racket}
+
+
+
+@subsection{Writing & editing}
+
+@subsection{Design & layout}
+
+@subsection{Programming}
+
+
+@section{License & credits}

scribblings/pollen.scrbl

@@ -1,169 +1,71 @@
 #lang scribble/manual
 
-@title{Pollen}
+@(require scribble/eval pollen/world (for-label racket (except-in pollen #%module-begin) pollen/world))
 
-@author{Matthew Butterick}
+@(define my-eval (make-base-eval))
+@(my-eval `(require pollen))
 
-Pollen is a publishing system that lets authors create beautiful and functional web-based books. Pollen brings together tools for writing, designing, programming, and testing. I used Pollen to make my book @link["http://practicaltypography.com"]{Butterick's Practical Typography}. You can use it to make your next web-based book — it's easy, powerful, and fun.
 
-Skip to: Why Pollen?
+@title{Pollen: the book is a program}
-Skip to: Why Racket?
-Skip to: Quick tutorial
 
-@section{Setting up Pollen}
+@author[(author+email "Matthew Butterick" "mb@mbtype.com")]
 
-Download Racket & install.
+Pollen is a publishing system that helps authors create beautiful and functional web-based books. Pollen is a package of tools for writing, designing, programming, testing, and publishing.
 
-@code{raco pkg install pollen}
+I used Pollen to make my book @link["http://practicaltypography.com"]{Butterick's Practical Typography}. If that looks better than the last digital book you encountered, read on.
 
-Open terminal and cd to your project directory.
+Pollen is built around two ideas. First, that digital books should be the best books we've ever had. (So far, they're not even close.) Second, that if digital books are software, authors shouldn't think of a book as merely data. The book is a program.
 
-Run @code{racket}.
+Not that you need to be a programmer to use Pollen. On the contrary, the Pollen language is markup-based, so you can write & edit text naturally. But when you want to automate repetitive tasks or add other features, you can access a full programming language from within the text.
 
-At the prompt, type @code{(require pollen/setup)}.
+That language is Racket. I chose Racket because while the idea for Pollen had been with me for several years, it simply wasn't possible to build it with other languages. So if it's unfamiliar to you, don't panic. It was unfamiliar to me. Once you see what you can do with Pollen & Racket, you may be persuaded. I was.
 
-Then type @code{(setup)}. 
+Or, if you can find a better digital book-publishing tool, use that. Personally, I'm never going back to the way I used to make web pages. After 20 years of web publishing, this is the best tool I've ever had.
 
-Quit Racket with ctrl+D.
+@section{Installation}
 
-At the terminal prompt, @code{./polcom start}.
+Install Racket, which includes DrRacket.
 
+Install Pollen from the command line:
+@verbatim{raco pkg install pollen}
 
+After that, you can update the package from the command line:
+@verbatim{raco pkg update pollen}
 
-@section{Quick start}
 
 
+@section{Pollen source formats}
 
-@subsection{You had me at Hello World}
+@defmodulelang[pollen]
 
-Launch DrRacket. Open a new document.
+This puts Pollen into automatic mode, where the source file is interpreted according to the file extension. 
 
-Change the top line to #lang pollen. This will be the first line of any source file where you want to invoke the Pollen language.
+If the file extension is ``@(format ".~a" world:markup-source-ext)'', the source is interpreted as @racket[pollen/markup].
 
-Type @code{Hello World} underneath.
+If the file extension is ``@(format ".~a" world:preproc-source-ext)'', the source is interpreted as @racket[pollen/pre] (``pre'' stands for ``preprocessor'').
 
-Run the file by clicking the Run button, or typing cmd-R.
+If the file extension is ``@(format ".~a" world:markdown-source-ext)'', the source is interpreted as @racket[pollen/markdown].
 
-The result window will show @code{Hello World}.
+@defmodulelang[pollen/markup]
 
-Congratulations, you've made your first Pollen document.
 
-If you like, change the text and run the file again. The new text will appear in the result window.
+@defmodulelang[pollen/pre]
 
-Pollen is a programming language that operates in text mode by default. Meaning, all plain text in the source file is considered valid input, and gets passed through intact.
+@defmodulelang[pollen/markdown]
 
-@subsection{The plot thickens}
 
-Start a new Pollen document. Remember to change the top line.
 
-Underneath, type @code{Hello ◊(+ 1 2) Worlds}. The character before the left parenthesis is called a lozenge. Type it by [doing such and such].
 
-Ask yourself: what are you likely to get when you run the file?
 
-OK, now run the file.
 
-The result will be @code{Hello 3 Worlds}. Hopefully, that's what you expected. 
+@section{Rendering}
 
-Feel free to change the numbers inside the parenthesized expression and run the file again. The printed sum will change. You can also change the @code{+} sign to a @code{*} sign and make really big numbers. If you want to see your first stupid Pollen trick, type @code{Hello ◊(/ 38 57) of a World} and watch what happens.
+@defmodule[pollen/render]
 
-Erase everything but the top line.
 
-Type this: @code{
+@section{License & source code}
-◊(define name "Roxy")
 
-Hello ◊name}. 
+This module is licensed under the LGPL.
 
-What do you suppose you'll get this time?
+Source repository at @link["http://github.com/mbutterick/pollen"]{http://github.com/mbutterick/pollen}. Suggestions & corrections welcome.
 
-Run the file. You'll see @code{Hello Roxy}.
-
-The lozenge character (◊) tells Pollen to interpret what follows as code rather than plain text. This character is therefore the gateway to all the programming functions available in Pollen. In the first case, it denoted a math expression. In the second case, it denoted the definition of a variable, and then the variable itself.
-
-
-@subsection{Making an HTML page with Pollen}
-
-By default, Pollen operates in preprocessor mode. That means it evaluates all the expressions in your document, renders each as text, and then outputs the whole document as a text file. 
-
-In this tutorial, you're going to make an HTML file. But you can use Pollen as a preprocessor for any kind of text file.
-
-@margin-note{That means Pollen can act as a preprocessor for CSS, JavaScript, XML — and even source files for other programming languages.}
-
-
-@subsection{Making an HTML page with Pollen in decoder mode}
-
-@subsection{Using the Pollen dashboard}
-
-
-
-
-@section{Why I made Pollen}
-
-The nerds have already raced ahead to the quick tutorial. That's okay. Because software isn't just data structures and functions. It's ideas, and choices, and policies. It's design. 
-
-I created Pollen to overcome certain tool limitations that surfaced repeatedly in my work. If you agree with my characterization of those problems, then you'll probably like the solution that Pollen offers. 
-
-If not, you probably won't.
-
-@subsection{The web-development problem}
-
-
-I made my first web page in 1994, shortly after the web was invented. I opened my text editor (at the time, @link["http://www.barebones.com/products/bbedit/"]{BBEdit}) and pecked out @code{<html><body>Hello world</body></html>}, then loaded it in @link["http://en.wikipedia.org/wiki/Mosaic_(web_browser)"]{Mosaic}. So did a million others.
-
-If you weren't around then, you didn't miss much. Everything about the web was horrible: the web browsers, the computers running the browsers, the dial-up connections feeding the browsers, and of course HTML itself. At that point, the desktop-software experience was already slick and refined. By comparison, using the web felt like banging rocks together.
-
-That's no longer true. The web is now 20 years old. During that time, most parts of the web have improved dramatically — the connections are faster, the browsers are more sophisticated, the screens have more pixels. 
-
-But one part has not: the way we make web pages. Over the years, tools promising to simplify HTML development have come and mostly gone — from PageMill to Dreamweaver. Meanwhile, true web jocks have remained loyal to the original HTML power tool: the humble text editor.
-
-In one way, this makes sense. Web pages are mostly made of text — HTML, CSS, JavaScript, and so on — and thus the simplest way to mainpulate them is with a text editor. While HTML and CSS are not programming languages, they lend themselves to semantic and logical structure that's most easily expressed by editing them as text. Text-based editing also makes debugging and performance improvements easier.
-
-But text-based editing is also limited. Though the underlying description of a web page is notionally human-readable, it's largely optimized to be readable by other software (namely, web browsers). HTML markup in particular is verbose and easily mistyped. And isn't it fatally dull to manage all the boilerplate, like surrounding every paragraph with @code{<p>...</p>}? Yes, it is.
-
-For these reasons, much of web development should lend itself to automation. But in practice, tools that enable this automation have been slow to arrive, and most come hobbled with unacceptable deficiencies.
-
-@subsubsection{Why not a content management system, like WordPress?}
-
-I used WordPress to make the original version of @link["http://typographyforlawyers.com/"]{Typography for Lawyers} (the precursor to Butterick's Practical Typography). Even WordPress founder Matt Mullenweg @link["http://ma.tt/2010/04/typography-for-lawyers/"]{thought} it was “a cool use of WordPress for a mini-book.” Thanks, Matt. At the time, WordPress was the best tool for the job.
-
-But at the risk of having my @link["http://gravatar.com"]{Gravatar} revoked, I'll tell you I became disenchanted with WordPress because:
-
-It's a resource hog.
-
-Performance is questionable.
-
-There's always a new security problem.
-
-No source control.
-
-PHP.
-
-@subsubsection{Why not a CSS preprocessor, like Sass or LESS?}
-
-A CSS preprocessor automates the generation of CSS data. These preprocessors do save time & effort, so using one is better than not using one. My objection is that they ask you to incur much of the overhead of learning a programming language but without delivering the benefits. Because unlike a general-purpose programming language, Sass and LESS can only manipulate CSS. Better to learn a programming language that can manipulate anything.
-
-@subsubsection{Why not a static blog generator, like Jekyll or Pelican?}
-
-
-
-@subsubsection{Why not a dynamic templating system, like Bottle?}
-
-
-
-
-
-
-@subsection{The book-publishing problem}
-
-
-@section{Why I used Racket}
-
-
-
-@subsection{Writing & editing}
-
-@subsection{Design & layout}
-
-@subsection{Programming}
-
-
-@section{License & credits}

scribblings/render.scrbl

@@ -0,0 +1,72 @@
+#lang scribble/manual
+
+@(require scribble/eval pollen/render pollen/world (for-label racket (except-in pollen #%module-begin) pollen/world pollen/render web-server/templates pollen/file-tools))
+
+@(define my-eval (make-base-eval))
+@(my-eval `(require pollen))
+
+@section{Rendering}
+
+@defmodule[pollen/render]
+
+@defproc[
+(render
+[source-path complete-path?]
+[template-path (or/c #f complete-path?) #f]) 
+bytes?]
+Renders the source file at @racket[_source-path]. The rendering behavior depends on the type of source file:
+
+A @racketmodname[pollen/pre] file is rendered without a template.
+
+A @racketmodname[pollen/markup] or @racketmodname[pollen/markdown] file is rendered with a template. If no template is provided with @racket[_template-path], Pollen tries to find one using @racket[get-template-for]. If that doesn't work, Pollen uses the fallback template. 
+
+Be aware that rendering with a template uses @racket[include-template] within @racket[eval]. This is necessary to allow dynamic refresh of pages. For complex pages, it can be slow the first time. Caching is used to make subsequent requests faster.
+
+For those panicked at the use of @racket[eval], please don't be. As the author of @racket[include-template] has already advised, ``If you insist on dynamicism'' — and yes, I do insist — ``there is always @racket[eval].''
+@secref["How do I use templates “dynamically\"?" #:doc '(lib "web-server/scribblings/faq.scrbl")]
+
+@defproc[
+(render-to-file
+[source-path complete-path?]
+[template-path (or/c #f complete-path?) #f]
+[output-path (or/c #f complete-path?) #f]) 
+void?]
+Like @racket[render], but saves the file to @racket[_output-path], overwriting whatever is already there. If no @racket[_output-path] is provided, it's derived from @racket[_source-path] using @racket[->output-path].
+
+@defproc[
+(render-to-file-if-needed
+[source-or-output-path complete-path?]
+[template-path (or/c #f complete-path?) #f]
+[output-path (or/c #f complete-path?) #f]) 
+void?]
+Like @racket[render-to-file], with two differences. First, the mandatory argument can be either a source or an output path, and the function will compute the other. Second, the render only happens if one of these conditions exist:
+@itemlist[#:style 'ordered
+@item{No file exists at @racket[_output-path]. (Corollary: an easy way to force a refresh of a particular @racket[_output-path] is to delete it.)}
+
+@item{Either @racket[_source-path] or @racket[_template-path] have changed since the last trip through @racket[render].}
+
+@item{One or more of the project requires have changed.}]
+
+If none of these conditions exist, @racket[_output-path] is assumed to be current, and the render is skipped.
+
+
+
+
+@defproc[
+(render-batch
+[source-paths (listof pathish?)] ...) 
+void?]
+Render multiple @racket[_source-paths] in one go. This can be faster than @racket[(map render _source-paths)] if your @racket[_source-paths] rely on a common set of templates. Templates may have their own Pollen source files that need to be compiled. If you use @racket[render], the templates will be repeatedly (and needlessly) re-compiled. Whereas if you use @racket[render-batch], each template will only be compiled once.
+
+
+@defproc[
+(get-template-for
+[source-path complete-path?])
+(or/c #f complete-path?)]
+Find a template file for @racket[_source-path], with the following priority:
+
+If the metas for @racket[_source-path] have a key for @code[(format "'~a" world:template-meta-key)], then use the value of this key. 
+
+If this key doesn't exist, or if it points to a nonexistent file, look for a default template in the project directory with the name @code[(format "~a.[output extension].~a" world:default-template-prefix world:template-source-ext)]. Meaning, if @racket[_source-path] is @code[(format "intro.html.~a" world:markup-source-ext)], the default template would be @code[(format "~a.html.~a" world:default-template-prefix world:template-source-ext)].
+
+If this file doesn't exist, use the fallback template.