More robust interface to Pygments (#230)

dev-output-files-201008
Shrutarshi Basu 4 years ago committed by GitHub
parent 88c354cb5f
commit 123547f3cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1,8 +1,22 @@
# This allows us to launch Python and pygments once, and pipe to it # This allows us to launch Python and pygments once, and pipe to it
# continuously. Input format is: # continuously.
# #
# <lexer-name> # There are four options:
# <list-of-highlighted-lines> # 1. Language (used to determine the lexer)
# 2. Line numbers to highlight
# 3. Encoding to use for the output
# 4. HTML Class to use for the output
#
# These can be specified as arguments when this script is first invoked, or on a
# per invocation.
#
# To set the options for each invocation, the format is:
#
# __LANG__ <lexer-name>
# __LINENOS__ <true|false>
# __LINES__ <list-of-highlighted-lines>
# __CSS__ <CSS-class>
# __ENC__ <encoding>
# <code> # <code>
# ... # ...
# __END__ # __END__
@ -17,57 +31,120 @@
# ... # ...
# __END__ # __END__
from __future__ import print_function
import sys import sys
import optparse import optparse
from pygments import highlight from pygments import highlight
from pygments.lexers import get_lexer_by_name from pygments.lexers import get_lexer_by_name
from pygments.util import ClassNotFound from pygments.util import ClassNotFound
from pygments.formatters import HtmlFormatter from pygments.formatters import HtmlFormatter
def get_lexer(lang):
if not lang:
get_lexer_by_name("text", encoding="guess")
try:
return get_lexer_by_name(lang, encoding="guess")
except ClassNotFound:
print("No lexer was found for the given language. Using plain text instead.", file=sys.stderr)
return get_lexer_by_name("text", encoding="guess")
parser = optparse.OptionParser() parser = optparse.OptionParser()
parser.add_option('--linenos', action="store_true", dest="linenos") parser.add_option('--linenos', action="store_true", dest="linenos")
parser.add_option('--cssclass', default="source", dest="cssclass") parser.add_option('--cssclass', default="source", dest="cssclass")
parser.add_option('--encoding', default="utf-8", dest="encoding")
parser.add_option('--language', dest="language")
(options, _) = parser.parse_args() (options, _) = parser.parse_args()
lexer = "" # Set initial options
config = {
'linenos': options.linenos,
'cssclass': options.cssclass,
'encoding': options.encoding,
'hl_lines': []
}
lexer = get_lexer(options.language)
code = "" code = ""
lines_to_highlight = ""
py_version = sys.version_info.major py_version = sys.version_info.major
sys.stdout.write("ready\n") sys.stdout.write("ready\n")
sys.stdout.flush() sys.stdout.flush()
while 1: while 1:
line_raw = sys.stdin.readline() line_raw = sys.stdin.readline()
if not line_raw: if not line_raw:
break break
# Without trailing space, \n, or \n # Without trailing space, \n, or \n
line = line_raw.rstrip() line = line_raw.rstrip()
if line == '__EXIT__': if line == '__EXIT__':
break break
elif line == '__END__': elif line == '__END__':
# Lex input finished. Lex it. # Lex input finished. Lex it.
formatter = HtmlFormatter(linenos=options.linenos, formatter = HtmlFormatter(linenos=config['linenos'],
cssclass=options.cssclass, cssclass=config['cssclass'],
encoding="utf-8", encoding=config['encoding'],
hl_lines=lines_to_highlight) hl_lines=config['hl_lines'])
if py_version >= 3: if py_version >= 3:
sys.stdout.write(highlight(code, lexer, formatter).decode("utf-8")) sys.stdout.write(highlight(code, lexer, formatter).decode("utf-8"))
else: else:
sys.stdout.write(highlight(code, lexer, formatter)) sys.stdout.write(highlight(code, lexer, formatter))
sys.stdout.write('\n__END__\n') sys.stdout.write('\n__END__\n')
sys.stdout.flush() sys.stdout.flush()
lexer = ""
# Reset the configuration for the next invocation. Most options are
# actually persisted between runs, except for the code itself and the
# lines to be highlighted.
code = "" code = ""
lines_to_highlight = "" config['hl_lines'] = []
elif lexer == "":
# Starting another lex. First line is the lexer name. elif code == "":
try: # Only check for new options at the beginning of a a fresh invocation
lexer = get_lexer_by_name(line, encoding="guess") if line.startswith("__LANG__"):
except ClassNotFound: # Use the provided language to find the appropriate lexer.
lexer = get_lexer_by_name("text", encoding="guess") try:
elif lines_to_highlight == "": lang = line.split()[1]
# Starting another lex. Second line is list of lines to highlight, lexer = get_lexer(lang)
# formatted as string of whitespace-separated integers except IndexError:
lines_to_highlight = [int(str) for str in line.split()] print("No lexer was found for the given language. Using plain text instead.", file=sys.stderr)
lexer = get_lexer_by_name("text", encoding="guess")
elif line.startswith("__LINENOS__"):
try:
option = line.split()[1]
if option.lower() == "true":
config['linenos'] = True
elif option.lower() == "false":
config['linenos'] = False
else:
pass
except IndexError:
print("__LINENOS__ option must be given a `true` or `false` value",
file=sys.stderr)
elif line.startswith("__LINES__"):
# The list of lines to highlight is formatted as string of
# whitespace-separated integers
lines = line.split()[1:]
config['hl_lines'] = [int(str) for str in lines]
elif line.startswith("__CSS__"):
try:
config['cssclass'] = line.split[1]
except IndexError:
print("Could not parse CSS class line.", file=sys.stderr)
elif line.startswith("__ENC__"):
try:
config['encoding'] = line.split[1]
except IndexError:
print("Could not parse encoding line.", file=sys.stderr)
else:
# Done with configuration for this invocation, start accumulating
# code. Use `line_raw` because we want trailing space, \n, \r
code += line_raw
else: else:
# Accumulate more code # Accumulate more code
# Use `line_raw`: Do want trailing space, \n, \r # Use `line_raw`: Do want trailing space, \n, \r

@ -130,11 +130,18 @@ if zero is False:
(unless (running?) (unless (running?)
(start python-executable line-numbers? css-class)) (start python-executable line-numbers? css-class))
(cond [(running?) (cond [(running?)
;; order of writing arguments is significant: cooperates with pipe.py ;; This works with a simple wrapper around Pygments defined in pipe.py.
(displayln lang pyg-out) ;; First send over the configuration options, then code to highlight.
(displayln (string-join (map number->string hl-lines) " ") pyg-out) ;; pipe.py also supports an encoding option, but seems unnecessary to
;; use at this point.
(fprintf pyg-out "__LANG__ ~a~n" lang)
(fprintf pyg-out "__LINES__ ~a~n" (string-join (map number->string hl-lines) " "))
(fprintf pyg-out "__LINENOS__ ~a~n"
(if line-numbers? "true" "false"))
(displayln code pyg-out) (displayln code pyg-out)
(displayln "__END__" pyg-out) (displayln "__END__" pyg-out)
;; Read back the highlighted code
(let loop ([s ""]) (let loop ([s ""])
(match (read-line pyg-in 'any) (match (read-line pyg-in 'any)
["__END__" (with-input-from-string s read-html-as-xexprs)] ["__END__" (with-input-from-string s read-html-as-xexprs)]

Loading…
Cancel
Save