Add syntax highlighting to all <pre> contents

This change adds https://github.com/tabatkins/highlighter as a submodule
and updates the build script to use the highlight server when running
wattsi, as documented at
tabatkins/highlighter#6 (comment)

This change reverts the revert made in ffe6f19. It fixes the problem
which made that revert necessary; the fix is to make the PYTHONPATH in
the local environment include the path to the Pygments copy in the
highlighter submodule.

Fixes #169.

Relies on whatwg/wattsi#63.
Addresses #113.
Relates to whatwg/html#3768
and whatwg/whatwg.org#215.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "highlighter"]
+	path = highlighter
+	url = git://github.com/tabatkins/highlighter.git

Dockerfile

@@ -3,7 +3,7 @@ FROM debian:sid
 ## dependency installation: nginx, wattsi, and other build tools
 ## cleanup freepascal since it is no longer needed after wattsi build
 RUN apt-get update && \
-    apt-get install -y ca-certificates curl git unzip fp-compiler fp-units-fcl fp-units-net libc6-dev nginx && \
+    apt-get install -y ca-certificates curl git unzip fp-compiler fp-units-fcl fp-units-net libc6-dev nginx python2.7 python-pip && \
     git clone https://github.com/whatwg/wattsi.git /whatwg/wattsi && \
     cd /whatwg/wattsi && \
     /whatwg/wattsi/build.sh && \

README.md

@@ -22,7 +22,7 @@ To build locally, you'll need the following commands installed on your system:
 
 - `curl`, `grep`, `perl`, `unzip`
 
-Optionally, for faster builds, you can install [Wattsi](https://github.com/whatwg/wattsi). If you don't bother with that, the build will use [Wattsi Server](https://github.com/domenic/wattsi-server), which requires an internet connection.
+Optionally, for faster builds, you can install [Wattsi](https://github.com/whatwg/wattsi) and Python 2.7 (necessary for applying syntax highlighting to `pre` contents). If you don't bother with that, the build will use [Wattsi Server](https://github.com/domenic/wattsi-server), which requires an internet connection.
 
 ### Running the build
 

build.sh

@@ -26,6 +26,7 @@ HTML_OUTPUT=${HTML_OUTPUT:-$DIR/output}
 export HTML_OUTPUT
 
 SHA_OVERRIDE=${SHA_OVERRIDE:-}
+HIGHLIGHT_SERVER_URL="http://127.0.0.1:8080"
 
 for arg in "$@"
 do
@@ -366,8 +367,10 @@ function processSource {
   perl .pre-process-tag-omission.pl < "$HTML_TEMP/source-expanded-2" | perl .pre-process-index-generator.pl > "$HTML_TEMP/source-whatwg-complete" # this one could be merged
 
   function runWattsi {
-    # Input arguments: $1 is the file to run wattsi on, $2 is a directory for wattsi to write output
-    # to
+    # Input arguments:
+    # - $1 is the file to run Wattsi on
+    # - $2 is a directory for Wattsi to write output to
+    # - $3 is the URL for the syntax-highlighter server
     # Output:
     # - Sets global variable $WATTSI_RESULT to an exit code (or equivalent, for HTTP version)
     # - $HTML_TEMP/wattsi-output directory will contain the output from wattsi on success
@@ -380,9 +383,9 @@ function processSource {
     if $QUIET; then
       WATTSI_ARGS+=( --quiet )
     fi
-    WATTSI_ARGS+=( "$1" "$HTML_SHA" "$2" "$BUILD_TYPE" "$HTML_CACHE/caniuse.json" "$HTML_CACHE/w3cbugs.csv")
+    WATTSI_ARGS+=( "$1" "$HTML_SHA" "$2" "$BUILD_TYPE" "$HTML_CACHE/caniuse.json" "$HTML_CACHE/w3cbugs.csv" "$HIGHLIGHT_SERVER_URL" )
     if hash wattsi 2>/dev/null; then
-      if [ "$(wattsi --version | cut -d' ' -f2)" -lt "$WATTSI_LATEST" ]; then
+      if [[ "$(wattsi --version | cut -d' ' -f2)" -lt "$WATTSI_LATEST" ]]; then
         echo
         echo "Warning: Your wattsi version is out of date. You should to rebuild an"
         echo "up-to-date wattsi binary from the wattsi sources."
@@ -434,7 +437,14 @@ function processSource {
     fi
   }
 
-  runWattsi "$HTML_TEMP/source-whatwg-complete" "$HTML_TEMP/wattsi-output"
+  # Setting PYTHONPATH is a workaround for https://github.com/whatwg/html-build/issues/169.
+  # See also https://github.com/tabatkins/highlighter/issues/5 and
+  # https://bitbucket.org/birkenfeld/pygments-main/issues/1448.
+  export PYTHONPATH="$DIR/highlighter/highlighter/pygments:$PYTHONPATH"
+  "$DIR/highlighter/server.py" &
+  HIGHLIGHT_SERVER_PID=$!
+
+  runWattsi "$HTML_TEMP/source-whatwg-complete" "$HTML_TEMP/wattsi-output" "$HIGHLIGHT_SERVER_URL"
   if [[ "$WATTSI_RESULT" == "0" ]]; then
     if [[ "$LOCAL_WATTSI" != true ]]; then
       "$QUIET" || grep -v '^$' "$HTML_TEMP/wattsi-output.txt" # trim blank lines
@@ -456,6 +466,9 @@ function processSource {
     echo "There were errors. Stopping."
     exit "$WATTSI_RESULT"
   fi
+  kill "$HIGHLIGHT_SERVER_PID"
+  # suppresses 'Terminated: 15 "$DIR/highlighter/server.py"' message
+  wait "$HIGHLIGHT_SERVER_PID" 2>/dev/null || # ignore non-zero exit code
 
   function generateBacklinks {
     perl .post-process-partial-backlink-generator.pl "$HTML_TEMP/wattsi-output/index-$1" > "$2/index.html";

ci-deploy/Dockerfile

@@ -7,13 +7,15 @@ RUN apt-get update && \
     fp-compiler fp-units-fcl fp-units-net libc6-dev         \
     default-jre                                             \
     libfontconfig1 libgomp1 libxml2                         \
+    python2.7 python-pip                                    \
     fonts-dejavu fonts-droid-fallback fonts-liberation fonts-symbola fonts-unfonts-core
 
 # Dependency lines above are:
 # - General
 # - Wattsi
 # - validator
 # - Prince
+# - Highlighter
 # - fonts
 
 # Dependecies of prince_11.3-1_debian8.0_amd64.deb (not used) are libc6 libcurl3 libfontconfig1

ci-deploy/outside-container.sh

@@ -16,6 +16,13 @@ TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false}
 # - DOCKER_PASSWORD is set from the outside
 # - ENCRYPTION_LABEL is set from the outside
 
+# Initialize the highlighter submodule for html-build
+(
+  cd html-build
+  git submodule init
+  git submodule update
+)
+
 git clone --depth 1 https://github.com/whatwg/wattsi.git wattsi
 
 git clone --depth 1 https://github.com/pts/pdfsizeopt.git pdfsizeopt