post-process/hyphenate html output

- add soft hyphens to text
- justify text on all browsers
- use htmld and hyphenate libs w/ en-US pattern
- run dpl-docs w/ hyphenation

Author: MartinNowak

css/style.css

@@ -520,17 +520,6 @@ body.std .subnav a.package
     text-align: justify;
 }
 
-/*  This is a CSS hack to only target Blink based browsers
-    Targets Chrome 28 and later and Opera 14 and later
-*/
-@media all and (-webkit-min-device-pixel-ratio:0) and (min-resolution: .001dpcm) {
-    #content
-    {
-        /* because Blink based browsers don't support hyphens */
-        text-align: left;
-    }
-}
-
 .subnav + #content
 {
     margin-left: 19em;

dlang.org.ddoc

@@ -41,13 +41,13 @@ _=
 
 CONSOLE=$(TC pre, console notranslate, $0)
 COPYRIGHT=Copyright © 1999-$(YEAR) by the $(LINK2 $(ROOT)/foundation.html, D Language Foundation)
-CPPCODE=$(TC pre, cppcode notranslate, $0)
+CPPCODE=$(TC pre, cppcode donthyphenate notranslate, $0)
 CPPLISTING=$(CPPCODE $0)
 CROSS=✘
 _=
 
 D=$(SPANC d_inlinecode donthyphenate notranslate, $0)
-D_CODE=$(TC pre, d_code notranslate, $0)
+D_CODE=$(TC pre, d_code donthyphenate notranslate, $0)
 _=
 
 DDOC=
@@ -127,7 +127,7 @@ DDOC_PSYMBOL = $(ADEF $0)$(SPANC ddoc_psymbol, $0)
 DDOC_ANCHOR = $(ADEF .$1)$(DIVCID quickindex, quickindex.$1, )
 DDOC_DECL  = $(TC dt, d_decl, $(DIV, $0))
 DDOC_UNDEFINED_MACRO = $(DDOC_COMMENT UNDEFINED MACRO: "$1")
-DDOCCODE=$(TC pre, ddoccode notranslate, $0)
+DDOCCODE=$(TC pre, ddoccode donthyphenate notranslate, $0)
 DDOCKEYVAL=$(DIVC keyval $1, $(SPANC key key$1, $1:) $(DIVC val val$1, $+))
 DDOCKEYVAL2=$(DIVC keyval $1, $(SPANC key key$1, $2:) $(DIVC val val$1, $(TAIL $+)))
 DDSUBLINK=$(LINK2 $(ROOT_DIR)$1.html#$2, $3)
@@ -161,7 +161,7 @@ GLINK2=$(DDSUBLINK spec/$1,$2,$(I $2))
 GLOSSARY = $(HTTP dlang.org/glossary.html#$0, $0)
 GLOSSARY2 = $(HTTP dlang.org/glossary.html#$1, $2)
 GNAME=<a id="$0">$(SPANC gname, $0)</a>
-GRAMMAR=$(TC pre, bnf notranslate, $0)
+GRAMMAR=$(TC pre, bnf donthyphenate notranslate, $0)
 GREEN=$(SPANC green, $0)
 GT=&gt;
 _=
@@ -199,7 +199,7 @@ MENU_W_SUBMENU_LINK = <li class='expand-container'><a class='expand-toggle' href
 MENU_W_SUBMENU_END = </li>
 META_KEYWORDS=D programming language
 META_DESCRIPTION=D Programming Language
-MODDEFFILE=$(TC pre, moddeffile notranslate, $0)
+MODDEFFILE=$(TC pre, moddeffile donthyphenate notranslate, $0)
 MULTICOL_CELL=<td colspan="$1">$+</td>
 MULTICOL_HEADER=<th colspan="$1">$+</th>
 MULTICOLS=$+
@@ -347,7 +347,7 @@ RELATIVE_LINK2=$(ALOCAL $1, $+)
 _=
 
 SAMPLESRC=$(SPANC sample_src, $(AHTTPS github.com/dlang/dmd/blob/master/samples/$0, /dmd/samples/d/$0))
-SCINI=$(TC pre, scini notranslate, $0)
+SCINI=$(TC pre, scini donthyphenate notranslate, $0)
 SCRIPTLOAD=<script type="text/javascript" src="$0"></script>
 SEARCHDEFAULT_PHOBOS=
 SEARCHDEFAULT_FORUM=

html-postprocessor/.gitignore

@@ -0,0 +1,6 @@
+html-postprocessor
+.dub
+docs.json
+__dummy.html
+*.o
+*.obj

html-postprocessor/dub.json

@@ -0,0 +1,8 @@
+{
+    "name": "html-postprocessor",
+    "authors": ["Martin Nowak"],
+    "dependencies": {
+	"hyphenate": "~>1.1.0",
+	"htmld": "~>0.2.12"
+    }
+}

html-postprocessor/dub.selections.json

@@ -0,0 +1,7 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"hyphenate": "1.1.1",
+		"htmld": "0.2.13"
+	}
+}

html-postprocessor/src/app.d

@@ -0,0 +1,69 @@
+import hyphenate;
+
+/// global immutable instance initialized for en-US
+static immutable Hyphenator h;
+shared static this()
+{
+    h = cast(immutable) Hyphenator(import("hyphen.tex"));
+}
+
+/// hyphenate a HTML file
+void hyphenateHTML(string path)
+{
+    import html : createDocument, Node, NodeWrapper;
+    import std.algorithm : canFind, splitter;
+    import std.array : appender, replace;
+    import std.file : readText;
+    import std.regex : ctRegex, replaceAllInto;
+
+    auto doc = createDocument(path.readText);
+
+    void visit(NodeWrapper!Node node, bool hyphenate)
+    {
+        enum wordsRE = ctRegex!("[\\w\­]+", "g");
+        if (node.isTextNode)
+        {
+            if (hyphenate)
+            {
+                auto app = appender!string;
+                replaceAllInto!((m, app) {
+                    auto word = m.hit.replace("\­", "");
+                    h.hyphenate(word, "\­", s => app.put(s));
+                })(app, node.text, wordsRE);
+                node.text = app.data;
+            }
+        }
+        else
+        {
+            if (node.tag == "script" || node.tag == "style")
+                return;
+
+            if (node.attr("class").splitter.canFind("hyphenate"))
+                hyphenate = true;
+            else if (node.attr("class").splitter.canFind("donthyphenate"))
+                hyphenate = false;
+
+            foreach (child; node.children)
+                visit(child, hyphenate);
+        }
+    }
+
+    visit(doc.root, false);
+
+    import std.stdio : File;
+
+    with (File(path, "w"))
+    {
+        auto orng = lockingTextWriter;
+        doc.root.innerHTML(orng);
+    }
+}
+
+void main(string[] args)
+{
+    import std.parallelism : parallel;
+    import std.file : readText;
+
+    foreach (path; args[1 .. $].parallel)
+        hyphenateHTML(path);
+}

posix.mak

@@ -21,6 +21,8 @@ DOC_OUTPUT_DIR:=$(shell pwd)/web
 GIT_HOME=https://github.com/dlang
 DPL_DOCS_PATH=dpl-docs
 DPL_DOCS=$(DPL_DOCS_PATH)/dpl-docs
+HTML_POSTPROCESSOR_PATH=html-postprocessor
+HTML_POSTPROCESSOR=$(HTML_POSTPROCESSOR_PATH)/html-postprocessor
 REMOTE_DIR=d-programming@digitalmars.com:data
 GENERATED=.generated
 
@@ -232,7 +234,7 @@ $(DOC_OUTPUT_DIR)/dmd-%.verbatim : %.ddoc dcompiler.dd verbatim.ddoc $(DMD)
 # Rulez
 ################################################################################
 
-all : docs html
+all : docs html postprocess-html
 
 docs : dmd-prerelease phobos-prerelease druntime-prerelease druntime-release \
   phobos-release apidocs-release apidocs-prerelease
@@ -245,6 +247,8 @@ kindle : ${DOC_OUTPUT_DIR}/dlangspec.mobi
 
 pdf : ${DOC_OUTPUT_DIR}/dlangspec.pdf
 
+postprocess-html : ${HTML_POSTPROCESSOR_PATH}/.done
+
 $(DOC_OUTPUT_DIR)/sitemap.html : $(ALL_FILES_BUT_SITEMAP) $(DMD)
 	cp -f sitemap-template.dd sitemap.dd
 	(true $(foreach F, $(TARGETS), \
@@ -274,7 +278,7 @@ rebase-druntime: ; cd $(DRUNTIME_DIR) && $(call REBASE,druntime)
 rebase-phobos: ; cd $(PHOBOS_DIR) && $(call REBASE,phobos)
 
 clean:
-	rm -rf $(DOC_OUTPUT_DIR) ${GENERATED} dpl-docs/.dub
+	rm -rf $(DOC_OUTPUT_DIR) ${GENERATED} ${DPL_DOCS_PATH}/.dub ${HTML_POSTPROCESSOR_PATH}/.done ${HTML_POSTPROCESSOR_PATH}/.dub
 	rm -rf auto dlangspec-consolidated.d $(addprefix dlangspec,.aux .d .dvi .fdb_latexmk .fls .log .out .pdf .tex .txt .verbatim.txt)
 	rm -f docs.json docs-prerelease.json dpl-docs/dpl-docs
 	@echo You should issue manually: rm -rf ${DMD_DIR}-${LATEST} ${DRUNTIME_DIR}-${LATEST} ${PHOBOS_DIR}-${LATEST} ${STABLE_DMD_ROOT} ${DUB_DIR}
@@ -427,19 +431,19 @@ apidocs-prerelease : ${DOC_OUTPUT_DIR}/library-prerelease/sitemap.xml ${DOC_OUTP
 apidocs-release : ${DOC_OUTPUT_DIR}/library/sitemap.xml ${DOC_OUTPUT_DIR}/library/.htaccess
 apidocs-serve : docs-prerelease.json
 	${DPL_DOCS} serve-html --std-macros=html.ddoc --std-macros=dlang.org.ddoc --std-macros=std.ddoc --std-macros=macros.ddoc --std-macros=std-ddox.ddoc \
-	  --override-macros=std-ddox-override.ddoc --package-order=std \
+	  --override-macros=std-ddox-override.ddoc --package-order=std --hyphenate \
 	  --git-target=master --web-file-dir=. docs-prerelease.json
 
 ${DOC_OUTPUT_DIR}/library-prerelease/sitemap.xml : docs-prerelease.json
 	@mkdir -p $(dir $@)
 	${DPL_DOCS} generate-html --file-name-style=lowerUnderscored --std-macros=html.ddoc --std-macros=dlang.org.ddoc --std-macros=std.ddoc --std-macros=macros.ddoc --std-macros=std-ddox.ddoc \
-	  --override-macros=std-ddox-override.ddoc --package-order=std \
+	  --override-macros=std-ddox-override.ddoc --package-order=std --hyphenate \
 	  --git-target=master docs-prerelease.json ${DOC_OUTPUT_DIR}/library-prerelease
 
 ${DOC_OUTPUT_DIR}/library/sitemap.xml : docs.json
 	@mkdir -p $(dir $@)
 	${DPL_DOCS} generate-html --file-name-style=lowerUnderscored --std-macros=html.ddoc --std-macros=dlang.org.ddoc --std-macros=std.ddoc --std-macros=macros.ddoc --std-macros=std-ddox.ddoc \
-	  --override-macros=std-ddox-override.ddoc --package-order=std \
+	  --override-macros=std-ddox-override.ddoc --package-order=std --hyphenate \
 	  --git-target=v${LATEST} docs.json ${DOC_OUTPUT_DIR}/library
 
 ${DOC_OUTPUT_DIR}/library/.htaccess : dpl_release_htaccess
@@ -506,6 +510,26 @@ ${DUB}: ${DUB_DIR} ${STABLE_DMD}
 chm-nav.json : $(DDOC) std.ddoc spec/spec.ddoc ${GENERATED}/modlist-${LATEST}.ddoc changelog/changelog.ddoc chm-nav.dd $(DMD)
 	$(DMD) -conf= -c -o- -Df$@ $(filter-out $(DMD),$^)
 
+################################################################################
+# HTML post-processing
+################################################################################
+
+# binary
+.PHONY: html-postprocessor
+html-postprocessor: ${DUB} ${STABLE_DMD}
+	DFLAGS="$(DPL_DOCS_DFLAGS)" ${DUB} build --root=${HTML_POSTPROCESSOR_PATH} \
+		--compiler=${STABLE_DMD}
+
+# post process html output (currently hyphenation)
+# process any html file that is newer than the rule's .done dummy file
+${HTML_POSTPROCESSOR_PATH}/.done: ${ALL_FILES} html-postprocessor \
+		html phobos-prerelease druntime-prerelease druntime-release phobos-release
+	if [ ! -f $@ ]; then touch -t 197001010000 $@; fi
+	find ${DOC_OUTPUT_DIR} -newer $@ -name '*.html' \
+	  -not \( -path '${DOC_OUTPUT_DIR}/library/*' -or -path '${DOC_OUTPUT_DIR}/library-prerelease/*' \) \
+	  | xargs ${HTML_POSTPROCESSOR}
+	touch $@
+
 ################################################################################
 # Dman tags
 ################################################################################

spec/iasm.dd

@@ -461,7 +461,7 @@ $(DL
 
 $(H2 $(LNAME2 supported_opcodes, Opcodes Supported))
 
-    $(LONGTABLE_5COLS Opcodes, ,
+    $(DIVC donthyphenate, $(LONGTABLE_5COLS Opcodes, ,
         $(TROW $(TT aaa)
         , $(TT aad)
         , $(TT aam)
@@ -1013,11 +1013,11 @@ $(H2 $(LNAME2 supported_opcodes, Opcodes Supported))
         , $(TT  )
         , $(TT  )
         )
-    )
+    ))
 
 $(H3 $(LNAME2 P4_opcode_support, Pentium 4 (Prescott) Opcodes Supported))
 
-    $(LONGTABLE_5COLS Pentium 4 Opcodes, ,
+    $(DIVC donthyphenate, $(LONGTABLE_5COLS Pentium 4 Opcodes, ,
         $(TROW $(TT addsubpd)
         , $(TT addsubps)
         , $(TT fisttp)
@@ -1034,11 +1034,11 @@ $(H3 $(LNAME2 P4_opcode_support, Pentium 4 (Prescott) Opcodes Supported))
         , $(TT  )
         , $(TT  )
         )
-    )
+    ))
 
 $(H3 $(LNAME2 amd_opcode_support, AMD Opcodes Supported))
 
-    $(LONGTABLE_5COLS AMD Opcodes, ,
+    $(DIVC donthyphenate, $(LONGTABLE_5COLS AMD Opcodes, ,
         $(TROW $(TT pavgusb)
         , $(TT pf2id)
         , $(TT pfacc)
@@ -1065,7 +1065,7 @@ $(H3 $(LNAME2 amd_opcode_support, AMD Opcodes Supported))
         , $(TT  )
         , $(TT  )
         )
-    )
+    ))
 
 $(H3 $(LNAME2 simd, SIMD))
 
@@ -1163,4 +1163,3 @@ getsec
 
 Macros:
         TITLE=Inline Assembler
-