Check in scripts to generate HTML versions of policies from Markdown

convert-all-policies.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+./convert-policy.py ../sg/SG\ Policy.md policy-template.html ../sg/policy-link-mapping.txt  > whatwg.org/sg-policy
+./convert-policy.py ../sg/Workstream\ Policy.md policy-template.html ../sg/policy-link-mapping.txt  > whatwg.org/workstream-policy
+./convert-policy.py ../sg/SG\ Agreement.md policy-template.html ../sg/policy-link-mapping.txt  > whatwg.org/sg-agreement
+./convert-policy.py ../sg/Principles.md policy-template.html ../sg/policy-link-mapping.txt  > whatwg.org/principles
+./convert-policy.py ../sg/IPR\ Policy.md policy-template.html ../sg/policy-link-mapping.txt  > whatwg.org/ipr-policy
+

convert-policy.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import argparse
+import markdown
+from mdx_partial_gfm import PartialGithubFlavoredMarkdownExtension
+import re
+import string
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Convert a Markdown-formatted WHATWG policy to HTML suitable for publishing on whatwg.org")
+    parser.add_argument('policy', metavar='POLICY', type=argparse.FileType(), help="The markdown policy document to convert")
+    parser.add_argument('template', metavar='TEMPLATE', type=argparse.FileType(), help="A template to add appropriate boilerplate to the policy")
+    parser.add_argument('link_mapping', metavar='MAPPING', type=argparse.FileType(), help="File defining link mappings")
+
+    args = parser.parse_args()
+
+    return (args.policy, args.template, args.link_mapping)
+
+
+def lower_headers(policy_markdown):
+    return re.sub(r'^#', '##', policy_markdown, flags=re.MULTILINE)
+
+
+def apply_link_mapping(policy_markdown, link_mapping):
+    mapping_pairs = [line.split('=',1) for line in link_mapping.split("\n") if len(line) > 0]
+    for mapping in mapping_pairs:
+        policy_markdown = policy_markdown.replace(mapping[0], mapping[1])
+
+    return policy_markdown
+
+
+def ascii_lower(str):
+    return "".join([char.lower() if char >= 'A' and char <= 'Z' else char for char in str])
+
+
+def header_text_to_id(header_text):
+    punctuation_regexp = r'[^\w\- ]'
+    header_id = ascii_lower(header_text)
+    header_id = re.sub(punctuation_regexp, '', header_id)
+    header_id = re.sub(' ', '-', header_id)
+
+    return header_id
+
+
+def add_one_header_anchor(line):
+    header_text = line.lstrip('#')
+    header_level = len(line) - len(header_text)
+
+    if header_level <= 2:
+        return line
+
+    header_text = header_text.lstrip(' ')
+    header_id = header_text_to_id(header_text)
+
+    return '<h{0} id="{1}">{2}<a class="self-link" href="#{1}"></a></h{0}>'.format(header_level, header_id, header_text)
+
+
+def add_header_anchors(policy_markdown):
+    return str.join('\n', [add_one_header_anchor(line) for line in policy_markdown.split('\n')])
+
+
+def rewrite_defs(policy_markdown):
+    return re.sub(r'<a id=([^>]*)>[*][*]([^*]*)[*][*]</a>', '<dfn id=\\1>\\2</dfn>', policy_markdown)
+
+
+def fix_nested_lists(policy_markdown):
+    return re.sub(r'^   1[.]', '    1.', policy_markdown, flags=re.MULTILINE)
+
+
+def preprocess_markdown(policy_markdown, link_mapping):
+    result = lower_headers(policy_markdown)
+    result = apply_link_mapping(result, link_mapping)
+    result = add_header_anchors(result)
+    result = rewrite_defs(result)
+    result = fix_nested_lists(result)
+
+    return result
+
+
+def markdown_title(policy_markdown):
+    for line in policy_markdown.split('\n'):
+        if line.startswith("#"):
+            header = line.lstrip("# ")
+            return "WHATWG " + header if "WHATWG" not in header else header
+
+    return "WHATWG Policies"
+
+
+def main():
+    (policy_file, template_file, link_mapping_file) = parse_arguments()
+    policy_markdown = policy_file.read().decode('utf-8')
+    template = template_file.read().decode('utf-8')
+    link_mapping = link_mapping_file.read().decode('utf-8')
+
+    preprocessed_policy_markdown = preprocess_markdown(policy_markdown, link_mapping)
+    title = markdown_title(preprocessed_policy_markdown)
+
+    policy_html = markdown.markdown(preprocessed_policy_markdown, extensions=[PartialGithubFlavoredMarkdownExtension()])
+
+    final_policy_html = template.replace("@POLICY_GOES_HERE@", policy_html)
+    final_policy_html = final_policy_html.replace("@TITLE_GOES_HERE@", title)
+
+    print(final_policy_html.encode('utf-8'))
+
+main()

policy-template.html

@@ -0,0 +1,25 @@
+<!doctype html>
+<html lang="en">
+ <head>
+  <title>@TITLE_GOES_HERE@</title>
+  <link rel="stylesheet" href="/style/tabbed-pages">
+  <link rel="icon" href="https://resources.whatwg.org/logo.svg">
+ </head>
+ <body>
+  <h1>
+   <span class="the">The</span>
+   <strong class="what">Web Hypertext Application Technology</strong>
+   <span class="wg">Working Group</span>
+  </h1>
+  <ul class="navigation">
+   <li><a href="/" rel="home">Home</a></li>
+   <li><a href="https://spec.whatwg.org/">Standards</a></li>
+   <li><a href="/faq">FAQ</a></li>
+   <li class="this"><a href="/policies">Policies</a></li>
+   <li><a href="https://participate.whatwg.org/">Participate</a></li>
+  </ul>
+
+@POLICY_GOES_HERE@
+
+ </body>
+</html>