[doctools] Export debug info for new ref_check.py tool

build/doc.sh

@@ -181,10 +181,24 @@ split-and-render() {
   #head _tmp/doc/*
   #return
 
+  # for ysh-tour code blocks
   local code_out=_tmp/code-blocks/$rel_path.txt
-  mkdir -v -p $(dirname $code_out)
 
-  cmark --code-block-output $code_out ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
+  # for doc/ref/index-* link info
+  local debug_out=_tmp/doctools/$rel_path.json
+  mkdir -v -p $(dirname $code_out) $(dirname $debug_out)
+
+  # code block output for verifying 'tour' info
+  # TODO: 
+  # - JSON debug info for ref/index-*
+  #   - languages-chapter-links- is actually a code block
+  # - code blocks could be an array of JSON too?  That might make more sense
+
+  cmark \
+    --code-block-output $code_out \
+    --debug-output $debug_out \
+    ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
+
   log "$tmp_prefix -> (doctools/cmark) -> $out"
 }
 
@@ -370,10 +384,18 @@ cards-from-chapters() {
   ### Turn h3 topics into cards
 
   local py_out=$CODE_DIR/help_.py
-  _make-help cards-from-chapters $TEXT_DIR $py_out h3 \
+
+  mkdir -p _tmp/doctools
+  local debug_out=_tmp/doctools/chapter-links.json
+
+  _make-help cards-from-chapters $TEXT_DIR $py_out $debug_out h3 \
     $HTML_DIR/doc/ref/chap-*.html
 }
 
+ref-check() {
+  doctools/ref_check.py _tmp/doctools/doc/ref/*.json
+}
+
 tour() {
   ### Build the Oil Language Tour and execute code
   local name=${1:-ysh-tour}

doctools/cmark.py

@@ -271,7 +271,6 @@ def _ApplyInsertions(lines, insertions, out_file):
 
 
 def Render(opts, meta, in_file, out_file, use_fastlex=True):
-
   # First convert to HTML
   html = md2html(in_file.read())
 
@@ -296,7 +295,14 @@ def Render(opts, meta, in_file, out_file, use_fastlex=True):
 
     # <code> blocks
     # Including class=language-oil-help-topics
-    html = oil_doc.HighlightCode(html, meta.get('default_highlighter'))
+    debug_out = []
+    html = oil_doc.HighlightCode(html, meta.get('default_highlighter'),
+                                 debug_out=debug_out)
+
+    # Pass to highlight code.  Or could make another pass.
+    if len(debug_out) and opts.debug_output:
+      with open(opts.debug_output, 'w') as f:
+        json.dump(debug_out, f, indent=2)
 
   # h2 is the title.  h1 is unused.
   if opts.toc_tags:
@@ -348,8 +354,12 @@ def Options():
 
   p.add_option(
       '--code-block-output', dest='code_block_output',
-      default=False,
+      default=None,
       help='Extract and print code blocks to this file')
+  p.add_option(
+      '--debug-output', dest='debug_output',
+      default=None,
+      help='JSON debug output to this file')
 
   return p
 

doctools/make_help.py

@@ -87,7 +87,7 @@ def _StringToHref(s):
 
 X_LEFT_SPAN = '<span style="color: darkred">'
 
-def HighlightLine(chapter, line):
+def IndexLineToHtml(chapter, line, debug_out):
   """Convert a line of text to HTML.
 
   Topics are highlighted and X made red.
@@ -118,14 +118,20 @@ def HighlightLine(chapter, line):
   # Highlight [Section] at the start of a line.
   m = SECTION_RE.match(line, pos)
   if m:
-    href = _StringToHref(m.group(1))
+    section_name = m.group(1)
+    href = _StringToHref(section_name)
 
     out.PrintUntil(m.start(1))
     out.Print('<a href="%s#%s" class="level2">' % (html_page, href))
     out.PrintUntil(m.end(1))  # anchor
     out.Print('</a>')
 
     pos = m.end(0)  # ADVANCE
+  else:
+    section_name = None
+
+  line_info = {'section': section_name, 'topics': []}
+  debug_out.append(line_info)
 
   _WHITESPACE = re.compile(r'[ ]+')
   m = _WHITESPACE.match(line, pos)
@@ -148,6 +154,7 @@ def HighlightLine(chapter, line):
 
     # The linked topic
     topic = m.group(2)
+    line_info['topics'].append(topic)
 
     out.PrintUntil(m.start(2))
     out.Print('<a href="%s#%s">' % (html_page, topic))
@@ -307,8 +314,9 @@ def SplitIntoCards(heading_tags, contents):
 
 def HelpTopics(s):
   """
-  Given an HTML page like {osh,ysh}-help.html,
-  Yield groups (id, desc, block of text)
+  Given an HTML page like index-{osh,ysh}.html,
+
+  Yield groups (section_id, section_name, block of text)
   """
   tag_lexer = html.TagLexer(s)
 
@@ -355,18 +363,49 @@ def HelpTopics(s):
     pos = end_pos
 
 
-class Node(object):
+class DocNode(object):
   """To visualize doc structure."""
 
   def __init__(self, name):
     self.name = name
     self.children = []
 
 
-def PrintTree(node, indent=0):
-  print('%s%s' % (indent * '  ', node.name))
+def PrintTree(node, f, indent=0):
+  print('%s%s' % (indent * '  ', node.name), file=f)
   for ch in node.children:
-    PrintTree(ch, indent+1)
+    PrintTree(ch, f, indent+1)
+
+
+def PrintJsonTree(node, f, indent=0):
+  # TODO: machine format
+  # Or make this pickle?
+
+  print('%s%s' % (indent * '  ', node.name), file=f)
+  for ch in node.children:
+    PrintJsonTree(ch, f, indent+1)
+
+
+def CardsFromIndex(sh, out_prefix):
+  sections = []
+  for section_id, section_name, text in HelpTopics(sys.stdin.read()):
+    if 0:
+      log('section_id = %r', section_id)
+      log('section_name = %r', section_name)
+      log('')
+      #log('text = %r', text[:20])
+
+    topic = '%s-%s' % (sh, section_id)  # e.g. ysh-overview
+
+    path = os.path.join(out_prefix, topic)
+    with open(path, 'w') as f:
+      f.write('%s %s %s\n\n' % (ansi.REVERSE, section_name, ansi.RESET))
+      f.write(text)
+      f.write('\n')  # extra
+    log('  Wrote %s', path)
+    sections.append(section_id)
+
+  log('  (doctools/make_help) -> %d sections -> %s', len(sections), out_prefix)
 
 
 def CardsFromChapters(out_dir, tag_level, pages):
@@ -375,7 +414,7 @@ def CardsFromChapters(out_dir, tag_level, pages):
   # - turn this into a dict with sections
   topics = []
 
-  root_node = Node('/')
+  root_node = DocNode('/')
   cur_h2_node = None
 
   seen = set()
@@ -384,7 +423,7 @@ def CardsFromChapters(out_dir, tag_level, pages):
       contents = f.read()
 
     page_name = os.path.basename(page_path)
-    page_node = Node(page_name)
+    page_node = DocNode(page_name)
 
     cards = SplitIntoCards(['h2', 'h3', 'h4'], contents)
 
@@ -402,11 +441,11 @@ def CardsFromChapters(out_dir, tag_level, pages):
         a_str = ''
 
       if tag == 'h2':
-        h2 = Node('%s %s' % (heading, a_str))
+        h2 = DocNode('%s %s' % (heading, a_str))
         page_node.children.append(h2)
         cur_h2_node = h2
       elif tag == 'h3':
-        h3 = Node('%s %s' % (heading, a_str))
+        h3 = DocNode('%s %s' % (heading, a_str))
         cur_h2_node.children.append(h3)
 
       if tag != tag_level:
@@ -438,10 +477,7 @@ def CardsFromChapters(out_dir, tag_level, pages):
   log('%d pages -> (doctools/make_help) -> %d <h3> cards from %d <h2> sections to %s',
       len(pages), len(topics), num_sections, out_dir)
 
-  if 0:
-    PrintTree(root_node)
-
-  return topics
+  return topics, root_node
 
 
 def main(argv):
@@ -451,36 +487,25 @@ def main(argv):
     sh = argv[2]  # osh or ysh
     out_prefix = argv[3]
 
-    f = sys.stdout
-    groups = []
-    for group_id, group_desc, text in HelpTopics(sys.stdin.read()):
-      #log('group_id = %r', group_id)
-      #log('group_desc = %r', group_desc)
-      #log('text = %r', text)
-
-      topic = '%s-%s' % (sh, group_id)  # e.g. ysh-overview
-
-      path = os.path.join(out_prefix, topic)
-      with open(path, 'w') as f:
-        f.write('%s %s %s\n\n' % (ansi.REVERSE, group_desc, ansi.RESET))
-        f.write(text)
-        f.write('\n')  # extra
-      log('  Wrote %s', path)
-      groups.append(group_id)
-
-    log('  (doctools/make_help) -> %d groups', len(groups))
+    CardsFromIndex(sh, out_prefix)
 
   elif action == 'cards-from-chapters':
 
     out_dir = argv[2]
     py_out = argv[3]
-    tag_level = argv[4]  # h4 or h3
-    pages = argv[5:]
+    debug_out = argv[4]
+    tag_level = argv[5]  # h4 or h3
+    pages = argv[6:]
 
-    topics = CardsFromChapters(out_dir, tag_level, pages)
+    topics, debug_info = CardsFromChapters(out_dir, tag_level, pages)
     with open(py_out, 'w') as f:
       f.write('TOPICS = %s\n' % pprint.pformat(topics))
 
+    with open(debug_out, 'w') as f:
+      #PrintTree(debug_info, f)
+      PrintJsonTree(debug_info, f)
+
+
     # Process pages first, so you can parse 
     # <h4 class="discouarged oil-language osh-only bash ksh posix"></h4>
     #

doctools/oil_doc.py

@@ -269,13 +269,15 @@ def __init__(self, s, start_pos, end_pos, chapter):
   def PrintHighlighted(self, out):
     from doctools import make_help
 
+    debug_out = []
+
     pos = self.start_pos
     for line_end in Lines(self.s, self.start_pos, self.end_pos):
-      # NOTE: HighlightLine accepts an HTML ESCAPED line.  It's valid to just
+      # NOTE: IndexLineToHtml accepts an HTML ESCAPED line.  It's valid to just
       # add tags and leave everything alone.
       line = self.s[pos : line_end]
 
-      html_line = make_help.HighlightLine(self.chapter, line)
+      html_line = make_help.IndexLineToHtml(self.chapter, line, debug_out)
 
       if html_line is not None:
         out.PrintUntil(pos)
@@ -284,6 +286,8 @@ def PrintHighlighted(self, out):
 
       pos = line_end
 
+    return debug_out
+
 
 class PygmentsPlugin(_Plugin):
 
@@ -356,14 +360,17 @@ def SimpleHighlightCode(s):
 
 
 
-def HighlightCode(s, default_highlighter):
+def HighlightCode(s, default_highlighter, debug_out=None):
   """
   Algorithm:
   1. Collect what's inside <pre><code> ...
   2. Then read lines with ShPromptPlugin.
   3. If the line looks like a shell prompt and command, highlight them with
      <span>
   """
+  if debug_out is None:
+    debug_out = []
+
   f = cStringIO.StringIO()
   out = html.Output(s, f)
 
@@ -434,7 +441,7 @@ def HighlightCode(s, default_highlighter):
 
               out.SkipTo(slash_code_left)
 
-            elif css_class == 'language-oil':
+            elif css_class == 'language-ysh':
               # TODO: Write an Oil syntax highlighter.
               pass
 
@@ -446,7 +453,8 @@ def HighlightCode(s, default_highlighter):
               out.PrintUntil(code_start_pos)
 
               plugin = HelpTopicsPlugin(s, code_start_pos, slash_code_left, chapter)
-              plugin.PrintHighlighted(out)
+              block_debug_info = plugin.PrintHighlighted(out)
+              debug_out.append(block_debug_info)
 
               out.SkipTo(slash_code_left)
 

doctools/ref_check.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python2
+"""
+ref_check.py: Check Links
+"""
+from __future__ import print_function
+
+import json
+import sys
+
+def main(argv):
+  for path in argv[1:]:
+    print(path)
+    with open(path) as f:
+      d = json.load(f)
+      print(d)
+
+
+if __name__ == '__main__':
+  main(sys.argv)
+
+
+# vim: sw=2