[doctools] Start simpler in-process ref-check

It's easier just to load the index and chapters into the same process.

build/doc.sh

@@ -183,20 +183,10 @@ split-and-render() {
 
   # for ysh-tour code blocks
   local code_out=_tmp/code-blocks/$rel_path.txt
-
-  # for doc/ref/index-* link info
-  local debug_out=_tmp/doctools/$rel_path.json
-  mkdir -v -p $(dirname $code_out) $(dirname $debug_out)
-
-  # code block output for verifying 'tour' info
-  # TODO: 
-  # - JSON debug info for ref/index-*
-  #   - languages-chapter-links- is actually a code block
-  # - code blocks could be an array of JSON too?  That might make more sense
+  mkdir -v -p $(dirname $code_out)
 
   cmark \
     --code-block-output $code_out \
-    --debug-output $debug_out \
     ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
 
   log "$tmp_prefix -> (doctools/cmark) -> $out"
@@ -385,15 +375,16 @@ cards-from-chapters() {
 
   local py_out=$CODE_DIR/help_.py
 
-  mkdir -p _tmp/doctools
-  local debug_out=_tmp/doctools/chapter-links.json
-
-  _make-help cards-from-chapters $TEXT_DIR $py_out $debug_out h3 \
+  _make-help cards-from-chapters $TEXT_DIR $py_out h3 \
     $HTML_DIR/doc/ref/chap-*.html
 }
 
 ref-check() {
-  doctools/ref_check.py _tmp/doctools/doc/ref/*.json
+  ### Check indexes and chapters against each other
+
+  PYTHONPATH=. doctools/make_help.py ref-check \
+    doc/ref/index-*.md \
+    _release/VERSION/doc/ref/chap-*.html
 }
 
 tour() {

doctools/cmark.py

@@ -270,7 +270,10 @@ def _ApplyInsertions(lines, insertions, out_file):
     out_file.write(line)
 
 
-def Render(opts, meta, in_file, out_file, use_fastlex=True):
+def Render(opts, meta, in_file, out_file, use_fastlex=True, debug_out=None):
+  if debug_out is None:
+    debug_out = []
+
   # First convert to HTML
   html = md2html(in_file.read())
 
@@ -295,15 +298,9 @@ def Render(opts, meta, in_file, out_file, use_fastlex=True):
 
     # <code> blocks
     # Including class=language-oil-help-topics
-    debug_out = []
     html = oil_doc.HighlightCode(html, meta.get('default_highlighter'),
                                  debug_out=debug_out)
 
-    # Pass to highlight code.  Or could make another pass.
-    if len(debug_out) and opts.debug_output:
-      with open(opts.debug_output, 'w') as f:
-        json.dump(debug_out, f, indent=2)
-
   # h2 is the title.  h1 is unused.
   if opts.toc_tags:
     toc_tags = opts.toc_tags
@@ -356,10 +353,6 @@ def Options():
       '--code-block-output', dest='code_block_output',
       default=None,
       help='Extract and print code blocks to this file')
-  p.add_option(
-      '--debug-output', dest='debug_output',
-      default=None,
-      help='JSON debug output to this file')
 
   return p
 

doctools/make_help.py

@@ -371,21 +371,6 @@ def __init__(self, name):
     self.children = []
 
 
-def PrintTree(node, f, indent=0):
-  print('%s%s' % (indent * '  ', node.name), file=f)
-  for ch in node.children:
-    PrintTree(ch, f, indent+1)
-
-
-def PrintJsonTree(node, f, indent=0):
-  # TODO: machine format
-  # Or make this pickle?
-
-  print('%s%s' % (indent * '  ', node.name), file=f)
-  for ch in node.children:
-    PrintJsonTree(ch, f, indent+1)
-
-
 def CardsFromIndex(sh, out_prefix):
   sections = []
   for section_id, section_name, text in HelpTopics(sys.stdin.read()):
@@ -495,28 +480,54 @@ def main(argv):
 
     out_dir = argv[2]
     py_out = argv[3]
-    debug_out = argv[4]
-    tag_level = argv[5]  # h4 or h3
-    pages = argv[6:]
+    tag_level = argv[4]  # h4 or h3
+    pages = argv[5:]
 
     topics, debug_info = CardsFromChapters(out_dir, tag_level, pages)
     with open(py_out, 'w') as f:
       f.write('TOPICS = %s\n' % pprint.pformat(topics))
 
-    with open(debug_out, 'w') as f:
-      #PrintTree(debug_info, f)
-      PrintJsonTree(debug_info, f)
-
   elif action == 'ref-check':
+    from doctools import cmark
+    from doctools import oil_doc
+    from doctools import ref_check
+
+    chapters = []
+    index_debug_info = []
+
+    for path in argv[2:]:
+      filename = os.path.basename(path)
+
+      if filename.endswith('.md'):
+        assert filename.startswith('index-'), path
+
+        # First convert to HTML
+        with open(path) as in_file:
+          html = cmark.md2html(in_file.read())
+
+        # Now highlight code, which # which gives debug output for the
+        # language-chapter-links-*
+        html = oil_doc.HighlightCode(html, None,
+                                     debug_out=index_debug_info)
+
+      elif filename.endswith('.html'):
+        assert filename.startswith('chap-'), path
+
+        # . CardsFromChapters() on chap-*, which gives you debug_Info above
+        chapters.append(path)
+
+      else:
+        raise RuntimeError('Expected index-* or chap-*, got %r' % filename)
+
+    out_dir = '_tmp/doctools'  # UNUSED
+    topics, chap_tree = CardsFromChapters(out_dir, 'h3', chapters)
+    #print(topics)
+
+    ref_check.Check(index_debug_info, chap_tree)
+
+
     # TODO: check all docs
-    #
-    # 1. Render() on index-*, which can have debug output for the
-    #    language-chapter-links-*
-    #
-    # 2. CardsFromChapters() on chap-*, which gives you debug_Info above
-    #
     # 3. Ref Check
-    pass
 
   else:
     raise RuntimeError('Invalid action %r' % action)

doctools/ref_check.py

@@ -7,16 +7,21 @@
 import json
 import sys
 
-def main(argv):
-  for path in argv[1:]:
-    print(path)
-    with open(path) as f:
-      d = json.load(f)
-      print(d)
 
+def PrintTree(node, f, indent=0):
+  """
+  Print DocNode tree in make_help.py
+  """
+  print('%s%s' % (indent * '  ', node.name), file=f)
+  for ch in node.children:
+    PrintTree(ch, f, indent+1)
 
-if __name__ == '__main__':
-  main(sys.argv)
 
+def Check(index_debug_info, chap_tree):
+
+  from pprint import pprint
+  pprint(index_debug_info)
+
+  PrintTree(chap_tree, sys.stdout)
 
 # vim: sw=2