[doctools] ref-check gives TOC stats

499 topic links in TOC, and 470 unique ones.

How many sections?  Does that matter?

Remove toc-tea.md.

doctools/help_gen.py

@@ -394,6 +394,11 @@ def CardsFromIndex(sh, out_prefix):
 
 
 def CardsFromChapters(out_dir, tag_level, paths):
+  """
+  Args:
+    paths: list of chap-*.html to read
+  """
+
   # TODO:
   # - we only need a few fixed cards
   # - turn this into a dict with sections
@@ -580,22 +585,25 @@ def TopicMetadata():
     from doctools import ref_check
 
     chapters = []
-    index_debug_info = []
+    all_toc_nodes = []
 
     for path in argv[2:]:
       filename = os.path.basename(path)
 
       if filename.endswith('.md'):
-        assert filename.startswith('index-'), path
+        assert filename.startswith('toc-'), path
 
         # First convert to HTML
         with open(path) as in_file:
           html = cmark.md2html(in_file.read())
 
         # Now highlight code, which # which gives debug output for the
         # language-chapter-links-*
+
+        box_nodes = []
         html = oil_doc.HighlightCode(html, None,
-                                     debug_out=index_debug_info)
+                                     debug_out=box_nodes)
+        all_toc_nodes.append({'toc': filename, 'boxes': box_nodes})
 
       elif filename.endswith('.html'):
         assert filename.startswith('chap-'), path
@@ -604,13 +612,17 @@ def TopicMetadata():
         chapters.append(path)
 
       else:
-        raise RuntimeError('Expected index-* or chap-*, got %r' % filename)
+        raise RuntimeError('Expected toc-* or chap-*, got %r' % filename)
 
     # out_dir=None so we don't write anything
     topics, chap_tree = CardsFromChapters(None, 'h3', chapters)
-    #print(topics)
 
-    ref_check.Check(index_debug_info, chap_tree)
+    #log('%d chapters: %s', len(chapters), chapters[:5])
+    log('%d topics: %s', len(topics), topics.keys()[:10])
+
+    #pprint.pprint(index_debug_info)
+
+    ref_check.Check(all_toc_nodes, chap_tree)
 
 
     # TODO: check all docs

doctools/oil_doc.py

@@ -468,7 +468,10 @@ def HighlightCode(s, default_highlighter, debug_out=None):
 
               plugin = HelpTopicsPlugin(s, code_start_pos, slash_code_left, chapter)
               block_debug_info = plugin.PrintHighlighted(out)
-              debug_out.append(block_debug_info)
+
+              # e.g. these are links to cmd-lang within a block in toc-ysh
+              chap_block = {'to_chap': chapter, 'lines': block_debug_info}
+              debug_out.append(chap_block)
 
               out.SkipTo(slash_code_left)
 

doctools/ref_check.py

@@ -6,6 +6,7 @@
 
 import collections
 import json
+from pprint import pprint
 import sys
 
 from doctools.util import log
@@ -26,15 +27,46 @@ def PrintTree(node, f, indent=0):
     PrintTree(ch, f, indent+1)
 
 
-def Check(index_debug_info, chap_tree):
+def Check(all_toc_nodes, chap_tree):
 
-  from pprint import pprint
-  pprint(index_debug_info)
+  #pprint(all_toc_nodes)
 
-  sections = []
-  topics = []
+  #sections = []
+  all_topics = []
 
-  for block in index_debug_info:
+  section_check = collections.defaultdict(list)
+  toc_topic_check = collections.defaultdict(list)
+
+  for toc_node in all_toc_nodes:
+    toc = toc_node['toc']
+    print(toc)
+    for box_node in toc_node['boxes']:
+      print('  %s' % box_node['to_chap'])
+      for line_info in box_node['lines']:
+        section = line_info['section']
+        topics = line_info['topics']
+        for topic in topics:
+          toc_topic_check[topic].append(toc)
+        all_topics.extend(topics)
+
+        print('    %s: %s' % (section or '?', ' '.join(topics)))
+
+  log('')
+
+  log('Topics in TOC: %d', len(all_topics))
+  log('Unique topics in TOC: %d', len(set(all_topics)))
+  log('')
+
+  log('Duplicate topics in TOC:')
+  log('')
+  for topic in sorted(toc_topic_check):
+    toc_list = toc_topic_check[topic]
+    if len(toc_list) > 1:
+      log('%20s: %s', topic, ' '.join(toc_list))
+
+  return
+
+  for box_nodes in all_toc_nodes:
     for line_info in block:
       if line_info['section']:
         sections.append(line_info['section'])
@@ -51,7 +83,9 @@ def Check(index_debug_info, chap_tree):
   index_topic_set = set(topics)
   log('  num unique topics = %d', len(index_topic_set))
 
-  if 1:
+  return
+
+  if 0:
     PrintTree(chap_tree, sys.stdout)
 
   num_chapters = 0
@@ -94,11 +128,11 @@ def Check(index_debug_info, chap_tree):
   assert 'j8-escape' in index_topic_set
   assert 'j8-escape' in chap_topic_set
 
-  log('')
-  log('%d topics not linked to:', len(not_linked_to))
-  for topic_id in not_linked_to:
-    log('  %s in %s', topic_id, chap_topics[topic_id])
-
+  if 0:
+    log('')
+    log('%d topics not linked to:', len(not_linked_to))
+    for topic_id in not_linked_to:
+      log('  %s in %s', topic_id, chap_topics[topic_id])
 
   log('')
   log('Topics in multiple chapters:')