[doctools] Checking chapter topic integrity

- 419 unique topics in ref/index-*
- 256 unique topics in ref/chap-*

So we have a gap to close.

doc/ref/chap-data-lang.md

@@ -18,7 +18,7 @@ This is a quick reference, not the official spec.
 
 ## J8 Strings
 
-### json-escape \n   
+<h3 id="json-escape">json-escape \n</h3>
 
 ### surrogate-pair
 
@@ -27,7 +27,7 @@ Inherited from JSON
 See [Surrogate Pair Blog
 Post](https://www.oilshell.org/blog/2023/06/surrogate-pair.html).
 
-### j8-escape \yff   
+<h3 id="j8-escape">j8-escape \yff</h3>
 
 ### j-prefix j""
 

doctools/make_help.py

@@ -366,8 +366,9 @@ def HelpTopics(s):
 class DocNode(object):
   """To visualize doc structure."""
 
-  def __init__(self, name):
+  def __init__(self, name, attrs=None):
     self.name = name
+    self.attrs = attrs  # for h2 and h3 links
     self.children = []
 
 
@@ -418,19 +419,12 @@ def CardsFromChapters(out_dir, tag_level, pages):
 
       topic_id = id_value if id_value else heading.replace(' ', '-')
 
-      # Debug Tree
-      if attrs:
-        a_str = ', '.join('%s=%s' % pair for pair in attrs)
-        a_str = '(%s)' % a_str
-      else:
-        a_str = ''
-
       if tag == 'h2':
-        h2 = DocNode('%s %s' % (heading, a_str))
+        h2 = DocNode(heading, attrs=attrs)
         page_node.children.append(h2)
         cur_h2_node = h2
       elif tag == 'h3':
-        h3 = DocNode('%s %s' % (heading, a_str))
+        h3 = DocNode(heading, attrs=attrs)
         cur_h2_node.children.append(h3)
 
       if tag != tag_level:

doctools/ref_check.py

@@ -4,15 +4,24 @@
 """
 from __future__ import print_function
 
+import collections
 import json
 import sys
 
+from doctools.util import log
+
 
 def PrintTree(node, f, indent=0):
   """
   Print DocNode tree in make_help.py
   """
-  print('%s%s' % (indent * '  ', node.name), file=f)
+  if node.attrs:
+    a_str = ', '.join('%s=%s' % pair for pair in node.attrs)
+    a_str = '(%s)' % a_str
+  else:
+    a_str = ''
+
+  print('%s%s %s' % (indent * '  ', node.name, a_str), file=f)
   for ch in node.children:
     PrintTree(ch, f, indent+1)
 
@@ -22,6 +31,81 @@ def Check(index_debug_info, chap_tree):
   from pprint import pprint
   pprint(index_debug_info)
 
-  PrintTree(chap_tree, sys.stdout)
+  sections = []
+  topics = []
+
+  for block in index_debug_info:
+    for line_info in block:
+      if line_info['section']:
+        sections.append(line_info['section'])
+      topics.extend(line_info['topics'])
+
+  log('Index stats')
+  log('  num blocks = %d', len(index_debug_info))
+  log('  num sections = %d', len(sections))
+  log('  num unique sections = %d', len(set(sections)))
+
+  # 449 topics, 419 unique topics!
+  log('  num topics = %d', len(topics))
+
+  index_topic_set = set(topics)
+  log('  num unique topics = %d', len(index_topic_set))
+
+  #PrintTree(chap_tree, sys.stdout)
+
+  num_chapters = 0
+  num_sections = 0
+  num_topics = 0
+
+  chap_topics = {}  # topic_id -> list of chapters
+
+  for chap in chap_tree.children:
+    num_chapters += 1
+
+    for section in chap.children:
+      num_sections += 1
+
+      for topic in section.children:
+        num_topics += 1
+
+        values = [v for k, v in topic.attrs if k == 'id']
+        if len(values) == 1:
+          topic_id = values[0]
+        else:
+          topic_id = topic.name
+
+        if topic_id not in chap_topics:
+          chap_topics[topic_id] = []
+        chap_topics[topic_id].append(chap.name)
+
+  num_sections = sum(len(child.children) for child in chap_tree.children)
+  num_sections = sum(len(child.children) for child in chap_tree.children)
+  log('Chapter stats')
+  log('  num chapters = %d', num_chapters)
+  log('  num_sections = %d', num_sections)
+  log('  num_topics = %d', num_topics)
+
+  chap_topic_set = set(chap_topics)
+  log('  num unique topics = %d', len(chap_topic_set))
+
+  not_linked_to = chap_topic_set - index_topic_set
+
+  assert 'j8-escape' in index_topic_set
+  assert 'j8-escape' in chap_topic_set
+
+  log('')
+  log('%d topics not linked to:', len(not_linked_to))
+  for topic_id in not_linked_to:
+    log('  %s in %s', topic_id, chap_topics[topic_id])
+
+
+  log('')
+  log('Topics in multiple chapters:')
+  for topic_id, chaps in chap_topics.iteritems():
+    if len(chaps) > 1:
+      print('%s %s' % (topic_id, chaps))
+
+
+
 
 # vim: sw=2