doc: fix graphviz scanning and processing

The doc build process copies files using script/extract-content.py from
outside of the doc/ folder (specifically content in the tools/ folders).
The script was not copying graphviz directive files.  This has been
fixed and the embedded graphviz directives are not (properly) stored in
separate image/*.dot files.

Note the extract-content.py file is derived from the Zephyr project.

Signed-off-by: David B. Kinder <david.b.kinder@intel.com>

Author: dbkinder

doc/Makefile

@@ -42,7 +42,7 @@ doxy:
 	$(Q)(cat acrn.doxyfile) | doxygen - > doc.log 2>&1
 
 content:
-	$(Q)scripts/extract_content.py
+	$(Q)scripts/extract_content.py . tools
 
 kconfig:
 	$(Q)srctree=../hypervisor \

doc/scripts/extract_content.py

@@ -1,62 +1,113 @@
 #!/usr/bin/env python3
 #
+# Copyright (c) 2018, Nordic Semiconductor ASA
 # Copyright (c) 2017, Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 
-# Script to move docs from different places into the doc directory
+# Very quick script to move docs from different places into the doc directory
 # to fix the website and external links
 
+import argparse
+import errno
+import filecmp
+import fnmatch
 import os
-import shutil
 import re
+import shutil
 import sys
-import fnmatch
-
-
-# direcories to search for .rst files
-CONTENT_DIRS = ["tools"]
 
 # directives to parse for included files
-DIRECTIVES = ["figure","include","image","literalinclude"]
+DIRECTIVES = ["figure","include","image","literalinclude", "graphviz"]
+
+ZEPHYR_BASE = "../"
+ZEPHYR_BUILD = None
 
-# should get this from the environment...
-ACRN_BASE = "../"
+def copy_if_different(src, dst):
+    # Copies 'src' as 'dst', but only if dst does not exist or if itx contents
+    # differ from src.This avoids unnecessary # timestamp updates, which
+    # trigger documentation rebuilds.
+    if os.path.exists(dst) and filecmp.cmp(src, dst):
+        return
+    shutil.copyfile(src, dst)
 
-def get_rst_files(dir):
+def get_files(all, dest, dir):
     matches = []
-    for root, dirnames, filenames in os.walk('%s/%s' %(ACRN_BASE, dir)):
-        for filename in fnmatch.filter(filenames, '*.rst'):
+    for root, dirnames, filenames in os.walk('%s/%s' %(ZEPHYR_BASE, dir)):
+        if ZEPHYR_BUILD:
+            if os.path.normpath(root).startswith(os.path.normpath(ZEPHYR_BUILD)):
+                # Build folder, skip it
+                continue
+
+        for filename in fnmatch.filter(filenames, '*' if all else '*.rst'):
             matches.append(os.path.join(root, filename))
     for file in matches:
-        frel = file.replace(ACRN_BASE,"").strip("/")
+        frel = file.replace(ZEPHYR_BASE,"").strip("/")
         dir=os.path.dirname(frel)
-        if not os.path.exists(os.path.join(ACRN_BASE, "doc", dir)):
-            os.makedirs(os.path.join(ACRN_BASE, "doc", dir))
-
-        shutil.copyfile(file, os.path.join(ACRN_BASE, "doc", frel))
-
-        with open(file, encoding="utf-8") as f:
-            content = f.readlines()
-        content = [x.strip() for x in content]
-        directives = "|".join(DIRECTIVES)
-        pattern = re.compile("\s*\.\.\s+(%s)::\s+(.*)" %directives)
-        for l in content:
-            m = pattern.match(l)
-            if m:
-                inf = m.group(2)
-                ind = os.path.dirname(inf)
-                if not os.path.exists(os.path.join(ACRN_BASE, "doc", dir, ind)):
-                    os.makedirs(os.path.join(ACRN_BASE, "doc", dir, ind))
-
-                shutil.copyfile(os.path.join(ACRN_BASE, dir, inf),
-                        os.path.join(ACRN_BASE, "doc", dir, inf))
+        if not os.path.exists(os.path.join(dest, dir)):
+            os.makedirs(os.path.join(dest, dir))
+
+        copy_if_different(file, os.path.join(dest, frel))
+
+        # Inspect only .rst files for directives referencing other files
+        # we'll need to copy (as configured in the DIRECTIVES variable)
+        if not fnmatch.fnmatch(file, "*.rst"):
+            continue
+
+        try:
+            with open(file, encoding="utf-8") as f:
+                content = f.readlines()
+
+            content = [x.strip() for x in content]
+            directives = "|".join(DIRECTIVES)
+            pattern = re.compile("\s*\.\.\s+(%s)::\s+(.*)" %directives)
+            for l in content:
+                m = pattern.match(l)
+                if m:
+                    inf = m.group(2)
+                    ind = os.path.dirname(inf)
+                    if not os.path.exists(os.path.join(dest, dir, ind)):
+                        os.makedirs(os.path.join(dest, dir, ind))
+
+                    src = os.path.join(ZEPHYR_BASE, dir, inf)
+                    dst = os.path.join(dest, dir, inf)
+                    try:
+                        copy_if_different(src, dst)
+
+                    except FileNotFoundError:
+                        sys.stderr.write("File not found: %s\n  reference by %s\n" % (inf, file))
+
+        except UnicodeDecodeError as e:
+            sys.stderr.write(
+                "Malformed {} in {}\n"
+                "  Context: {}\n"
+                "  Problematic data: {}\n"
+                "  Reason: {}\n".format(
+                    e.encoding, file,
+                    e.object[max(e.start - 40, 0):e.end + 40],
+                    e.object[e.start:e.end],
+                    e.reason))
 
         f.close()
 
 def main():
-    for d in CONTENT_DIRS:
-        get_rst_files(d)
+
+    parser = argparse.ArgumentParser(description='Recursively copy .rst files '
+                                     'from the origin folder(s) to the '
+                                     'destination folder, plus files referenced '
+                                     'in those .rst files by a configurable '
+                                     'list of directives: {}.'.format(DIRECTIVES))
+
+    parser.add_argument('-a', '--all', action='store_true', help='Copy all files '
+                        '(recursively) in the specified source folder(s).')
+    parser.add_argument('dest', nargs=1)
+    parser.add_argument('src', nargs='+')
+    args = parser.parse_args()
+
+    dest = args.dest[0]
+
+    for d in args.src:
+        get_files(args.all, dest, d)
 
 if __name__ == "__main__":
     main()

tools/acrn-crashlog/README.rst

@@ -79,20 +79,11 @@ telemetrics-client on the system:
    telemetrics daemon. The work flow of ``acrnprobe`` and
    telemetrics-client is shown in :numref:`crashlog-workflow`:
 
-.. graphviz::
+.. graphviz:: images/crashlog-workflow.dot
    :name: crashlog-workflow
    :align: center
    :caption: acrnprobe and telemetrics-client workflow
 
-    digraph {
-       bgcolor=transparent; rankdir=LR;
-       node [shape="rectangle" style="filled" color="lightblue"]
-       edge [fontsize="12" fontcolor="blue"]
-
-       "acrnprobe" -> "telemetrics-client" [label="crashlog\npath"]
-       "telemetrics-client" -> "backend"   [label="log\ncontent"]
-    }
-
 
 Crashlog can be retrieved with ``telem_journal`` command:
 

tools/acrn-crashlog/acrnprobe/conf.rst

@@ -97,29 +97,10 @@ D inherit from crash A, and crash C is the child of crash B.
 Build crash tree in configuration
 =================================
 
-.. graphviz::
-
-   digraph {
-      {
-         node [shape=plaintext];
-         "level 1" -> "level 2" -> "level 3";
-      }
-
-      node [shape=box;style="rounded,filled";color=AntiqueWhite;];
-      c1 [ label="crash A\nid 1\ncrash root" ];
-      c2 [ label="crash B\nid 2" ];
-      c3 [ label="crash C\nid 3\ncrash leaf" ];
-      c4 [ label="crash D\nid 4\ncrash leaf" ];
-      c5 [ label="crash E\nid 5\ncrash root\ncrash leaf" ];
-      { rank = same; "level 1"; c1; c5;}
-      { rank = same; "level 2"; c2; c4;}
-      { rank = same; "level 3"; c3;}
-
-      node [shape=box;color="transparent";];
-      "None" -> {c1 c5} [ label="inherit 0" ];
-      c1 -> {c2 c4} [ label="inherit 1" ];
-      c2 -> c3 [ label="inherit 2" ];
-   }
+.. graphviz:: images/crash-config.dot
+   :name: crash-config
+   :align: center
+   :caption: Build crash tree in configuration
 
 Match crash at runtime
 ======================
@@ -136,33 +117,10 @@ If a crash A is triggered, then the candidates are crash A, B, C and D.
 The following diagram describes what ``acrnprobe`` will do if the matched
 result is crash D.
 
-.. graphviz::
-
-   digraph {
-      {
-         node [shape=plaintext];
-         "level 1" -> "level 2" -> "level 3";
-      }
-
-      node [shape=box;style="rounded,filled";color=AntiqueWhite;];
-      c1 [ label="crash A\nid 1\ncrash root" ];
-      c2 [ label="crash B\nid 2" ];
-      c3 [ label="crash C\nid 3\ncrash leaf" ];
-      c4 [ label="crash D\nid 4\ncrash leaf" ];
-      { rank = same; "level 1"; c1;}
-      { rank = same; "level 2"; c2; c4;}
-      { rank = same; "level 3"; c3;}
-
-      node [shape=box;style="rounded,dashed";];
-      exp1 [ label="crash B matches fail\nmatch for the next child\nof crash A"];
-      exp2 [ label="crash D matches successfully\nreturn crash D"];
-
-      node [shape=box;style="invis";];
-      "channel" -> c1 [ label="trigger" ]
-      c1 -> {exp1 exp2}
-      exp1 -> c2 -> c3 [ style=dashed dir=none]
-      exp2 -> c4
-   }
+.. graphviz:: images/crash-match.dot
+   :name: crash-match
+   :align: center
+   :caption: Match crash at runtime
 
 Sections
 ********

tools/acrn-crashlog/acrnprobe/images/crash-config.dot

@@ -0,0 +1,21 @@
+digraph {
+  {
+     node [shape=plaintext];
+     "level 1" -> "level 2" -> "level 3";
+  }
+
+  node [shape=box;style="rounded,filled";color=AntiqueWhite;];
+  c1 [ label="crash A\nid 1\ncrash root" ];
+  c2 [ label="crash B\nid 2" ];
+  c3 [ label="crash C\nid 3\ncrash leaf" ];
+  c4 [ label="crash D\nid 4\ncrash leaf" ];
+  c5 [ label="crash E\nid 5\ncrash root\ncrash leaf" ];
+  { rank = same; "level 1"; c1; c5;}
+  { rank = same; "level 2"; c2; c4;}
+  { rank = same; "level 3"; c3;}
+
+  node [shape=box;color="transparent";];
+  "None" -> {c1 c5} [ label="inherit 0" ];
+  c1 -> {c2 c4} [ label="inherit 1" ];
+  c2 -> c3 [ label="inherit 2" ];
+}

tools/acrn-crashlog/acrnprobe/images/crash-match.dot

@@ -0,0 +1,26 @@
+digraph {
+  {
+     node [shape=plaintext];
+     "level 1" -> "level 2" -> "level 3";
+  }
+
+  node [shape=box;style="rounded,filled";color=AntiqueWhite;];
+  c1 [ label="crash A\nid 1\ncrash root" ];
+  c2 [ label="crash B\nid 2" ];
+  c3 [ label="crash C\nid 3\ncrash leaf" ];
+  c4 [ label="crash D\nid 4\ncrash leaf" ];
+  { rank = same; "level 1"; c1;}
+  { rank = same; "level 2"; c2; c4;}
+  { rank = same; "level 3"; c3;}
+
+  node [shape=box;style="rounded,dashed";];
+  exp1 [ label="crash B matches fail\nmatch for the next child\nof crash A"];
+  exp2 [ label="crash D matches successfully\nreturn crash D"];
+
+  node [shape=box;style="invis";];
+  "channel" -> c1 [ label="trigger" ]
+  c1 -> {exp1 exp2}
+  exp1 -> c2 -> c3 [ style=dashed dir=none]
+  exp2 -> c4
+}
+

tools/acrn-crashlog/images/crashlog-workflow.dot

@@ -0,0 +1,8 @@
+digraph {
+   bgcolor=transparent; rankdir=LR;
+   node [shape="rectangle" style="filled" color="lightblue"]
+   edge [fontsize="12" fontcolor="blue"]
+
+   "acrnprobe" -> "telemetrics-client" [label="crashlog\npath"]
+   "telemetrics-client" -> "backend"   [label="log\ncontent"]
+}

tools/acrn-crashlog/usercrash/README.rst

@@ -7,7 +7,7 @@ Description
 ***********
 
 The ``usercrash`` tool gets the crash info for the crashing process in
-userpace. The collected information is saved as usercrash_xx under
+userspace. The collected information is saved as usercrash_xx under
 ``/var/log/usercrashes/``.
 
 Design