[WIP] parallel read

doc/extdev/appapi.rst

@@ -442,7 +442,8 @@ handlers to the events.  Example:
    Emitted after the environment has determined the list of all added and
    changed files and just before it reads them.  It allows extension authors to
    reorder the list of docnames (*inplace*) before processing, or add more
-   docnames that Sphinx did not consider changed.
+   docnames that Sphinx did not consider changed (but never add any docnames
+   that are not in ``env.found_docs``).
 
    You can also remove document names; do this with caution since it will make
    Sphinx treat changed files as unchanged.

doc/extdev/index.rst

@@ -33,6 +33,13 @@ as metadata of the extension.  Metadata keys currently recognized are:
 * ``'version'``: a string that identifies the extension version.  It is used for
   extension version requirement checking (see :confval:`needs_extensions`) and
   informational purposes.  If not given, ``"unknown version"`` is substituted.
+* ``'parallel_read_safe'``: a boolean that specifies if parallel reading of
+  source files can be used when the extension is loaded.  It defaults to
+  ``False``, i.e. you have to explicitly specify your extension to be
+  parallel-read-safe after checking that it is.
+* ``'parallel_write_safe'``: a boolean that specifies if parallel writing of
+  output files can be used when the extension is loaded.  Since extensions
+  usually don't negatively influence the process, this defaults to ``True``.
 
 APIs used for writing extensions
 --------------------------------

sphinx/application.py

@@ -53,6 +53,7 @@
     'env-before-read-docs': 'env, docnames',
     'source-read': 'docname, source text',
     'doctree-read': 'the doctree before being pickled',
+    'env-merge-info': 'env, read docnames, other env instance',
     'missing-reference': 'env, node, contnode',
     'doctree-resolved': 'doctree, docname',
     'env-updated': 'env',

sphinx/builders/__init__.py

@@ -238,18 +238,11 @@ def build(self, docnames, summary=None, method='update'):
         # while reading, collect all warnings from docutils
         warnings = []
         self.env.set_warnfunc(lambda *args: warnings.append(args))
-        self.info(bold('updating environment: '), nonl=1)
-        msg, length, iterator = self.env.update(self.config, self.srcdir,
-                                                self.doctreedir, self.app)
-        self.info(msg)
-        for docname in self.status_iterator(iterator, 'reading sources... ',
-                                            purple, length):
-            updated_docnames.add(docname)
-            # nothing further to do, the environment has already
-            # done the reading
+        updated_docnames = self.env.update(self.config, self.srcdir,
+                                           self.doctreedir, self.app)
+        self.env.set_warnfunc(self.warn)
         for warning in warnings:
             self.warn(*warning)
-        self.env.set_warnfunc(self.warn)
 
         doccount = len(updated_docnames)
         self.info(bold('looking for now-outdated files... '), nonl=1)
@@ -325,16 +318,24 @@ def write(self, build_docnames, updated_docnames, method='update'):
         # check for prerequisites to parallel build
         # (parallel only works on POSIX, because the forking impl of
         # multiprocessing is required)
-        if not (multiprocessing and
+        if (multiprocessing and
                 self.app.parallel > 1 and
                 self.allow_parallel and
                 os.name == 'posix'):
-            self._write_serial(sorted(docnames), warnings)
-        else:
-            # number of subprocesses is parallel-1 because the main process
-            # is busy loading doctrees and doing write_doc_serialized()
-            self._write_parallel(sorted(docnames), warnings,
-                                 nproc=self.app.parallel - 1)
+            for extname, md in self.app._extension_metadata.items():
+                par_ok = md.get('parallel_write_safe', True)
+                if not par_ok:
+                    self.app.warn('the %s extension is not safe for parallel '
+                                  'writing, doing serial read' % extname)
+                    break
+            else:  # means no break, means everything is safe
+                # number of subprocesses is parallel-1 because the main process
+                # is busy loading doctrees and doing write_doc_serialized()
+                self._write_parallel(sorted(docnames), warnings,
+                                     nproc=self.app.parallel - 1)
+                self.env.set_warnfunc(self.warn)
+                return
+        self._write_serial(sorted(docnames), warnings)
         self.env.set_warnfunc(self.warn)
 
     def _write_serial(self, docnames, warnings):

sphinx/domains/__init__.py

@@ -202,6 +202,14 @@ def clear_doc(self, docname):
         """Remove traces of a document in the domain-specific inventories."""
         pass
 
+    def merge_domaindata(self, docnames, otherdata):
+        """Merge in data regarding *docnames* from a different domaindata
+        inventory.
+        """
+        raise NotImplementedError('merge_domaindata must be implemented in %s '
+                                  'to be able to do parallel builds!' %
+                                  self.__class__)
+
     def process_doc(self, env, docname, document):
         """Process a document after it is read by the environment."""
         pass

sphinx/domains/c.py

@@ -269,6 +269,12 @@ def clear_doc(self, docname):
             if fn == docname:
                 del self.data['objects'][fullname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for fullname, (fn, objtype) in otherdata['objects'].items():
+            if fn in docnames:
+                self.data['objects'][fullname] = (fn, objtype)
+
     def resolve_xref(self, env, fromdocname, builder,
                      typ, target, node, contnode):
         # strip pointer asterisk

sphinx/domains/cpp.py

@@ -1836,6 +1836,12 @@ def clear_doc(self, docname):
             if data[0] == docname:
                 del self.data['objects'][fullname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for fullname, data in otherdata['objects'].items():
+            if data[0] in docnames:
+                self.data['objects'][fullname] = data
+
     def _resolve_xref_inner(self, env, fromdocname, builder,
                             target, node, contnode, warn=True):
         def _create_refnode(nameAst):

sphinx/domains/javascript.py

@@ -187,6 +187,12 @@ def clear_doc(self, docname):
             if fn == docname:
                 del self.data['objects'][fullname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for fullname, (fn, objtype) in otherdata['objects'].items():
+            if fn in docnames:
+                self.data['objects'][fullname] = (fn, objtype)
+
     def find_obj(self, env, obj, name, typ, searchorder=0):
         if name[-2:] == '()':
             name = name[:-2]

sphinx/domains/python.py

@@ -627,6 +627,15 @@ def clear_doc(self, docname):
             if fn == docname:
                 del self.data['modules'][modname]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates?
+        for fullname, (fn, objtype) in otherdata['objects'].items():
+            if fn in docnames:
+                self.data['objects'][fullname] = (fn, objtype)
+        for modname, data in otherdata['modules'].items():
+            if data[0] in docnames:
+                self.data['modules'][modname] = data
+
     def find_obj(self, env, modname, classname, name, type, searchmode=0):
         """Find a Python object for "name", perhaps using the given module
         and/or classname.  Returns a list of (name, object entry) tuples.

sphinx/domains/rst.py

@@ -123,6 +123,12 @@ def clear_doc(self, docname):
             if doc == docname:
                 del self.data['objects'][typ, name]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX check duplicates
+        for (typ, name), doc in otherdata['objects'].items():
+            if doc in docnames:
+                self.data['objects'][typ, name] = doc
+
     def resolve_xref(self, env, fromdocname, builder, typ, target, node,
                      contnode):
         objects = self.data['objects']

sphinx/domains/std.py

@@ -506,6 +506,21 @@ def clear_doc(self, docname):
             if fn == docname:
                 del self.data['anonlabels'][key]
 
+    def merge_domaindata(self, docnames, otherdata):
+        # XXX duplicates?
+        for key, data in otherdata['progoptions'].items():
+            if data[0] in docnames:
+                self.data['progoptions'][key] = data
+        for key, data in otherdata['objects'].items():
+            if data[0] in docnames:
+                self.data['objects'][key] = data
+        for key, data in otherdata['labels'].items():
+            if data[0] in docnames:
+                self.data['labels'][key] = data
+        for key, data in otherdata['anonlabels'].items():
+            if data[0] in docnames:
+                self.data['anonlabels'][key] = data
+
     def process_doc(self, env, docname, document):
         labels, anonlabels = self.data['labels'], self.data['anonlabels']
         for name, explicit in iteritems(document.nametypes):