Add feature to build json with html in the same build

readthedocs/doc_builder/backends/sphinx.py

@@ -8,6 +8,7 @@
     absolute_import, division, print_function, unicode_literals)
 
 import codecs
+import shutil
 import logging
 import os
 import sys
@@ -23,6 +24,7 @@
 from readthedocs.projects.exceptions import ProjectConfigurationError
 from readthedocs.projects.utils import safe_write
 from readthedocs.restapi.client import api
+from readthedocs.projects.models import Feature
 
 from ..base import BaseBuilder, restoring_chdir
 from ..constants import PDF_RE, SPHINX_STATIC_DIR, SPHINX_TEMPLATE_DIR
@@ -132,6 +134,11 @@ def get_config_params(self):
             'gitlab_version': remote_version,
             'gitlab_version_is_editable': gitlab_version_is_editable,
             'display_gitlab': display_gitlab,
+
+            # Features
+            'generate_json_artifacts': self.project.has_feature(
+                Feature.BUILD_JSON_ARTIFACTS_WITH_HTML
+            ),
         }
 
         finalize_sphinx_context_data.send(
@@ -216,6 +223,30 @@ def __init__(self, *args, **kwargs):
         super(HtmlBuilder, self).__init__(*args, **kwargs)
         self.sphinx_builder = 'readthedocs'
 
+    def move(self, **__):
+        super(HtmlBuilder, self).move()
+        if self.project.has_feature(Feature.BUILD_JSON_ARTIFACTS_WITH_HTML):
+            # Copy json artifacts to its own directory
+            # to keep compatibility with the older builder.
+            json_path = os.path.abspath(
+                os.path.join(self.old_artifact_path, '..', 'json')
+            )
+            json_path_target = self.project.artifact_path(
+                version=self.version.slug, type_='sphinx_search'
+            )
+            if os.path.exists(json_path):
+                if os.path.exists(json_path_target):
+                    shutil.rmtree(json_path_target)
+                log.info('Copying json on the local filesystem')
+                shutil.copytree(
+                    json_path,
+                    json_path_target
+                )
+            else:
+                log.warning(
+                    'Not moving json because the build dir is unknown.'
+                )
+
 
 class HtmlDirBuilder(HtmlBuilder):
     type = 'sphinx_htmldir'

readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl

@@ -140,3 +140,6 @@ if 'extensions' in globals():
     extensions.insert(0, "readthedocs_ext.readthedocs")
 else:
     extensions = ["readthedocs_ext.readthedocs"]
+
+# Build the json artifacts with the html build step
+rtd_generate_json_artifacts = {{ generate_json_artifacts }}

readthedocs/projects/models.py

@@ -1015,13 +1015,16 @@ def add_features(sender, **kwargs):
     ALLOW_DEPRECATED_WEBHOOKS = 'allow_deprecated_webhooks'
     PIP_ALWAYS_UPGRADE = 'pip_always_upgrade'
     SKIP_SUBMODULES = 'skip_submodules'
+    BUILD_JSON_ARTIFACTS_WITH_HTML = 'build_json_artifacts_with_html'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
         (USE_SETUPTOOLS_LATEST, _('Use latest version of setuptools')),
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
         (PIP_ALWAYS_UPGRADE, _('Always run pip install --upgrade')),
         (SKIP_SUBMODULES, _('Skip git submodule checkout')),
+        (BUILD_JSON_ARTIFACTS_WITH_HTML, _(
+            'Build the json artifacts with the html build step')),
     )
 
     projects = models.ManyToManyField(

readthedocs/projects/tasks.py

@@ -29,7 +29,7 @@
 
 from .constants import LOG_TEMPLATE
 from .exceptions import RepositoryError
-from .models import ImportedFile, Project, Domain
+from .models import ImportedFile, Project, Domain, Feature
 from .signals import before_vcs, after_vcs, before_build, after_build, files_changed
 from readthedocs.builds.constants import (LATEST,
                                           BUILD_STATE_CLONING,
@@ -671,7 +671,18 @@ def build_docs_html(self):
         return success
 
     def build_docs_search(self):
-        """Build search data with separate build."""
+        """
+        Build search data with separate build.
+
+        Unless the project has the feature to allow
+        building the JSON search artifacts in the html build step.
+        """
+        build_json_in_html_builder = self.project.has_feature(
+            Feature.BUILD_JSON_ARTIFACTS_WITH_HTML
+        )
+        if self.build_search and build_json_in_html_builder:
+            # Already built in the html step
+            return True
         if self.build_search and self.project.is_type_sphinx:
             return self.build_docs_class('sphinx_search')
         return False