Add docker image from the YAML config integration

docs/yaml-config.rst

@@ -69,6 +69,40 @@ The file option specified the Conda `environment file`_ to use.
 
 .. note:: Conda is only supported via the YAML file.
 
+
+build
+~~~~~
+
+The ``build`` block configures specific aspects of the documentation build.
+
+.. _yaml_build_image:
+
+build.image
+```````````
+
+
+* Default: :djangosetting:`DOCKER_IMAGE`
+* Options: ``1.0``, ``2.0``, ``latest``
+
+The build image to use for specific builds.
+This lets users specify a more experimental build image,
+if they want to be on the cutting edge.
+
+Certain Python versions require a certain build image,
+as defined here::
+
+* `'1.0': 2, 2.7, 3, 3.4`
+* `'2.0': 2, 2.7, 3, 3.5`
+* `'latest': 2, 2.7, 3, 3.3, 3.4, 3.5, 3.6`
+
+.. code-block:: yaml
+
+    build:
+        image: latest
+
+    python:
+        version: 3.6
+
 python
 ~~~~~~
 
@@ -85,15 +119,12 @@ This is the version of Python to use when building your documentation. If you
 specify only the major version of Python, the highest supported minor version
 will be selected.
 
-The supported Python versions depends on the version of the build image your
-project is using. The default build image that is used to build documentation
-contains support for Python ``2.7`` and ``3.5``.
-
-There is also an image in testing that supports Python versions ``2.7``,
-``3.3``, ``3.4``, ``3.5``, and ``3.6``. If you would like access to this build
-image, you can sign up for beta access here:
+.. warning:: 
 
-https://goo.gl/forms/AKEoeWHixlzVfqKT2
+    The supported Python versions depends on the version of the build image your
+    project is using. The default build image that is used to build documentation
+    contains support for Python ``2.7`` and ``3.5``. 
+    See the :ref:`yaml_build_image` for more information on supported Python versions.
 
 .. code-block:: yaml
 

readthedocs/doc_builder/config.py

@@ -8,7 +8,7 @@
 from readthedocs_build.config import load as load_config
 from readthedocs_build.config import BuildConfig, ConfigError, InvalidConfig
 
-from .constants import DOCKER_BUILD_IMAGES, DOCKER_IMAGE
+from .constants import DOCKER_IMAGE_SETTINGS, DOCKER_IMAGE
 
 
 class ConfigWrapper(object):
@@ -108,6 +108,13 @@ def formats(self):
             formats += ['pdf']
         return formats
 
+    @property
+    def build_image(self):
+        if self._project.container_image:
+            # Allow us to override per-project still
+            return self._project.container_image
+        return self._yaml_config['build']['image']
+
     # Not implemented until we figure out how to keep in sync with the webs.
     # Probably needs to be version-specific as well, not project.
     # @property
@@ -126,19 +133,21 @@ def load_yaml_config(version):
     parsing consistent between projects.
     """
     checkout_path = version.project.checkout_path(version.slug)
-    env_config = {}
 
     # Get build image to set up the python version validation. Pass in the
     # build image python limitations to the loaded config so that the versions
     # can be rejected at validation
-    build_image = DOCKER_BUILD_IMAGES.get(
-        version.project.container_image,
-        DOCKER_BUILD_IMAGES.get(DOCKER_IMAGE, None),
-    )
-    if build_image:
-        env_config = {
-            'python': build_image['python'],
+
+    img_name = version.project.container_image or DOCKER_IMAGE
+    env_config = {
+        'build': {
+            'image': img_name,
         }
+    }
+    img_settings = DOCKER_IMAGE_SETTINGS.get(img_name, None)
+    if img_settings:
+        env_config.update(img_settings)
+        env_config['DOCKER_IMAGE_SETTINGS'] = img_settings
 
     try:
         sphinx_env_config = env_config.copy()

readthedocs/doc_builder/constants.py

@@ -4,11 +4,14 @@
 from __future__ import (
     absolute_import, division, print_function, unicode_literals)
 
+import logging
 import os
 import re
 
 from django.conf import settings
 
+log = logging.getLogger(__name__)
+
 SPHINX_TEMPLATE_DIR = os.path.join(
     settings.SITE_ROOT,
     'readthedocs',
@@ -33,24 +36,17 @@
 )
 DOCKER_VERSION = getattr(settings, 'DOCKER_VERSION', 'auto')
 DOCKER_IMAGE = getattr(settings, 'DOCKER_IMAGE', 'readthedocs/build:2.0')
+DOCKER_IMAGE_SETTINGS = getattr(settings, 'DOCKER_IMAGE_SETTINGS', {})
+
+old_config = getattr(settings, 'DOCKER_BUILD_IMAGES', None)
+if old_config:
+    log.warning('Old config detected, DOCKER_BUILD_IMAGES->DOCKER_IMAGE_SETTINGS')
+    DOCKER_IMAGE_SETTINGS.update(old_config)
+
 DOCKER_LIMITS = {'memory': '200m', 'time': 600}
 DOCKER_LIMITS.update(getattr(settings, 'DOCKER_LIMITS', {}))
 
 DOCKER_TIMEOUT_EXIT_CODE = 42
 DOCKER_OOM_EXIT_CODE = 137
 
 DOCKER_HOSTNAME_MAX_LEN = 64
-
-# Build images
-DOCKER_BUILD_IMAGES = {
-    'readthedocs/build:1.0': {
-        'python': {'supported_versions': [2, 2.7, 3, 3.4]},
-    },
-    'readthedocs/build:2.0': {
-        'python': {'supported_versions': [2, 2.7, 3, 3.5]},
-    },
-    'readthedocs/build:latest': {
-        'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]},
-    },
-}
-DOCKER_BUILD_IMAGES.update(getattr(settings, 'DOCKER_BUILD_IMAGES', {}))

readthedocs/doc_builder/environments.py

@@ -294,11 +294,12 @@ class BuildEnvironment(object):
                               successful
     """
 
-    def __init__(self, project=None, version=None, build=None, record=True,
-                 environment=None, update_on_success=True):
+    def __init__(self, project=None, version=None, build=None, config=None,
+                 record=True, environment=None, update_on_success=True):
         self.project = project
         self.version = version
         self.build = build
+        self.config = config
         self.record = record
         self.environment = environment or {}
         self.update_on_success = update_on_success
@@ -526,6 +527,8 @@ def __init__(self, *args, **kwargs):
                 project_name=self.project.slug,
             )[:DOCKER_HOSTNAME_MAX_LEN]
         )
+        if self.config and self.config.build_image:
+            self.container_image = self.config.build_image
         if self.project.container_mem_limit:
             self.container_mem_limit = self.project.container_mem_limit
         if self.project.container_time_limit:

readthedocs/projects/tasks.py

@@ -245,7 +245,7 @@ def run_build(self, docker=False, record=True):
             env_cls = DockerEnvironment
         else:
             env_cls = LocalEnvironment
-        self.build_env = env_cls(project=self.project, version=self.version,
+        self.build_env = env_cls(project=self.project, version=self.version, config=self.config,
                                  build=self.build, record=record, environment=env_vars)
 
         # Environment used for building code, usually with Docker

readthedocs/rtd_tests/tests/test_config_wrapper.py

@@ -54,45 +54,37 @@ def test_python_supported_versions_default_image_1_0(self, load_config):
         self.assertEqual(load_config.call_count, 1)
         load_config.assert_has_calls([
             mock.call(path=mock.ANY, env_config={
-                'python': {'supported_versions': [2, 2.7, 3, 3.4]},
+                'build': {'image': 'readthedocs/build:1.0'},
                 'type': 'sphinx',
                 'output_base': '',
                 'name': mock.ANY
             }),
         ])
         self.assertEqual(config.python_version, 2)
 
+    def test_python_supported_versions_image_1_0(self, load_config):
+        load_config.side_effect = create_load()
+        self.project.container_image = 'readthedocs/build:1.0'
+        self.project.save()
+        config = load_yaml_config(self.version)
+        self.assertEqual(config._yaml_config.get_valid_python_versions(),
+                         [2, 2.7, 3, 3.4])
+
     def test_python_supported_versions_image_2_0(self, load_config):
         load_config.side_effect = create_load()
         self.project.container_image = 'readthedocs/build:2.0'
         self.project.save()
         config = load_yaml_config(self.version)
-        self.assertEqual(load_config.call_count, 1)
-        load_config.assert_has_calls([
-            mock.call(path=mock.ANY, env_config={
-                'python': {'supported_versions': [2, 2.7, 3, 3.5]},
-                'type': 'sphinx',
-                'output_base': '',
-                'name': mock.ANY
-            }),
-        ])
-        self.assertEqual(config.python_version, 2)
+        self.assertEqual(config._yaml_config.get_valid_python_versions(),
+                         [2, 2.7, 3, 3.5])
 
     def test_python_supported_versions_image_latest(self, load_config):
         load_config.side_effect = create_load()
         self.project.container_image = 'readthedocs/build:latest'
         self.project.save()
         config = load_yaml_config(self.version)
-        self.assertEqual(load_config.call_count, 1)
-        load_config.assert_has_calls([
-            mock.call(path=mock.ANY, env_config={
-                'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]},
-                'type': 'sphinx',
-                'output_base': '',
-                'name': mock.ANY
-            }),
-        ])
-        self.assertEqual(config.python_version, 2)
+        self.assertEqual(config._yaml_config.get_valid_python_versions(),
+                         [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6])
 
     def test_python_default_version(self, load_config):
         load_config.side_effect = create_load()

requirements/pip.txt

@@ -10,7 +10,9 @@ mkdocs==0.14.0
 django==1.9.12
 six==1.10.0
 future==0.16.0
-readthedocs-build==2.0.7
+#readthedocs-build==2.0.8
+# For testing
+git+https://github.com/rtfd/readthedocs-build.git@79f78d23d367f71#egg=readthedocs_build
 
 django-tastypie==0.13.0
 django-haystack==2.6.0