Merged in feature/arbitrary-docker-image (pull request #35)

Support arbitrary Docker image

Dockerfile

@@ -1,5 +1,4 @@
 FROM ubuntu:16.04
-MAINTAINER Messense Lv "messense@icloud.com"
 
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8

badwolf/builder.py

@@ -3,9 +3,11 @@
 import os
 import io
 import time
+import base64
 import logging
 
 import deansi
+from six.moves import shlex_quote
 from flask import current_app, render_template, url_for
 from requests.exceptions import ReadTimeout
 from docker import Client
@@ -67,6 +69,7 @@ def run(self):
             return
 
         exit_code, output = self.run_in_container(docker_image_name)
+        logger.debug('Docker run output: %s', output)
         if exit_code == 0:
             # Success
             logger.info('Test succeed for repo: %s', self.context.repository)
@@ -92,25 +95,34 @@ def get_docker_image(self):
         output = []
         docker_image = self.docker.images(docker_image_name)
         if not docker_image or self.context.rebuild:
-            dockerfile = os.path.join(self.context.clone_path, self.spec.dockerfile)
             build_options = {
                 'tag': docker_image_name,
                 'rm': True,
                 'stream': True,
                 'decode': True,
                 'nocache': self.context.nocache,
             }
-            if not os.path.exists(dockerfile):
-                logger.warning(
-                    'No Dockerfile: %s found for repo: %s, using simple runner image',
-                    dockerfile,
-                    self.context.repository
-                )
-                dockerfile_content = 'FROM messense/badwolf-test-runner:python\n'
+            if self.spec.image:
+                from_image_name, from_image_tag = self.spec.image.split(':', 2)
+                logger.info('Pulling Docker image %s', self.spec.image)
+                self.docker.pull(from_image_name, tag=from_image_tag)
+                logger.info('Pulled Docker image %s', self.spec.image)
+                dockerfile_content = 'FROM {}\n'.format(self.spec.image)
                 fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
                 build_options['fileobj'] = fileobj
             else:
-                build_options['dockerfile'] = self.spec.dockerfile
+                dockerfile = os.path.join(self.context.clone_path, self.spec.dockerfile)
+                if os.path.exists(dockerfile):
+                    build_options['dockerfile'] = self.spec.dockerfile
+                else:
+                    logger.warning(
+                        'No Dockerfile: %s found for repo: %s, using simple runner image',
+                        dockerfile,
+                        self.context.repository
+                    )
+                    dockerfile_content = 'FROM messense/badwolf-test-runner:python\n'
+                    fileobj = io.BytesIO(dockerfile_content.encode('utf-8'))
+                    build_options['fileobj'] = fileobj
 
             build_success = False
             logger.info('Building Docker image %s', docker_image_name)
@@ -136,13 +148,13 @@ def get_docker_image(self):
         return docker_image_name, ''.join(output)
 
     def run_in_container(self, docker_image_name):
-        command = '/bin/sh -c badwolf-run'
         environment = {}
         if self.spec.environments:
             # TODO: Support run in multiple environments
             environment = self.spec.environments[0]
 
         # TODO: Add more test context related env vars
+        script = shlex_quote(to_text(base64.b64encode(to_binary(self.spec.shell_script))))
         environment.update({
             'DEBIAN_FRONTEND': 'noninteractive',
             'HOME': '/root',
@@ -153,14 +165,17 @@ def run_in_container(self, docker_image_name):
             'BADWOLF_COMMIT': self.commit_hash,
             'BADWOLF_BUILD_DIR': '/mnt/src',
             'BADWOLF_REPO_SLUG': self.context.repository,
+            'BADWOLF_SCRIPT': script,
         })
         environment.setdefault('TERM', 'xterm-256color')
         if self.context.pr_id:
             environment['BADWOLF_PULL_REQUEST'] = to_text(self.context.pr_id)
 
+        logger.debug('Docker container environment: \n %r', environment)
         container = self.docker.create_container(
             docker_image_name,
-            command=command,
+            entrypoint=['/bin/sh', '-c'],
+            command=['echo $BADWOLF_SCRIPT | base64 --decode | /bin/sh'],
             environment=environment,
             working_dir='/mnt/src',
             volumes=['/mnt/src'],

badwolf/spec.py

@@ -1,18 +1,26 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
 import io
+import base64
+import logging
 
 import yaml
 try:
     from yaml import CLoader as _Loader
 except ImportError:
     from yaml import Loader as _Loader
+from six.moves import shlex_quote
+from flask import render_template
 
-from badwolf.utils import ObjectDict
+from badwolf.utils import ObjectDict, to_text, to_binary
+
+
+logger = logging.getLogger(__name__)
 
 
 class Specification(object):
     def __init__(self):
+        self.image = None
         self.services = []
         self.scripts = []
         self.dockerfile = 'Dockerfile'
@@ -56,11 +64,16 @@ def parse(cls, conf):
                 key, val = env_str.split('=', 1)
                 env_map[key] = val
             env_map_list.append(env_map)
+        image = conf.get('image')
+        if image and ':' not in image:
+            # Ensure we have tag name in image
+            image = image + ':latest'
 
         linters = cls._parse_linters(cls._get_list(conf.get('linter', [])))
         privileged = conf.get('privileged', False)
 
         spec = cls()
+        spec.image = image
         spec.services = services
         spec.scripts = scripts
         spec.dockerfile = dockerfile.strip()
@@ -109,3 +122,29 @@ def is_branch_enabled(self, branch):
         if not self.branch:
             return True
         return branch in self.branch
+
+    @property
+    def shell_script(self):
+        def _trace(command):
+            return 'echo + {}\n{} '.format(
+                shlex_quote(command),
+                command
+            )
+
+        commands = []
+        after_success = [_trace(cmd) for cmd in self.after_success]
+        after_failure = [_trace(cmd) for cmd in self.after_failure]
+        for service in self.services:
+            commands.append(_trace('service {} start'.format(service)))
+        for script in self.scripts:
+            commands.append(_trace(script))
+
+        command_encoded = shlex_quote(to_text(base64.b64encode(to_binary('\n'.join(commands)))))
+        context = {
+            'command': command_encoded,
+            'after_success': '    \n'.join(after_success),
+            'after_failure': '    \n'.join(after_failure),
+        }
+        script = render_template('script.sh', **context)
+        logger.debug('Build script: \n%s', script)
+        return script

badwolf/templates/script.sh

@@ -0,0 +1,15 @@
+set +e
+echo {{ command }} | base64 --decode | /bin/sh
+SCRIPT_EXIT_CODE=$?
+set -e
+{% if after_success -%}
+if [ $SCRIPT_EXIT_CODE -eq 0 ]; then
+    {{ after_success }}
+fi
+{%- endif %}
+{% if after_failure -%}
+if [ $SCRIPT_EXIT_CODE -ne 0 ]; then
+    {{ after_failure }}
+fi
+{%- endif %}
+exit $SCRIPT_EXIT_CODE

docs/build.rst

@@ -14,6 +14,7 @@
 ============================= ===================== ===================================================================
 选项名                        类型                  说明
 ============================= ===================== ===================================================================
+image                         string                用于构建的 Docker 镜像，提供此选项时可不提供 `dockerfile` 选项
 dockerfile                    string                用于构建 Docker 镜像的 dockerfile 文件名称, 默认为 Dockerfile
 branch                        string/list           仅在这些分支上运行构建和测试
 script                        string/list           构建/测试的命令
@@ -27,15 +28,7 @@ notification.slack_webhook    string/list           Slack webhook 地址列表
 privileged                    boolean               使用特权模式启动 Docker 容器
 ============================= ===================== ===================================================================
 
-如果需要自定义 Docker 镜像中安装的软件、库等，需要在根目录中提供上述 `dockerfile` 字段配置的 dockerfile 文件名称，
-默认为 Dockerfile，需要将 Dockerfile 的 `FROM` 设置为以下几个选项：
-
-* `messense/badwolf-test-runner:base` ： 基于 Ubuntu 16.04 LTS 的基础镜像，仅包含 CI 必须的软件包。
-* `messense/badwolf-test-runner:python` ：包含 Python 2.6、2.7、3.4、3.5 和 pypy 的镜像。
-* `messense/badwolf-test-runner:node` ：包含 nodejs 的镜像
-* `messense/badwolf-test-runner:rust` ：包含 Rust 的镜像
-
-比如：`FROM messense/badwolf-test-runner:python`
+请注意，当 `image` 和 `dockerfile` 选项同时提供时， `image` 选项优先使用。
 
 然后，在 BitBucket 项目设置中配置 webhook，假设部署机器的可访问地址为：http://badwolf.example.com:8000，
 则 webhook 地址应配置为：`http://badwolf.example.com:8000/webhook/push`。

requirements.txt

@@ -18,6 +18,6 @@ bandit>=0.17.3
 restructuredtext-lint>=0.14.2
 pylint>=1.5.4
 certifi
-requests[security]<2.11
+requests[security]
 flake8-import-order>=0.9.2
 deansi>=1.2

test.Dockerfile

@@ -1,15 +1,33 @@
-FROM messense/badwolf-test-runner:python
-MAINTAINER Messense Lv "messense@icloud.com"
+FROM ubuntu:16.04
 
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 ENV NPM_CONFIG_LOGLEVEL warn
 
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - \
-    && apt-get install -y nodejs
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    ca-certificates \
+    shellcheck \
+    libffi-dev \
+    python \
+    python-dev \
+    python-pip \
+    python-pkg-resources \
+    python3 \
+    python3-dev \
+    python3-setuptools \
+    python3-pip \
+    python3-pkg-resources \
+    libssl-dev \
+    && pip3 install -U pip tox
 
-RUN add-apt-repository "deb http://archive.ubuntu.com/ubuntu trusty-backports restricted main universe" \
-    && apt-get update \
-    && apt-get install -y shellcheck libffi-dev \
-    && pip install -U pip tox \
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - \
+    && apt-get install -y nodejs \
     && npm config set color false -g \
-    && npm install -g jscs eslint csslint sass-lint jsonlint stylelint \
+    && npm install -g \
+    jscs eslint csslint sass-lint jsonlint stylelint \
+    eslint-plugin-react eslint-plugin-react-native \
+    babel-eslint \
     && rm -rf /var/lib/apt/list/* /tmp/* /var/tmp/*

tests/test_spec.py

@@ -203,3 +203,66 @@ def test_parse_privileged():
     f = io.StringIO(s)
     spec = Specification.parse_file(f)
     assert not spec.privileged
+
+
+def test_parse_image():
+    s = """script: ls"""
+    f = io.StringIO(s)
+    spec = Specification.parse_file(f)
+    assert not spec.image
+
+    s = """image: python"""
+    f = io.StringIO(s)
+    spec = Specification.parse_file(f)
+    assert spec.image == 'python:latest'
+
+    s = """image: python:2.7"""
+    f = io.StringIO(s)
+    spec = Specification.parse_file(f)
+    assert spec.image == 'python:2.7'
+
+
+def test_generate_script_full_feature(app):
+    s = """script:
+  - ls
+service:
+  - redis-server
+after_success:
+  - pwd
+after_failure:
+  - exit"""
+    f = io.StringIO(s)
+    spec = Specification.parse_file(f)
+    script = spec.shell_script
+    assert 'echo + pwd\npwd' in script
+    assert 'echo + exit\nexit' in script
+
+
+def test_generate_script_after_success(app):
+    s = """script:
+  - ls
+service:
+  - redis-server
+after_success:
+  - pwd"""
+    f = io.StringIO(s)
+    spec = Specification.parse_file(f)
+    script = spec.shell_script
+    assert 'echo + pwd\npwd' in script
+    assert 'if [ $SCRIPT_EXIT_CODE -eq 0 ]; then' in script
+    assert 'if [ $SCRIPT_EXIT_CODE -ne 0 ]; then' not in script
+
+
+def test_generate_script_after_failure(app):
+    s = """script:
+  - ls
+service:
+  - redis-server
+after_failure:
+  - exit"""
+    f = io.StringIO(s)
+    spec = Specification.parse_file(f)
+    script = spec.shell_script
+    assert 'echo + exit\nexit' in script
+    assert 'if [ $SCRIPT_EXIT_CODE -eq 0 ]; then' not in script
+    assert 'if [ $SCRIPT_EXIT_CODE -ne 0 ]; then' in script